diff --git a/services/distributeddataservice/libs/distributeddb/BUILD.gn b/services/distributeddataservice/libs/distributeddb/BUILD.gn index e4e9652ffdbc4a9e427605e6562eee1d5f5a6f72..c763dcedcbfdd239139cd06b64260be4f4029553 100755 --- a/services/distributeddataservice/libs/distributeddb/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/BUILD.gn @@ -36,9 +36,10 @@ config("distrdb_config") { "_FILE_OFFSET_BITS=64", "SQLITE_HAS_CODEC", "SQLITE_ENABLE_JSON1", - "USE_SQLITE_SYMBOLS", "USING_HILOG_LOGGER", + "USE_SQLITE_SYMBOLS", "USING_DB_JSON_EXTRACT_AUTOMATICALLY", + "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", ] } @@ -58,6 +59,8 @@ group("build_module") { ohos_shared_library("distributeddb") { sources = [ "common/src/auto_launch.cpp", + "common/src/data_compression.cpp", + "common/src/db_ability.cpp", "common/src/db_common.cpp", "common/src/db_constant.cpp", "common/src/evloop/src/event_impl.cpp", @@ -90,6 +93,7 @@ ohos_shared_library("distributeddb") { "common/src/time_tick_monitor.cpp", "common/src/types_export.cpp", "common/src/value_object.cpp", + "common/src/zlib_compression.cpp", "communicator/src/combine_status.cpp", "communicator/src/communicator.cpp", "communicator/src/communicator_aggregator.cpp", @@ -102,6 +106,7 @@ ohos_shared_library("distributeddb") { "communicator/src/protocol_proto.cpp", "communicator/src/send_task_scheduler.cpp", "communicator/src/serial_buffer.cpp", + "interfaces/src/intercepted_data_impl.cpp", "interfaces/src/kv_store_changed_data_impl.cpp", "interfaces/src/kv_store_delegate_impl.cpp", "interfaces/src/kv_store_delegate_manager.cpp", @@ -143,6 +148,7 @@ ohos_shared_library("distributeddb") { "storage/src/result_entries_window.cpp", "storage/src/single_ver_natural_store_commit_notify_data.cpp", "storage/src/sqlite/query_object.cpp", + "storage/src/sqlite/query_sync_object.cpp", "storage/src/sqlite/sqlite_local_kvdb.cpp", "storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -150,6 +156,8 @@ ohos_shared_library("distributeddb") { "storage/src/sqlite/sqlite_local_storage_executor.cpp", "storage/src/sqlite/sqlite_multi_ver_data_storage.cpp", "storage/src/sqlite/sqlite_multi_ver_transaction.cpp", + "storage/src/sqlite/sqlite_query_helper.cpp", + "storage/src/sqlite/sqlite_single_ver_continue_token.cpp", "storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp", "storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "storage/src/sqlite/sqlite_single_ver_natural_store.cpp", @@ -159,6 +167,7 @@ ohos_shared_library("distributeddb") { "storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", "storage/src/sqlite/sqlite_single_ver_storage_executor.cpp", "storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp", + "storage/src/sqlite/sqlite_single_ver_storage_executor_subscribe.cpp", "storage/src/sqlite/sqlite_storage_engine.cpp", "storage/src/sqlite/sqlite_storage_executor.cpp", "storage/src/sqlite/sqlite_utils.cpp", @@ -171,6 +180,7 @@ ohos_shared_library("distributeddb") { "storage/src/upgrader/single_ver_schema_database_upgrader.cpp", "syncer/src/ability_sync.cpp", "syncer/src/commit_history_sync.cpp", + "syncer/src/communicator_proxy.cpp", "syncer/src/device_manager.cpp", "syncer/src/generic_syncer.cpp", "syncer/src/meta_data.cpp", @@ -179,8 +189,12 @@ ohos_shared_library("distributeddb") { "syncer/src/multi_ver_sync_state_machine.cpp", "syncer/src/multi_ver_sync_task_context.cpp", "syncer/src/multi_ver_syncer.cpp", + "syncer/src/query_sync_water_mark_helper.cpp", + "syncer/src/single_ver_data_packet.cpp", "syncer/src/single_ver_data_sync.cpp", "syncer/src/single_ver_data_sync_with_sliding_window.cpp", + "syncer/src/single_ver_kv_syncer.cpp", + "syncer/src/single_ver_serialize_manager.cpp", "syncer/src/single_ver_sync_engine.cpp", "syncer/src/single_ver_sync_state_machine.cpp", "syncer/src/single_ver_sync_target.cpp", @@ -188,6 +202,7 @@ ohos_shared_library("distributeddb") { "syncer/src/single_ver_syncer.cpp", "syncer/src/sliding_window_receiver.cpp", "syncer/src/sliding_window_sender.cpp", + "syncer/src/subscribe_manager.cpp", "syncer/src/sync_engine.cpp", "syncer/src/sync_operation.cpp", "syncer/src/sync_state_machine.cpp", @@ -205,6 +220,7 @@ ohos_shared_library("distributeddb") { deps = [ "//third_party/sqlite:sqlite", + "//third_party/zlib:libz", "//utils/native/base:utils", ] diff --git a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h index fb48035bc42db77d9949926a8b8f75917abc7479..53bed5681e2f487dc18cd85579147386e6e97d61 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/auto_launch.h @@ -47,6 +47,7 @@ struct AutoLaunchItem { AutoLaunchItemState state = AutoLaunchItemState::UN_INITIAL; bool isDisable = false; bool inObserver = false; + bool isAutoSync = true; }; class AutoLaunch { @@ -60,7 +61,7 @@ public: void SetCommunicatorAggregator(ICommunicatorAggregator *aggregator); int EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier); + const AutoLaunchOption &option); int DisableKvStoreAutoLaunch(const std::string &identifier); diff --git a/services/distributeddataservice/libs/distributeddb/common/include/data_compression.h b/services/distributeddataservice/libs/distributeddb/common/include/data_compression.h new file mode 100644 index 0000000000000000000000000000000000000000..6b39d2ea231e418619955ca7a93c31bf4c451b11 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/include/data_compression.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATA_COMPRESSION_H +#define DATA_COMPRESSION_H + +#include +#include +#include + +#include "types_export.h" + +namespace DistributedDB { +class DataCompression { +public: + static DataCompression *GetInstance(CompressAlgorithm algo); + static void GetCompressionAlgo(std::set &algorithmSet); + static int TransferCompressionAlgo(uint32_t compressAlgoType, CompressAlgorithm &algoType); + + virtual int Compress(const std::vector &srcData, std::vector &destData) const = 0; + virtual int Uncompress(const std::vector &srcData, std::vector &destData, unsigned long destLen) + const = 0; + +protected: + DataCompression() = default; + virtual ~DataCompression() = default; + DataCompression(const DataCompression& compression) = delete; + DataCompression& operator= (const DataCompression& compression) = delete; + + static void Register(CompressAlgorithm algo, DataCompression *compression); + +private: + static std::map &GetCompressionAlgos(); + static std::map &GetTransMap(); +}; +} // namespace DistributedDB +#endif // DATA_COMPRESSION_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_ability.h b/services/distributeddataservice/libs/distributeddb/common/include/db_ability.h new file mode 100644 index 0000000000000000000000000000000000000000..75466cfec6779b7ec1f742e3c1469a243446e0e9 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_ability.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DB_ABILITY_H +#define DB_ABILITY_H + +#include +#include +#include +#include "macro_utils.h" +#include "parcel.h" +#include "sync_config.h" + +namespace DistributedDB { +class DbAbility { +public: + DbAbility(); + DbAbility(const DbAbility &other); + DbAbility& operator=(const DbAbility &other); + ~DbAbility() {}; + + bool operator==(const DbAbility &other) const; + // translate dbAbility_ to std::vector + static int Serialize(Parcel &parcel, const DbAbility &curAbility); + + static int DeSerialize(Parcel &parcel, DbAbility &curAbility); + + static uint32_t CalculateLen(const DbAbility &curAbility); + + void SetDbAbilityBuff(std::vector &buff); + + const std::vector &GetDbAbilityBuff() const; + + uint32_t GetAbilityBitsLen() const; + + uint8_t GetAbilityItem(const AbilityItem abilityType) const; + + int SetAbilityItem(const AbilityItem &abilityType, uint8_t data); +private: + constexpr static int SERIALIZE_BIT_SIZE = 64; // uint64_t bit size + + std::vector dbAbility_; + std::set dbAbilityItemSet_; +}; +} // namespace DistributedDB + +#endif // DB_ABILITY_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_common.h b/services/distributeddataservice/libs/distributeddb/common/include/db_common.h index 840a3c8afae10dfcce2d8934f489967413e3b33b..3dedbdd0dfd1f067c875f0c9d4838df13fcafa39 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_common.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_common.h @@ -52,10 +52,13 @@ public: static void SetDatabaseIds(KvDBProperties &properties, const std::string &appId, const std::string &userId, const std::string &storeId); + + static std::string StringMasking(const std::string &oriStr, size_t remain = 3); // remain 3 unmask }; // Define short macro substitute for original long expression for convenience of using #define VEC_TO_STR(x) DBCommon::VectorToHexString(x).c_str() +#define STR_MASK(x) DBCommon::StringMasking(x).c_str() } // namespace DistributedDB #endif // DISTRIBUTEDDB_COMMON_H diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h index a2f2dbbc12574b32c9594f4a6d3d070bf70953d9..99a6944b9081d6801617d966b1cb8d3b0c0158ce 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_constant.h @@ -84,12 +84,31 @@ public: static const std::string ID_CONNECTOR; static const std::string DELETE_KVSTORE_REMOVING; + static const std::string DB_LOCK_POSTFIX; + + static const std::string SUBSCRIBE_QUERY_PREFIX; + static const std::string TRIGGER_REFERENCES_NEW; + static const std::string TRIGGER_REFERENCES_OLD; + + static const std::string UPDATE_META_FUNC; static constexpr uint32_t AUTO_SYNC_TIMEOUT = 5000; // 5s static constexpr uint32_t MANUAL_SYNC_TIMEOUT = 5000; // 5s static const size_t MAX_NORMAL_PACK_ITEM_SIZE = 4000; static const size_t MAX_HPMODE_PACK_ITEM_SIZE = 2000; // slide window mode to reduce last ack transfer time + + static constexpr uint32_t MIN_MTU_SIZE = 1024; // 1KB + static constexpr uint32_t MAX_MTU_SIZE = 5242880; // 5MB + + static constexpr uint32_t MIN_TIMEOUT = 5000; // 5s + static constexpr uint32_t MAX_TIMEOUT = 60000; // 60s + + static constexpr uint8_t DEFAULT_COMPTRESS_RATE = 100; + + static constexpr size_t MAX_SYNC_BLOCK_SIZE = 31457280; // 30MB + + static constexpr int DOUBLE_PRECISION = 15; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h index d0736fa6ba1d72bee0d9551040fc475ba8000fa4..5dd9a6e1fcb6c35721e30271ef252590329252e4 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_errno.h @@ -48,7 +48,7 @@ constexpr int E_DESERIALIZE_ERROR = (E_BASE + 24); // deserialize error constexpr int E_NOT_REGISTER = (E_BASE + 25); // handler or function not registered constexpr int E_LENGTH_ERROR = (E_BASE + 26); // error relative to length constexpr int E_UNFINISHED = (E_BASE + 27); // get sync data unfinished. -constexpr int E_FINISHED = (E_BASE + 28); // get sync data unfinished. +constexpr int E_FINISHED = (E_BASE + 28); // get sync data finished. constexpr int E_INVALID_MESSAGE_ID = (E_BASE + 29); // invalid messageId error constexpr int E_MESSAGE_ID_ERROR = (E_BASE + 30); // messageId is not expected constexpr int E_MESSAGE_TYPE_ERROR = (E_BASE + 31); // messageType is not expected @@ -115,8 +115,10 @@ constexpr int E_SYSTEM_API_ADAPTER_CALL_FAILED = (E_BASE + 91); // Adapter call constexpr int E_NOT_NEED_DELETE_MSG = (E_BASE + 92); // not need delete msg, will be delete by sliding window receiver constexpr int E_SLIDING_WINDOW_SENDER_ERR = (E_BASE + 93); // sliding window sender err constexpr int E_SLIDING_WINDOW_RECEIVER_INVALID_MSG = (E_BASE + 94); // sliding window receiver invalid msg -constexpr int E_IGNOR_DATA = (E_BASE + 95); // ignore the data changed by other devices. +constexpr int E_IGNOR_DATA = (E_BASE + 95); // ignore the data changed by other devices and ignore the same data. constexpr int E_FORBID_CACHEDB = (E_BASE + 96); // such after rekey can not check passwd due to file control. +constexpr int E_INTERCEPT_DATA_FAIL = (E_BASE + 97); // Intercept push data failed. +constexpr int E_INVALID_COMPRESS_ALGO = (E_BASE + 98); // The algo is defined, but there's no implement for the algo. // Num 150+ is reserved for schema related errno, since it may be added regularly constexpr int E_JSON_PARSE_FAIL = (E_BASE + 150); // Parse json fail in grammatical level constexpr int E_JSON_INSERT_PATH_EXIST = (E_BASE + 151); // Path already exist before insert diff --git a/services/distributeddataservice/libs/distributeddb/common/include/db_types.h b/services/distributeddataservice/libs/distributeddb/common/include/db_types.h index b9e7bc05c4f4cd5a3f5048350e73222206251239..697600aa8179561019f8988ddcd8b73d1447e96c 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/db_types.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/db_types.h @@ -50,6 +50,9 @@ struct DataItem { static constexpr uint64_t LOCAL_FLAG = 0x02; static constexpr uint64_t REMOVE_DEVICE_DATA_FLAG = 0x04; // only use for cachedb static constexpr uint64_t REMOVE_DEVICE_DATA_NOTIFY_FLAG = 0x08; // only use for cachedb + // Only use for query sync and subscribe. ATTENTION!!! this flag should not write into mainDB. + // Mark the changed row data does not match with query sync(or subscribe) condition. + static constexpr uint64_t REMOTE_DEVICE_DATA_MISS_QUERY = 0x10; }; struct PragmaPublishInfo { @@ -121,5 +124,4 @@ enum SingleVerConflictResolvePolicy { DENY_OTHER_DEV_AMEND_CUR_DEV_DATA = 1, }; } // namespace DistributedDB - #endif // DISTRIBUTEDDB_TYPES_H diff --git a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h index 1285b6279586afc72172020517414fde31658301..c9ecd26e6e7b1e79d01cf08151e9b1a207039285 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/json_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/json_object.h @@ -32,6 +32,7 @@ class JsonObject { public: // Set max allowed nest depth and return the value before set. static uint32_t SetMaxNestDepth(uint32_t nestDepth); + // Calculate nest depth when json string is legal or estimate depth by legal part from illegal json. static uint32_t CalculateNestDepth(const std::string &inString); static uint32_t CalculateNestDepth(const uint8_t *dataBegin, const uint8_t *dataEnd); @@ -41,32 +42,39 @@ public: ~JsonObject() = default; JsonObject(const JsonObject &); JsonObject& operator=(const JsonObject &); - // Move constructor and move assignment is not need currently - JsonObject(JsonObject &&) = delete; - JsonObject& operator=(JsonObject &&) = delete; + + explicit JsonObject(const Json::Value &value); // Should be called on an invalid JsonObject, create new JsonObject if need to reparse // Require the type of the root to be JsonObject, otherwise parse fail int Parse(const std::string &inString); int Parse(const std::vector &inData); // Whether ends with '\0' in vector is OK + // The end refer to the byte after the last valid byte int Parse(const uint8_t *dataBegin, const uint8_t *dataEnd); bool IsValid() const; - // Unnecessary spacing will be removed and fieldname resorted by lexicographical order + + // Unnecessary spacing will be removed and fieldName resorted by lexicographical order std::string ToString() const; bool IsFieldPathExist(const FieldPath &inPath) const; int GetFieldTypeByFieldPath(const FieldPath &inPath, FieldType &outType) const; int GetFieldValueByFieldPath(const FieldPath &inPath, FieldValue &outValue) const; - // An empty fieldpath indicate the root, the outSubPath should be empty before call, we will not empty it at first. + + int GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; + int GetStringArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const; + + // An empty fieldPath indicate the root, the outSubPath should be empty before call, we will not empty it at first. // If inPath is of multiple path, then outSubPath is combination of result of each inPath. int GetSubFieldPath(const FieldPath &inPath, std::set &outSubPath) const; int GetSubFieldPath(const std::set &inPath, std::set &outSubPath) const; int GetSubFieldPathAndType(const FieldPath &inPath, std::map &outSubPathType) const; int GetSubFieldPathAndType(const std::set &inPath, std::map &outSubPathType) const; + // If inPath not refer to an array, return error. int GetArraySize(const FieldPath &inPath, uint32_t &outSize) const; + // If inPath not refer to an array, return error. If not all members are string or array type, return error. // If array-type member is empty, ignore. If not all members of the array-type member are string, return error. int GetArrayContentOfStringOrStringArray(const FieldPath &inPath, @@ -74,21 +82,31 @@ public: // Can be called no matter JsonObject valid or not. Invalid turn into valid after call(insert on invalid never fail // if parameter is valid). An empty inPath is not allowed. LEAF_FIELD_ARRAY and INTERNAL_FIELD_OBJECT is not - // supported. infinite double is not support. inValue is ignored for LEAF_FIELD_NULL. If inPath already exist, - // returns -E_JSON_INSERT_PATH_EXIST, if nearest path ends with type not object, rets -E_JSON_INSERT_PATH_CONFLICT. - // Otherwise insert field as well as filling up intermediate field, then returns E_OK; - int InsertField(const FieldPath &inPath, FieldType inType, const FieldValue &inValue); + // supported. infinite double is not support. inValue is ignored for LEAF_FIELD_NULL. + // When inPath already exist, append value if it's an arrayObject, otherwise returns -E_JSON_INSERT_PATH_EXIST + // if nearest path ends with type not object, rets -E_JSON_INSERT_PATH_CONFLICT. + // Otherwise, insert field as well as filling up intermediate field, then returns E_OK; + // isAppend: when it's true, append inValue as path is an arrayObject if path not exist. + int InsertField(const FieldPath &inPath, FieldType inType, const FieldValue &inValue, bool isAppend = false); + + // Add json object to an array field. should be called on an valid JsonObject. Never turn into invalid after call. + // If inPath not refer to an array, return error. + int InsertField(const FieldPath &inPath, const JsonObject &inValue, bool isAppend = false); + // Should be called on an valid JsonObject. Never turn into invalid after call. An empty inPath is not allowed. - // If inPath not exist, returns -E_JSON_DELETE_PATH_NOT_FOUND. Otherwise delete field from its parent returns E_OK; + // If inPath not exist, returns -E_JSON_DELETE_PATH_NOT_FOUND. Otherwise, delete field from its parent returns E_OK; int DeleteField(const FieldPath &inPath); private: #ifndef OMIT_JSON // Auxiliary Method: If inPath not refer to an array, return error. If not all members are string, return error. int GetStringArrayContentByJsonValue(const Json::Value &value, std::vector &outStringArray) const; + // Common Type Judgement Logic int GetFieldTypeByJsonValue(const Json::Value &value, FieldType &outType) const; + // Return E_OK if JsonValueNode found at exact the path, otherwise not E_OK const Json::Value &GetJsonValueByFieldPath(const FieldPath &inPath, int &errCode) const; + // REQUIRE: JsonObject is valid(Root value is object type). // If inPath empty(means root), set exact and nearest to root value and nearDepth to 0, then ret E_OK; // If JsonValue exist at exact path, set exact to this JsonValue, set nearest to its parent JsonValue, set nearDepth @@ -98,6 +116,9 @@ private: int LocateJsonValueByFieldPath(const FieldPath &inPath, Json::Value *&exact, Json::Value *&nearest, uint32_t &nearDepth); + // create if path not exist + int MoveToPath(const FieldPath &inPath, Json::Value *&exact, Json::Value *&nearest); + static uint32_t maxNestDepth_; bool isValid_ = false; diff --git a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h index 7b712481475961cf258c13b8c2cbde0a19dd13ce..0e4c0dbc8df7b8cb418f397b79735159fa6725bd 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/param_check_utils.h @@ -47,6 +47,8 @@ public: static int CheckAndTransferAutoLaunchParam(const AutoLaunchParam ¶m, SchemaObject &schemaObject, std::string &canonicalDir); + + static uint8_t GetValidCompressionRate(uint8_t compressionRate); }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/parcel.h b/services/distributeddataservice/libs/distributeddb/common/include/parcel.h index 5d7b05ed4d61d277d6ed6041df4ef8efa165fe57..7658f860dcb43a67a02e477aa86eb10754fb84aa 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/parcel.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/parcel.h @@ -36,8 +36,14 @@ public: Parcel(uint8_t *inBuf, uint32_t length); ~Parcel(); bool IsError() const; + int WriteBool(bool data); + uint32_t ReadBool(bool &data); int WriteInt(int data); uint32_t ReadInt(int &val); + int WriteDouble(double data); + uint32_t ReadDouble(double &val); + int WriteInt64(int64_t data); + uint32_t ReadInt64(int64_t &val); int WriteUInt32(uint32_t data); uint32_t ReadUInt32(uint32_t &val); int WriteUInt64(uint64_t data); @@ -46,6 +52,7 @@ public: uint32_t ReadVectorChar(std::vector &val); int WriteString(const std::string &inVal); uint32_t ReadString(std::string &outVal); + bool IsContinueRead(); #ifndef OMIT_MULTI_VER int WriteMultiVerCommit(const MultiVerCommitNode &commit); uint32_t ReadMultiVerCommit(MultiVerCommitNode &commit); @@ -58,18 +65,25 @@ public: { static_assert(std::is_pod::value, "type T is not pod"); if (data.size() > INT32_MAX || sizeof(T) > INT32_MAX) { + LOGE("[WriteVector] invalid vector. vec.size:%zu, sizeof(T):%zu", data.size(), sizeof(T)); isError_ = true; return -E_PARSE_FAIL; } + if (IsError()) { + return -E_PARSE_FAIL; + } uint32_t len = data.size(); uint64_t stepLen = static_cast(data.size()) * sizeof(T) + sizeof(uint32_t); len = HostToNet(len); - if (isError_ || bufPtr_ == nullptr || stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + if (bufPtr_ == nullptr || stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + LOGE("[WriteVector] bufPtr:%d, stepLen:%llu, totalLen:%llu, parcelLen:%llu", + bufPtr_ != nullptr, stepLen, totalLen_, parcelLen_); isError_ = true; return -E_PARSE_FAIL; } errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &len, sizeof(uint32_t)); if (errCode != EOK) { + LOGE("[ReadVector] totalLen:%llu, parcelLen:%llu", totalLen_, parcelLen_); isError_ = true; return -E_SECUREC_ERROR; } @@ -87,19 +101,25 @@ public: uint32_t ReadVector(std::vector &val) { static_assert(std::is_pod::value, "type T is not pod"); - uint32_t len; - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_ || sizeof(T) > INT32_MAX) { + if (IsError()) { + return 0; + } + if (bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_ || sizeof(T) > INT32_MAX) { + LOGE("[ReadVector] bufPtr:%d, totalLen:%llu, parcelLen:%llu, sizeof(T):%zu", + bufPtr_ != nullptr, totalLen_, parcelLen_, sizeof(T)); isError_ = true; return 0; } - len = *(reinterpret_cast(bufPtr_)); + uint32_t len = *(reinterpret_cast(bufPtr_)); len = NetToHost(len); if (len > INT32_MAX) { + LOGE("[ReadVector] invalid length:%u", len); isError_ = true; return 0; } uint64_t stepLen = static_cast(len) * sizeof(T) + sizeof(uint32_t); if (stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + LOGE("[ReadVector] stepLen:%llu, totalLen:%llu, parcelLen:%llu", stepLen, totalLen_, parcelLen_); isError_ = true; return 0; } @@ -117,10 +137,13 @@ public: int WriteBlob(const char *buffer, uint32_t bufLen); uint32_t ReadBlob(char *buffer, uint32_t bufLen); - void EightByteAlign(); + void EightByteAlign(); // Avoid reading a single data type across 8 bytes + static uint32_t GetBoolLen(); static uint32_t GetIntLen(); static uint32_t GetUInt32Len(); static uint32_t GetUInt64Len(); + static uint32_t GetInt64Len(); + static uint32_t GetDoubleLen(); static uint32_t GetVectorCharLen(const std::vector &data); template @@ -144,13 +167,62 @@ public: static uint32_t GetMultiVerCommitsLen(const std::vector &commits); #endif static uint32_t GetAppendedLen(); + private: + template + int WriteInteger(T integer); + template + uint32_t ReadInteger(T &integer); + bool isError_ = false; uint8_t *buf_ = nullptr; uint8_t *bufPtr_ = nullptr; uint64_t parcelLen_ = 0; uint64_t totalLen_ = 0; }; + +template +uint32_t Parcel::ReadInteger(T &integer) +{ + if (IsError()) { + return 0; + } + if (bufPtr_ == nullptr || parcelLen_ + sizeof(T) > totalLen_) { + LOGE("[ReadInteger] bufPtr:%d, totalLen:%llu, parcelLen:%llu, sizeof(T):%zu", + bufPtr_ != nullptr, totalLen_, parcelLen_, sizeof(T)); + isError_ = true; + return 0; + } + integer = *(reinterpret_cast(bufPtr_)); + bufPtr_ += sizeof(T); + parcelLen_ += sizeof(T); + integer = NetToHost(integer); + return sizeof(T); +} + +template +int Parcel::WriteInteger(T integer) +{ + if (IsError()) { + return -E_PARSE_FAIL; + } + T inData = HostToNet(integer); + if (parcelLen_ + sizeof(T) > totalLen_) { + LOGE("[WriteInteger] totalLen:%llu, parcelLen:%llu, sizeof(T):%zu", totalLen_, parcelLen_, sizeof(T)); + isError_ = true; + return -E_PARSE_FAIL; + } + errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &inData, sizeof(T)); + if (errCode != EOK) { + LOGE("[WriteInteger] bufPtr:%d, totalLen:%llu, parcelLen:%llu, sizeof(T):%zu", + bufPtr_ != nullptr, totalLen_, parcelLen_, sizeof(T)); + isError_ = true; + return -E_SECUREC_ERROR; + } + bufPtr_ += sizeof(T); + parcelLen_ += sizeof(T); + return errCode; +} } // namespace DistributedDB #endif // PARCEL_H diff --git a/services/distributeddataservice/libs/distributeddb/common/include/platform_specific.h b/services/distributeddataservice/libs/distributeddb/common/include/platform_specific.h index b95beb866798103910f4620bee78646c6eb8d107..0c2773f013a164b185a319d01e86fc6af66e99d8 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/platform_specific.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/platform_specific.h @@ -34,6 +34,11 @@ struct FileAttr { uint64_t fileLen; }; +// Shield the representation method of file handles on different platforms +struct FileHandle { + int handle = -1; +}; + int CalFileSize(const std::string &fileUrl, uint64_t &size); bool CheckPathExistence(const std::string &filePath); @@ -61,6 +66,12 @@ int GetFilePermissions(const std::string &fileName, uint32_t &permissions); int SetFilePermissions(const std::string &fileName, uint32_t permissions); int RenameFilePath(const std::string &oldFilePath, const std::string &newFilePath); + +int OpenFile(const std::string &fileName, FileHandle &handle); +int CloseFile(FileHandle &handle); + +int FileLock(const FileHandle &handle, bool isBlock); // be careful use block=true, may block process +int FileUnlock(FileHandle &handle); } // namespace OS } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h new file mode 100644 index 0000000000000000000000000000000000000000..47e4f99aa591388bd1d5c2d227d95c3d97f6a657 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/include/relational/relational_schema_object.h @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_SCHEMA_OBJECT_H +#define RELATIONAL_SCHEMA_OBJECT_H +#ifdef RELATIONAL_STORE +#include +#include "parcel.h" +#include "schema.h" +#include "data_value.h" +#include "json_object.h" + +namespace DistributedDB { +using CompositeFields = std::vector; +class FieldInfo { +public: + const std::string &GetFieldName() const; + void SetFieldName(const std::string &fileName); + const std::string &GetDataType() const; + void SetDataType(const std::string &dataType); + bool IsNotNull() const; + void SetNotNull(bool isNotNull); + // Use string type to save the default value define in the create table sql. + // No need to use the real value because sqlite will complete them. + bool HasDefaultValue() const; + const std::string &GetDefaultValue() const; + void SetDefaultValue(const std::string &value); + // convert to StorageType according "Determination Of Column Affinity" + StorageType GetStorageType() const; + void SetStorageType(StorageType storageType); + + int GetColumnId() const; + void SetColumnId(int cid); + + // return field define string like ("fieldName": "MY INT(21), NOT NULL, DEFAULT 123") + std::string ToAttributeString() const; +private: + std::string fieldName_; + std::string dataType_; // Type may be null + StorageType storageType_ = StorageType::STORAGE_TYPE_NONE; + bool isNotNull_ = false; + bool hasDefaultValue_ = false; + std::string defaultValue_; + int64_t cid_ = -1; +}; + +class TableInfo { +public: + const std::string &GetTableName() const; + bool GetAutoIncrement() const; + const std::string &GetCreateTableSql() const; + const std::map &GetFields() const; // + const std::vector &GetUniqueDefine() const; + const std::map &GetIndexDefine() const; + const FieldName &GetPrimaryKey() const; + const std::vector &GetTriggers() const; + const std::string &GetDevId() const; + + void SetTableName(const std::string &tableName); + void SetAutoIncrement(bool autoInc); + void SetCreateTableSql(std::string sql); // set 'autoInc_' flag when set sql + void AddField(const FieldInfo &field); + void AddUniqueDefine(const CompositeFields &uniqueDefine); + void SetUniqueDefines(const std::vector &uniqueDefines); + void AddIndexDefine(const std::string &indexName, const CompositeFields &indexDefine); + void SetPrimaryKey(const FieldName &fieldName); // not support composite index now + void AddTrigger(const std::string &triggerName); + std::string ToTableInfoString() const; + void SetDevId(const std::string &devId); +private: + void AddFieldDefineString(std::string &attrStr) const; + void AddUniqueDefineString(std::string &attrStr) const; + void AddIndexDefineString(std::string &attrStr) const; + std::string tableName_; + std::string devId_; + bool autoInc_ = false; // only 'INTEGER PRIMARY KEY' could be defined as 'AUTOINCREMENT' + std::string sql_; + std::map fields_; + FieldName primaryKey_; + std::vector uniqueDefines_; + std::map indexDefines_; + std::vector triggers_; + JsonObject ToJsonObject() const; +}; + +class RelationalSyncOpinion { +public: + int CalculateParcelLen(uint32_t softWareVersion) const; + int Serialize(Parcel &parcel, uint32_t softWareVersion) const; + int Deserialization(const Parcel &parcel); + const SyncOpinion &GetTableOpinion(const std::string& tableName) const; + void AddSyncOpinion(const std::string &tableName, const SyncOpinion &opinion); +private: + std::map opinions_; +}; + +class RelationalSyncStrategy { +public: + const SyncStrategy &GetTableStrategy(const std::string &tableName) const; + void AddSyncStrategy(const std::string &tableName, const SyncStrategy &strategy); +private: + std::map strategies_; +}; + +class RelationalSchemaObject : public ISchema { +public: + RelationalSchemaObject() = default; + ~RelationalSchemaObject() override = default; + + static RelationalSyncOpinion MakeLocalSyncOpinion(const RelationalSchemaObject &localSchema, + const std::string &remoteSchema, uint8_t remoteSchemaType); + // The remoteOpinion.checkOnReceive is ignored + + static RelationalSyncStrategy ConcludeSyncStrategy(const RelationalSyncOpinion &localOpinion, + const RelationalSyncOpinion &remoteOpinion); + + bool IsSchemaValid() const override; + + SchemaType GetSchemaType() const override; + + std::string ToSchemaString() const override; + + // Should be called on an invalid SchemaObject, create new SchemaObject if need to reparse + int ParseFromSchemaString(const std::string &inSchemaString) override; + + void AddRelationalTable(const TableInfo& tb); + + const std::map &GetTables() const; + + const TableInfo &GetTable(const std::string& tableName) const; + +private: + int CompareAgainstSchemaObject(const std::string &inSchemaString, std::map &cmpRst) const; + + int CompareAgainstSchemaObject(const RelationalSchemaObject &inSchemaObject, + std::map &cmpRst) const; + + int ParseRelationalSchema(const JsonObject &inJsonObject); + int ParseCheckSchemaType(const JsonObject &inJsonObject); + int ParseCheckSchemaVersionMode(const JsonObject &inJsonObject); + int ParseCheckSchemaTableDefine(const JsonObject &inJsonObject); + int ParseCheckTableInfo(const JsonObject &inJsonObject); + int ParseCheckTableName(const JsonObject &inJsonObject, TableInfo &resultTable); + int ParseCheckTableDefine(const JsonObject &inJsonObject, TableInfo &resultTable); + int ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable); + int ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable); + int ParseCheckTableIndex(const JsonObject &inJsonObject, TableInfo &resultTable); + int ParseCheckTablePrimaryKey(const JsonObject &inJsonObject, TableInfo &resultTable); + + bool isValid_ = false; // set to true after parse success from string or add at least one relational table + SchemaType schemaType_ = SchemaType::NONE; // Default NONE + std::string schemaString_; // The minified and valid schemaString + std::string schemaVersion_; + std::map tables_; +}; +} // namespace DistributedDB +#endif // RELATIONAL_STORE +#endif // RELATIONAL_SCHEMA_OBJECT_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h index 34622e8686f7b080dbc225c3e78c3e185752e85d..ac76d7a6d9da43d02f6b025ac0b264a2be2e5de8 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/runtime_context.h @@ -44,6 +44,7 @@ public: // Global setting interfaces. virtual void SetProcessLabel(const std::string &label) = 0; virtual std::string GetProcessLabel() const = 0; + // If the pre adapter is not nullptr, set new adapter will release the pre adapter, // must be called after SetCommunicatorAggregator virtual int SetCommunicatorAdapter(IAdapter *adapter) = 0; @@ -78,7 +79,7 @@ public: const std::string &deviceId, uint8_t flag) const = 0; virtual int EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier) = 0; + const AutoLaunchOption &option) = 0; virtual int DisableKvStoreAutoLaunch(const std::string &identifier) = 0; @@ -86,7 +87,7 @@ public: virtual void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) = 0; - virtual NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errorCode) = 0; + virtual NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errCode) = 0; virtual bool IsAccessControlled() const = 0; diff --git a/services/distributeddataservice/libs/distributeddb/common/include/schema.h b/services/distributeddataservice/libs/distributeddb/common/include/schema.h new file mode 100644 index 0000000000000000000000000000000000000000..c5670b1c7e6f5bbb8055098784b58daf0be7319b --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/include/schema.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SCHEMA_H +#define SCHEMA_H + +#include +#include + +namespace DistributedDB { +// SchemaType::NONE represent for KV database which do not have schema. Only invalid SchemaObject is NONE type. +// Enum value must not be changed except SchemaType::UNRECOGNIZED. +enum class SchemaType : uint8_t { + NONE = 0, + JSON = 1, + FLATBUFFER = 2, + RELATIVE = 3, + UNRECOGNIZED = 4 +}; +struct SyncOpinion { + bool permitSync = false; + bool requirePeerConvert = false; + bool checkOnReceive = false; +}; +struct SyncStrategy { + bool permitSync = false; + bool convertOnSend = false; + bool convertOnReceive = false; + bool checkOnReceive = false; +}; + +struct SchemaAttribute { + FieldType type = FieldType::LEAF_FIELD_NULL; + bool isIndexable = false; + bool hasNotNullConstraint = false; + bool hasDefaultValue = false; + FieldValue defaultValue; // Has default value in union part and default construction in string part + std::string customFieldType {}; // Custom field type like BIGINT, DECIMAL, CHARACTER ... +}; + +class ISchema { +public: + ISchema() = default; + virtual ~ISchema() = default; + virtual int ParseFromSchemaString(const std::string &inSchemaString) = 0; + virtual bool IsSchemaValid() const = 0; + virtual SchemaType GetSchemaType() const = 0; + virtual std::string ToSchemaString() const = 0; +}; +} +#endif // SCHEMA_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/include/schema_object.h b/services/distributeddataservice/libs/distributeddb/common/include/schema_object.h index e13005cba1d3739cde93246f0aed456f3d7df7fc..48621c087bc37a14f9cab56d3d79c2306253fe0f 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/schema_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/schema_object.h @@ -24,25 +24,9 @@ #include "db_types.h" #include "macro_utils.h" #include "value_object.h" +#include "schema.h" namespace DistributedDB { -// SchemaType::NONE represent for KV database which do not have schema. Only invalid SchemaObject is NONE type. -// Enum value must not be changed except SchemaType::UNRECOGNIZED. -enum class SchemaType : uint8_t { - NONE = 0, - JSON = 1, - FLATBUFFER = 2, - UNRECOGNIZED = 3, -}; - -struct SchemaAttribute { - FieldType type = FieldType::LEAF_FIELD_NULL; - bool isIndexable = false; - bool hasNotNullConstraint = false; - bool hasDefaultValue = false; - FieldValue defaultValue; // Has default value in union part and default construction in string part -}; - using IndexName = FieldPath; using IndexFieldInfo = std::pair; using IndexInfo = std::vector; @@ -54,28 +38,16 @@ struct IndexDifference { std::set decrease; }; -struct SyncOpinion { - bool permitSync = false; - bool requirePeerConvert = false; - bool checkOnReceive = false; -}; - -struct SyncStrategy { - bool permitSync = false; - bool convertOnSend = false; - bool convertOnReceive = false; - bool checkOnReceive = false; -}; - -class SchemaObject { +class SchemaObject : public ISchema { public: static std::string GetExtractFuncName(SchemaType inSchemaType); static std::string GenerateExtractSQL(SchemaType inSchemaType, const FieldPath &inFieldpath, FieldType inFieldType, - uint32_t skipSize); + uint32_t skipSize, const std::string &accessStr = ""); // The remoteSchemaType may beyond local SchemaType definition static SyncOpinion MakeLocalSyncOpinion(const SchemaObject &localSchema, const std::string &remoteSchema, uint8_t remoteSchemaType); + // The remoteOpinion.checkOnReceive is ignored static SyncStrategy ConcludeSyncStrategy(const SyncOpinion &localOpinion, const SyncOpinion &remoteOpinion); @@ -84,22 +56,25 @@ public: ~SchemaObject() = default; SchemaObject(const SchemaObject &); SchemaObject& operator=(const SchemaObject &); + // Move constructor and move assignment is not need currently SchemaObject(SchemaObject &&) = delete; SchemaObject& operator=(SchemaObject &&) = delete; // Should be called on an invalid SchemaObject, create new SchemaObject if need to reparse - int ParseFromSchemaString(const std::string &inSchemaString); + int ParseFromSchemaString(const std::string &inSchemaString) override; + + bool IsSchemaValid() const override; + SchemaType GetSchemaType() const override; - bool IsSchemaValid() const; - SchemaType GetSchemaType() const; // For Json-Schema : Unnecessary spacing will be removed and fieldname resorted by lexicographical order // For FlatBuffer-Schema : Original binary schema(Base64 decoded if need) - std::string ToSchemaString() const; + std::string ToSchemaString() const override; uint32_t GetSkipSize() const; std::map GetIndexInfo() const; bool IsIndexExist(const IndexName &indexName) const; + // Return E_OK if queryale. outType will be set if path exist no matter binary or not int CheckQueryableAndGetFieldType(const FieldPath &inPath, FieldType &outType) const; @@ -128,6 +103,7 @@ public: // Accept the original entry-value, return E_OK or E_FLATBUFFER_VERIFY_FAIL. int VerifyValue(ValueSource sourceType, const Value &inValue) const; int VerifyValue(ValueSource sourceType, const RawValue &inValue) const; + // Accept the original value from database. The cache will not be expanded. Return E_OK if nothing error. // The ExtractValue is with nice performance by carefully not use std-class to avoid memory allocation. // But currently it can only deal with path with $. prefix and only one depth. However, meet current demand. @@ -189,8 +165,10 @@ private: // Compare based on self. // return E_SCHEMA_EQUAL_EXACTLY or E_SCHEMA_UNEQUAL_COMPATIBLE_UPGRADE or E_SCHEMA_UNEQUAL_INCOMPATIBLE int CompareFlatBufferDefine(const FlatBufferSchema &other) const; + // Accept a no-skipsize(so byte-aligned) value, return E_OK or E_FLATBUFFER_VERIFY_FAIL. int VerifyFlatBufferValue(const RawValue &inValue, bool tryNoSizePrefix) const; + // Accept a no-skipsize(so byte-aligned) value. int ExtractFlatBufferValue(RawString inPath, const RawValue &inValue, TypeValue &outExtract, bool tryNoSizePrefix) const; diff --git a/services/distributeddataservice/libs/distributeddb/common/include/schema_utils.h b/services/distributeddataservice/libs/distributeddb/common/include/schema_utils.h index 26f3a54527d714eacdc1d27b5839774e9be8891d..8848b9188668d24c79a28fb9737cf2cd2df8f7b6 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/schema_utils.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/schema_utils.h @@ -26,6 +26,8 @@ const std::string KEYWORD_SCHEMA_MODE = "SCHEMA_MODE"; const std::string KEYWORD_SCHEMA_DEFINE = "SCHEMA_DEFINE"; const std::string KEYWORD_SCHEMA_INDEXES = "SCHEMA_INDEXES"; const std::string KEYWORD_SCHEMA_SKIPSIZE = "SCHEMA_SKIPSIZE"; +const std::string KEYWORD_SCHEMA_TYPE = "SCHEMA_TYPE"; +const std::string KEYWORD_SCHEMA_TABLE = "TABLES"; const std::string KEYWORD_INDEX = "INDEX"; // For FlatBuffer-Schema const std::string KEYWORD_MODE_STRICT = "STRICT"; @@ -43,6 +45,8 @@ const std::string KEYWORD_ATTR_VALUE_NULL = "null"; const std::string KEYWORD_ATTR_VALUE_TRUE = "true"; const std::string KEYWORD_ATTR_VALUE_FALSE = "false"; +const std::string KEYWORD_TYPE_RELATIVE = "RELATIVE"; + const uint32_t SCHEMA_META_FEILD_COUNT_MAX = 5; const uint32_t SCHEMA_META_FEILD_COUNT_MIN = 3; const uint32_t SCHEMA_FEILD_NAME_LENGTH_MAX = 64; @@ -55,6 +59,7 @@ const uint32_t SCHEMA_STRING_SIZE_LIMIT = 524288; // 512K const uint32_t SCHEMA_DEFAULT_STRING_SIZE_LIMIT = 4096; // 4K const uint32_t SCHEMA_SKIPSIZE_MAX = 4194302; // 4M - 2 Bytes const std::string SCHEMA_SUPPORT_VERSION = "1.0"; +const std::string SCHEMA_SUPPORT_VERSION_V2 = "2.0"; const uint32_t SECURE_BYTE_ALIGN = 8; // 8 bytes align @@ -63,7 +68,9 @@ public: // Check if any invalid exist, parse it into SchemaAttribute if totally valid and return E_OK // Number don't support format of scientific notation. SchemaAttribute.isIndexable always set true. // Prefix and postfix spaces or tabs is allowed. - static int ParseAndCheckSchemaAttribute(const std::string &inAttrString, SchemaAttribute &outAttr); + // Customer FieldType will be accepted if useAffinity is true. + static int ParseAndCheckSchemaAttribute(const std::string &inAttrString, SchemaAttribute &outAttr, + bool useAffinity = false); // Check if any invalid exist, parse it into FieldPath if totally valid and return E_OK // Each fieldName of the fieldPath will be check valid as well. Path depth will be check. @@ -75,11 +82,13 @@ public: // Remove prefix and postfix spaces or tabs static std::string Strip(const std::string &inString); + // Strip the namespace from the full-name, this method mainly for flatbuffer-type schema static std::string StripNameSpace(const std::string &inFullName); static std::string FieldTypeString(FieldType inType); static std::string SchemaTypeString(SchemaType inType); + // Restore to string representation of fieldPath with $. prefix static std::string FieldPathString(const FieldPath &inPath); @@ -91,7 +100,7 @@ private: static int SplitSchemaAttribute(const std::string &inAttrString, std::vector &outAttrString); static int MakeTrans(const std::string &oriContent, size_t &pos); - static int ParseSchemaAttribute(std::vector &attrContext, SchemaAttribute &outAttr); + static int ParseSchemaAttribute(std::vector &attrContext, SchemaAttribute &outAttr, bool useAffinity); static int TransformDefaultValue(std::string &defaultContent, SchemaAttribute &outAttr); diff --git a/services/distributeddataservice/libs/distributeddb/common/include/semaphore_utils.h b/services/distributeddataservice/libs/distributeddb/common/include/semaphore_utils.h old mode 100755 new mode 100644 diff --git a/services/distributeddataservice/libs/distributeddb/common/include/value_object.h b/services/distributeddataservice/libs/distributeddb/common/include/value_object.h index 2071c0662a32d292b0b6350de705c5899d3773d7..78ef91552d0e6798c0b9f65851c346ba0a5431ce 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/value_object.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/value_object.h @@ -28,6 +28,7 @@ public: ~ValueObject() = default; ValueObject(const ValueObject &); ValueObject& operator=(const ValueObject &); + // Move constructor and move assignment is not need currently ValueObject(ValueObject &&) = delete; ValueObject& operator=(ValueObject &&) = delete; @@ -35,10 +36,12 @@ public: // Should be called on an invalid ValueObject, create new ValueObject if need to reparse int Parse(const std::string &inString); int Parse(const std::vector &inData); // Whether ends with '\0' in vector is OK + // The end refer to the byte after the last valid byte int Parse(const uint8_t *dataBegin, const uint8_t *dataEnd, uint32_t offset = 0); bool IsValid() const; + // Unnecessary spacing will be removed and fieldname resorted by lexicographical order std::string ToString() const; void WriteIntoVector(std::vector &outData) const; // An vector version ToString @@ -46,6 +49,7 @@ public: bool IsFieldPathExist(const FieldPath &inPath) const; int GetFieldTypeByFieldPath(const FieldPath &inPath, FieldType &outType) const; int GetFieldValueByFieldPath(const FieldPath &inPath, FieldValue &outValue) const; + // An empty fieldpath indicate the root, the outSubPath should be empty before call, we will not empty it at first. // If inPath is of multiple path, then outSubPath is combination of result of each inPath. int GetSubFieldPath(const FieldPath &inPath, std::set &outSubPath) const; @@ -58,6 +62,7 @@ public: // inValue is ignored for LEAF_FIELD_NULL. If inPath already exist or nearest path ends with type not object, // returns not E_OK. Otherwise insert field as well as filling up intermediate field, then returns E_OK; int InsertField(const FieldPath &inPath, FieldType inType, const FieldValue &inValue); + // Should be called on an valid ValueObject. Never turn into invalid after call. An empty inPath is not allowed. // If inPath not exist, returns not E_OK. Otherwise delete field from its parent returns E_OK; int DeleteField(const FieldPath &inPath); diff --git a/services/distributeddataservice/libs/distributeddb/common/include/version.h b/services/distributeddataservice/libs/distributeddb/common/include/version.h index 4ee201df8ff6c34d5524d3012c6832182c9856f2..857d186f2412d7e423abd7d3f4e001612b9baee2 100755 --- a/services/distributeddataservice/libs/distributeddb/common/include/version.h +++ b/services/distributeddataservice/libs/distributeddb/common/include/version.h @@ -30,13 +30,20 @@ namespace DistributedDB { // if now version of B is 105, and C is 101, thus version of A is 105; if now release version is 106 and we upgrade // submodule C, if we simply change version of C to 102 then version of A is still 105, but if we change version of C // to 106 then version of A is now 106, so we can know that something had changed for module A. -const std::string SOFTWARE_VERSION_STRING = "1.1.3"; // DistributedDB current version string. +const std::string SOFTWARE_VERSION_STRING = "1.1.5"; // DistributedDB current version string. constexpr uint32_t SOFTWARE_VERSION_BASE = 100; // Software version base value, do not change it constexpr uint32_t SOFTWARE_VERSION_RELEASE_1_0 = SOFTWARE_VERSION_BASE + 1; // 1 for first released version constexpr uint32_t SOFTWARE_VERSION_RELEASE_2_0 = SOFTWARE_VERSION_BASE + 2; // 2 for second released version -constexpr uint32_t SOFTWARE_VERSION_RELEASE_3_0 = SOFTWARE_VERSION_BASE + 3; // 3 for second released version +constexpr uint32_t SOFTWARE_VERSION_RELEASE_3_0 = SOFTWARE_VERSION_BASE + 3; // 3 for third released version +constexpr uint32_t SOFTWARE_VERSION_RELEASE_4_0 = SOFTWARE_VERSION_BASE + 4; // 4 for fourth released version +constexpr uint32_t SOFTWARE_VERSION_RELEASE_5_0 = SOFTWARE_VERSION_BASE + 5; // 5 for fifth released version +constexpr uint32_t SOFTWARE_VERSION_RELEASE_6_0 = SOFTWARE_VERSION_BASE + 6; // 6 for sixth released version constexpr uint32_t SOFTWARE_VERSION_EARLIEST = SOFTWARE_VERSION_RELEASE_1_0; -constexpr uint32_t SOFTWARE_VERSION_CURRENT = SOFTWARE_VERSION_RELEASE_3_0; +#ifdef RELATIONAL_STORE +constexpr uint32_t SOFTWARE_VERSION_CURRENT = SOFTWARE_VERSION_RELEASE_6_0; +#else +constexpr uint32_t SOFTWARE_VERSION_CURRENT = SOFTWARE_VERSION_RELEASE_5_0; +#endif constexpr int VERSION_INVALID = INT32_MAX; // Storage Related Version diff --git a/services/distributeddataservice/libs/distributeddb/common/include/zlib_compression.h b/services/distributeddataservice/libs/distributeddb/common/include/zlib_compression.h new file mode 100644 index 0000000000000000000000000000000000000000..3ddccd3f9e2fcf3dfa67864879bc6579464cec0c --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/include/zlib_compression.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef ZLIB_COMPRESSION_H +#define ZLIB_COMPRESSION_H +#ifndef OMIT_ZLIB +#include +#include "data_compression.h" + +namespace DistributedDB { +class ZlibCompression final : public DataCompression { +public: + ZlibCompression(); + ~ZlibCompression() = default; + + int Compress(const std::vector &srcData, std::vector &destData) const override; + int Uncompress(const std::vector &srcData, std::vector &destData, unsigned long destLen) const + override; + +protected: + ZlibCompression(const ZlibCompression& compression) = delete; + ZlibCompression& operator= (const ZlibCompression& compression) = delete; +}; +} // namespace DistributedDB +#endif // OMIT_ZLIB +#endif // ZLIB_COMPRESSION_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp index 1354f8ee4cb992b2d6cb1951808e8e0c07dd3aa7..b55ad13d8edc6ee87200c1162e0a29e36d85f920 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/auto_launch.cpp @@ -18,16 +18,16 @@ #include #include "db_errno.h" -#include "log_print.h" -#include "runtime_context.h" -#include "kvdb_pragma.h" -#include "kvdb_manager.h" +#include "db_common.h" #include "kv_store_changed_data_impl.h" -#include "sync_able_kvdb_connection.h" -#include "semaphore_utils.h" #include "kv_store_nb_conflict_data_impl.h" -#include "db_common.h" +#include "kvdb_manager.h" +#include "kvdb_pragma.h" +#include "log_print.h" #include "param_check_utils.h" +#include "runtime_context.h" +#include "semaphore_utils.h" +#include "sync_able_kvdb_connection.h" namespace DistributedDB { namespace { @@ -132,11 +132,12 @@ int AutoLaunch::EnableKvStoreAutoLaunchParmCheck(AutoLaunchItem &autoLaunchItem, } int AutoLaunch::EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier) + const AutoLaunchOption &option) { LOGI("[AutoLaunch] EnableKvStoreAutoLaunch"); std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); - AutoLaunchItem autoLaunchItem{properties, notifier, observer, conflictType, conflictNotifier}; + AutoLaunchItem autoLaunchItem { properties, notifier, option.observer, option.conflictType, option.notifier }; + autoLaunchItem.isAutoSync = option.isAutoSync; int errCode = EnableKvStoreAutoLaunchParmCheck(autoLaunchItem, identifier); if (errCode != E_OK) { LOGE("[AutoLaunch] EnableKvStoreAutoLaunch failed errCode:%d", errCode); @@ -168,7 +169,12 @@ int AutoLaunch::GetConnectionInEnable(AutoLaunchItem &autoLaunchItem, const std: autoLaunchItemMap_.erase(identifier); return errCode; } - if (onlineDevices_.empty()) { + bool isEmpty = false; + { + std::lock_guard onlineDevicesLock(dataLock_); + isEmpty = onlineDevices_.empty(); + } + if (isEmpty) { LOGI("[AutoLaunch] GetConnectionInEnable no online device, ReleaseDatabaseConnection"); errCode = KvDBManager::ReleaseDatabaseConnection(autoLaunchItem.conn); if (errCode != E_OK) { @@ -277,14 +283,14 @@ int AutoLaunch::RegisterObserverAndLifeCycleCallback(AutoLaunchItem &autoLaunchI return errCode; } - bool enAutoSync = true; + bool enAutoSync = autoLaunchItem.isAutoSync; errCode = static_cast(autoLaunchItem.conn)->Pragma(PRAGMA_AUTO_SYNC, static_cast(&enAutoSync)); if (errCode != E_OK) { LOGE("[AutoLaunch] PRAGMA_AUTO_SYNC failed, errCode:%d", errCode); return errCode; } - LOGI("[AutoLaunch] PRAGMA_AUTO_SYNC ok"); + LOGI("[AutoLaunch] set PRAGMA_AUTO_SYNC ok, enAutoSync=%d", enAutoSync); return errCode; } @@ -564,16 +570,16 @@ void AutoLaunch::GetConnInDoOpenMap(std::map &doOpe SemaphoreUtils sema(1 - doOpenMap.size()); for (auto &iter : doOpenMap) { int errCode = RuntimeContext::GetInstance()->ScheduleTask([&sema, &iter, this] { - int errCode; - iter.second.conn = GetOneConnection(iter.second.properties, errCode); - LOGI("[AutoLaunch] GetConnInDoOpenMap GetOneConnection errCode:%d\n", errCode); + int ret; + iter.second.conn = GetOneConnection(iter.second.properties, ret); + LOGI("[AutoLaunch] GetConnInDoOpenMap GetOneConnection errCode:%d\n", ret); if (iter.second.conn == nullptr) { sema.SendSemaphore(); LOGI("[AutoLaunch] GetConnInDoOpenMap in open thread finish SendSemaphore"); return; } - errCode = RegisterObserverAndLifeCycleCallback(iter.second, iter.first, false); - if (errCode != E_OK) { + ret = RegisterObserverAndLifeCycleCallback(iter.second, iter.first, false); + if (ret != E_OK) { LOGE("[AutoLaunch] GetConnInDoOpenMap failed, we do CloseConnection"); TryCloseConnection(iter.second); // if here failed, do nothing iter.second.conn = nullptr; @@ -724,6 +730,7 @@ int AutoLaunch::AutoLaunchExt(const std::string &identifier) } AutoLaunchItem autoLaunchItem{properties, param.notifier, param.option.observer, param.option.conflictType, param.option.notifier}; + autoLaunchItem.isAutoSync = param.option.isAutoSync; errCode = RuntimeContext::GetInstance()->ScheduleTask(std::bind(&AutoLaunch::AutoLaunchExtTask, this, identifier, autoLaunchItem)); if (errCode != E_OK) { @@ -904,6 +911,11 @@ int AutoLaunch::GetAutoLaunchProperties(const AutoLaunchParam ¶m, KvDBProper properties.SetIntProp(KvDBProperties::SECURITY_LABEL, param.option.secOption.securityLabel); properties.SetIntProp(KvDBProperties::SECURITY_FLAG, param.option.secOption.securityFlag); } + properties.SetBoolProp(KvDBProperties::COMPRESS_ON_SYNC, param.option.isNeedCompressOnSync); + if (param.option.isNeedCompressOnSync) { + properties.SetIntProp(KvDBProperties::COMPRESSION_RATE, + ParamCheckUtils::GetValidCompressionRate(param.option.compressionRate)); + } DBCommon::SetDatabaseIds(properties, param.appId, param.userId, param.storeId); return E_OK; } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/data_compression.cpp b/services/distributeddataservice/libs/distributeddb/common/src/data_compression.cpp new file mode 100644 index 0000000000000000000000000000000000000000..8465e9df45f17ddb7f7ad2f149e43983cd13ddfb --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/src/data_compression.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "data_compression.h" +#include +#include "db_errno.h" + +namespace DistributedDB { +void DataCompression::GetCompressionAlgo(std::set &algorithmSet) +{ + algorithmSet.clear(); + for (const auto &item : GetCompressionAlgos()) { + algorithmSet.insert(item.first); + } + return; +} + +int DataCompression::TransferCompressionAlgo(uint32_t compressAlgoType, CompressAlgorithm &algoType) +{ + auto iter = GetTransMap().find(compressAlgoType); + if (iter == GetTransMap().end()) { + return -E_INVALID_ARGS; + } + algoType = iter->second; + return E_OK; +} + +DataCompression *DataCompression::GetInstance(CompressAlgorithm algo) +{ + auto iter = GetCompressionAlgos().find(algo); + if (iter == GetCompressionAlgos().end()) { + return nullptr; + } + return iter->second; +} + +// All supported compression algorithm should call this function to register their instance. +void DataCompression::Register(CompressAlgorithm algo, DataCompression *compressionPtr) +{ + if (GetInstance(algo) != nullptr) { + return; + } + GetCompressionAlgos().insert({algo, compressionPtr}); + GetTransMap().insert({static_cast(algo), algo}); +} + +std::map &DataCompression::GetCompressionAlgos() +{ + static std::map compressionAlgos; + return compressionAlgos; +} + +std::map &DataCompression::GetTransMap() +{ + static std::map transferMap; + return transferMap; +} +} // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e7c5a74ad63a078173cbd4cd2f810040e88ca3c7 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_ability.cpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "db_ability.h" + +#include +#include "db_errno.h" +#include "types_export.h" + +namespace DistributedDB { +DbAbility::DbAbility() +{ + for (auto & item : ABILITYBITS) { + dbAbilityItemSet_.insert(item); + } + dbAbility_.resize(ABILITYBITS.back().first + ABILITYBITS.back().second); +} + +DbAbility::DbAbility(const DbAbility &other) +{ + if (&other != this) { + dbAbility_ = other.dbAbility_; + dbAbilityItemSet_ = other.dbAbilityItemSet_; + } +} + +DbAbility& DbAbility::operator=(const DbAbility &other) +{ + if (&other != this) { + dbAbility_ = other.dbAbility_; + dbAbilityItemSet_ = other.dbAbilityItemSet_; + } + return *this; +} + +bool DbAbility::operator==(const DbAbility &other) const +{ + return (dbAbility_ == other.dbAbility_) && (dbAbilityItemSet_ == other.dbAbilityItemSet_); +} + +int DbAbility::Serialize(Parcel &parcel, const DbAbility &curAbility) +{ + uint32_t div = curAbility.GetAbilityBitsLen() / SERIALIZE_BIT_SIZE; + uint32_t buffLen = (curAbility.GetAbilityBitsLen() % SERIALIZE_BIT_SIZE) ? div + 1 : div; + std::vector dstBuf(buffLen, 0); + uint32_t buffOffset = 0; + uint32_t innerBuffOffset = 0; + const std::vector &abilityBuff = curAbility.GetDbAbilityBuff(); + for (uint32_t pos = 0; pos < curAbility.GetAbilityBitsLen(); pos++, innerBuffOffset++) { + if (innerBuffOffset >= SERIALIZE_BIT_SIZE) { + innerBuffOffset = 0; + buffOffset++; + } + uint64_t value = static_cast(abilityBuff[pos]) << innerBuffOffset; + dstBuf[buffOffset] = dstBuf[buffOffset] | value; + } + int errCode = parcel.WriteVector(dstBuf); + if (errCode != E_OK) { + return errCode; + } + return E_OK; +} + +int DbAbility::DeSerialize(Parcel &parcel, DbAbility &curAbility) +{ + if (!parcel.IsContinueRead()) { + return E_OK; + } + std::vector dstBuf; + parcel.ReadVector(dstBuf); + if (parcel.IsError()) { + LOGE("[DbAbility][DeSerialize] deserialize failed."); + return -E_LENGTH_ERROR; + } + if (dstBuf.size() == 0) { + LOGE("[DbAbility][DeSerialize] buf length get failed."); + return -E_LENGTH_ERROR; + } + std::vector targetBuff(ABILITYBITS.back().first + ABILITYBITS.back().second); + uint32_t buffOffset = 0; + uint32_t innerBuffOffset = 0; + for (uint32_t pos = 0; pos < targetBuff.size() && pos < SERIALIZE_BIT_SIZE * dstBuf.size(); pos++) { + if (innerBuffOffset >= SERIALIZE_BIT_SIZE) { + innerBuffOffset = 0; + buffOffset++; + } + targetBuff[pos] = (dstBuf[buffOffset] >> innerBuffOffset) & 0x1; + innerBuffOffset++; + } + curAbility.SetDbAbilityBuff(targetBuff); + return E_OK; +} + +uint32_t DbAbility::CalculateLen(const DbAbility &curAbility) +{ + uint32_t div = curAbility.GetAbilityBitsLen() / SERIALIZE_BIT_SIZE; + uint32_t buffLen = (curAbility.GetAbilityBitsLen() % SERIALIZE_BIT_SIZE) ? div + 1 : div; + return Parcel::GetVectorLen(std::vector(buffLen, 0)); +} + +void DbAbility::SetDbAbilityBuff(std::vector &buff) +{ + dbAbility_ = buff; +} + +const std::vector &DbAbility::GetDbAbilityBuff() const +{ + return dbAbility_; +} + +uint32_t DbAbility::GetAbilityBitsLen() const +{ + return dbAbility_.size(); +} + +uint8_t DbAbility::GetAbilityItem(const AbilityItem abilityType) const +{ + uint8_t data = 0; + auto iter = dbAbilityItemSet_.find(abilityType); + if (iter != dbAbilityItemSet_.end()) { + if ((iter->first + iter->second) > dbAbility_.size()) { + LOGE("[DbAbility] abilityType is error, start=%d, use_bit=%d, totalLen=%d", iter->first, iter->second, + dbAbility_.size()); + return 0; + } + int skip = 0; + // dbAbility_ bit[0..len] : low-->high + for (uint32_t pos = iter->first; pos < (iter->first + iter->second); pos++, skip++) { + if (dbAbility_[pos]) { + data += dbAbility_[pos] << skip; + } + } + } + return data; +} + +int DbAbility::SetAbilityItem(const AbilityItem &abilityType, uint8_t data) +{ + auto iter = dbAbilityItemSet_.find(abilityType); + if (iter != dbAbilityItemSet_.end()) { + if (data >= pow(2, iter->second)) { + LOGE("[DbAbility] value is invalid, data=%d, use_bit=%d", data, iter->second); + return -E_INTERNAL_ERROR; + } + if ((iter->first + iter->second) > dbAbility_.size()) { + dbAbility_.resize(iter->first + iter->second); + } + int pos = iter->first; + while (data) { + dbAbility_[pos] = data % 2; // means binary + data = (data >> 1); + pos++; + } + } + return E_OK; +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp index d326f97ab71ee86374974fd87df2168d10c646e6..de9192da34e253fc94c6d0364d84460f19db762e 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_common.cpp @@ -31,8 +31,8 @@ namespace { if (item.fileType != type) { continue; } - int errCode = remove(item.fileName.c_str()); - if (errCode != 0) { + int errCode = OS::RemoveFile(item.fileName.c_str()); + if (errCode != E_OK) { LOGE("Remove file failed:%d", errno); } } @@ -40,11 +40,11 @@ namespace { void RemoveDirectories(const std::list &fileList, OS::FileType type) { - for (const auto &item : fileList) { - if (item.fileType != type) { + for (auto item = fileList.rbegin(); item != fileList.rend(); ++item) { + if (item->fileType != type) { continue; } - int errCode = OS::RemoveDBDirectory(item.fileName); + int errCode = OS::RemoveDBDirectory(item->fileName); if (errCode != 0) { LOGE("Remove directory failed:%d", errno); } @@ -164,7 +164,7 @@ int DBCommon::CalcValueHash(const std::vector &value, std::vector &value, std::vector remain) { + return oriStr.substr(0, remain); + } + return oriStr; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp index 827121d4d5b6913a479e36c5e7742865876719b8..410e011373cf4b6ddf2b3b86f7539d2ecc5dc325 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/db_constant.cpp @@ -55,4 +55,12 @@ const std::string DBConstant::PATH_BACKUP_POSTFIX = "_bak"; const std::string DBConstant::ID_CONNECTOR = "-"; const std::string DBConstant::DELETE_KVSTORE_REMOVING = "_removing"; +const std::string DBConstant::DB_LOCK_POSTFIX = ".lock"; + +const std::string DBConstant::SUBSCRIBE_QUERY_PREFIX = "subscribe_query_"; + +const std::string DBConstant::TRIGGER_REFERENCES_NEW = "NEW."; +const std::string DBConstant::TRIGGER_REFERENCES_OLD = "OLD."; + +const std::string DBConstant::UPDATE_META_FUNC = "update_meta_within_trigger"; } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/flatbuffer_schema.cpp b/services/distributeddataservice/libs/distributeddb/common/src/flatbuffer_schema.cpp index 55374327c90fd5c263c61a96477236f2f9eb4c35..7adade66406f61b185ab37828c25d24b04c5cbfc 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/flatbuffer_schema.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/flatbuffer_schema.cpp @@ -33,7 +33,7 @@ inline bool IsDoubleNearlyEqual(double left, double right) double relativeDiff = ((absBigger == 0.0) ? 0.0 : (std::fabs(left - right) / absBigger)); // 0.0 for double 0 return relativeDiff < EPSILON; } -#endif +#endif // OMIT_FLATBUFFER } void SchemaObject::FlatBufferSchema::CopyFrom(const FlatBufferSchema &other) @@ -64,7 +64,6 @@ bool SchemaObject::FlatBufferSchema::IsFlatBufferSchema(const std::string &inOri outDecoded = inOriginal; // The original one is the decoded one return true; } - outDecoded.clear(); return false; } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp index 60a63efa29fbaa354b8e0ea24cc76e0a0a1a26bd..37a008ddbfc929a2018d09c2f3f6e1f5323ddd2e 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/json_object.cpp @@ -24,6 +24,12 @@ namespace DistributedDB { #ifndef OMIT_JSON namespace { const uint32_t MAX_NEST_DEPTH = 100; +#ifdef JSONCPP_USE_BUILDER + const int JSON_VALUE_PRECISION = 16; + const std::string JSON_CONFIG_INDENTATION = "indentation"; + const std::string JSON_CONFIG_COLLECT_COMMENTS = "collectComments"; + const std::string JSON_CONFIG_PRECISION = "precision"; +#endif } uint32_t JsonObject::maxNestDepth_ = MAX_NEST_DEPTH; @@ -94,6 +100,10 @@ JsonObject& JsonObject::operator=(const JsonObject &other) return *this; } +JsonObject::JsonObject(const Json::Value &value) : isValid_(true), value_(value) +{ +} + int JsonObject::Parse(const std::string &inString) { // The jsoncpp lib parser in strict mode will still regard root type jsonarray as valid, but we require jsonobject @@ -106,12 +116,28 @@ int JsonObject::Parse(const std::string &inString) LOGE("[Json][Parse] Json nest depth=%u exceed max allowed=%u.", nestDepth, maxNestDepth_); return -E_JSON_PARSE_FAIL; } +#ifdef JSONCPP_USE_BUILDER + JSONCPP_STRING errs; + Json::CharReaderBuilder builder; + Json::CharReaderBuilder::strictMode(&builder.settings_); + builder[JSON_CONFIG_COLLECT_COMMENTS] = false; + std::unique_ptr const jsonReader(builder.newCharReader()); + + auto begin = reinterpret_cast(inString.c_str()); + auto end = reinterpret_cast(inString.c_str() + inString.length()); + if (!jsonReader->parse(begin, end, &value_, &errs)) { + value_ = Json::Value(); + LOGE("[Json][Parse] Parse string to JsonValue fail, reason=%s.", errs.c_str()); + return -E_JSON_PARSE_FAIL; + } +#else Json::Reader reader(Json::Features::strictMode()); if (!reader.parse(inString, value_, false)) { value_ = Json::Value(); LOGE("[Json][Parse] Parse string to JsonValue fail, reason=%s.", reader.getFormattedErrorMessages().c_str()); return -E_JSON_PARSE_FAIL; } +#endif // The jsoncpp lib parser in strict mode will still regard root type jsonarray as valid, but we require jsonobject if (value_.type() != Json::ValueType::objectValue) { value_ = Json::Value(); @@ -144,6 +170,22 @@ int JsonObject::Parse(const uint8_t *dataBegin, const uint8_t *dataEnd) LOGE("[Json][Parse] Json nest depth=%u exceed max allowed=%u.", nestDepth, maxNestDepth_); return -E_JSON_PARSE_FAIL; } +#ifdef JSONCPP_USE_BUILDER + auto begin = reinterpret_cast(dataBegin); + auto end = reinterpret_cast(dataEnd); + + JSONCPP_STRING errs; + Json::CharReaderBuilder builder; + Json::CharReaderBuilder::strictMode(&builder.settings_); + builder[JSON_CONFIG_COLLECT_COMMENTS] = false; + std::unique_ptr const jsonReader(builder.newCharReader()); + // The endDoc parameter of reader::parse refer to the byte after the string itself + if (!jsonReader->parse(begin, end, &value_, &errs)) { + value_ = Json::Value(); + LOGE("[Json][Parse] Parse dataRange to JsonValue fail, reason=%s.", errs.c_str()); + return -E_JSON_PARSE_FAIL; + } +#else Json::Reader reader(Json::Features::strictMode()); auto begin = reinterpret_cast(dataBegin); auto end = reinterpret_cast(dataEnd); @@ -153,6 +195,7 @@ int JsonObject::Parse(const uint8_t *dataBegin, const uint8_t *dataEnd) LOGE("[Json][Parse] Parse dataRange to JsonValue fail, reason=%s.", reader.getFormattedErrorMessages().c_str()); return -E_JSON_PARSE_FAIL; } +#endif // The jsoncpp lib parser in strict mode will still regard root type jsonarray as valid, but we require jsonobject if (value_.type() != Json::ValueType::objectValue) { value_ = Json::Value(); @@ -174,12 +217,23 @@ std::string JsonObject::ToString() const LOGE("[Json][ToString] Not Valid Yet."); return std::string(); } +#ifdef JSONCPP_USE_BUILDER + Json::StreamWriterBuilder writerBuilder; + writerBuilder[JSON_CONFIG_INDENTATION] = ""; + writerBuilder[JSON_CONFIG_PRECISION] = JSON_VALUE_PRECISION; + std::unique_ptr const jsonWriter(writerBuilder.newStreamWriter()); + std::stringstream ss; + jsonWriter->write(value_, &ss); + // The endingLineFeedSymbol is left empty by default. + return ss.str(); +#else Json::FastWriter fastWriter; // Call omitEndingLineFeed to let JsonCpp not append an \n at the end of string. If not doing so, when passing a // minified jsonString, the result of this function will be one byte longer then the original, which may cause the // result checked as length invalid by upper logic when the original length is just at the limitation boundary. fastWriter.omitEndingLineFeed(); return fastWriter.write(value_); +#endif } bool JsonObject::IsFieldPathExist(const FieldPath &inPath) const @@ -395,6 +449,18 @@ bool InsertFieldCheckParameter(const FieldPath &inPath, FieldType inType, const return true; } +void LeafJsonNodeAppendValue(Json::Value &leafNode, FieldType inType, const FieldValue &inValue) +{ + switch (inType) { + case FieldType::LEAF_FIELD_STRING: + leafNode.append(Json::Value(inValue.stringValue)); + break; + default: + // Do nothing. + break; + } +} + // Function design for InsertField call on an null-type Json::Value void LeafJsonNodeAssignValue(Json::Value &leafNode, FieldType inType, const FieldValue &inValue) { @@ -427,21 +493,12 @@ void LeafJsonNodeAssignValue(Json::Value &leafNode, FieldType inType, const Fiel } } -int JsonObject::InsertField(const FieldPath &inPath, FieldType inType, const FieldValue &inValue) +// move the nearest to the leaf of inPath, if not exist, will create it, else it should be an array object. +int JsonObject::MoveToPath(const FieldPath &inPath, Json::Value *&exact, Json::Value *&nearest) { - if (!InsertFieldCheckParameter(inPath, inType, inValue, maxNestDepth_)) { - return -E_INVALID_ARGS; - } - if (!isValid_) { - // Insert on invalid object never fail after parameter check ok, so here no need concern rollback. - value_ = Json::Value(Json::ValueType::objectValue); - isValid_ = true; - } - Json::Value *exact = nullptr; - Json::Value *nearest = nullptr; uint32_t nearDepth = 0; int errCode = LocateJsonValueByFieldPath(inPath, exact, nearest, nearDepth); - if (errCode != -E_NOT_FOUND) { // Path already exist + if (errCode != -E_NOT_FOUND) { // Path already exist and it's not an array object return -E_JSON_INSERT_PATH_EXIST; } // nearDepth 0 represent for root value. nearDepth equal to inPath.size indicate an exact path match @@ -457,8 +514,55 @@ int JsonObject::InsertField(const FieldPath &inPath, FieldType inType, const Fie // object-type after member adding. Then move "nearest" to point to the new JsonValue. nearest = &((*nearest)[inPath[lackFieldIndex]]); } + return E_OK; +} + +int JsonObject::InsertField(const FieldPath &inPath, const JsonObject &inValue, bool isAppend) +{ + if (inPath.empty() || inPath.size() > maxNestDepth_|| !inValue.IsValid()) { + return -E_INVALID_ARGS; + } + if (!isValid_) { + value_ = Json::Value(Json::ValueType::objectValue); + isValid_ = true; + } + Json::Value *exact = nullptr; + Json::Value *nearest = nullptr; + int errCode = MoveToPath(inPath, exact, nearest); + if (errCode != E_OK) { + return errCode; + } + LOGD("nearest type is %d", nearest->type()); + if (isAppend || nearest->type() == Json::ValueType::arrayValue) { + nearest->append(inValue.value_); + } else { + *nearest = inValue.value_; + } + return E_OK; +} + +int JsonObject::InsertField(const FieldPath &inPath, FieldType inType, const FieldValue &inValue, bool isAppend) +{ + if (!InsertFieldCheckParameter(inPath, inType, inValue, maxNestDepth_)) { + return -E_INVALID_ARGS; + } + if (!isValid_) { + // Insert on invalid object never fail after parameter check ok, so here no need concern rollback. + value_ = Json::Value(Json::ValueType::objectValue); + isValid_ = true; + } + Json::Value *exact = nullptr; + Json::Value *nearest = nullptr; + int errCode = MoveToPath(inPath, exact, nearest); + if (errCode != E_OK) { + return errCode; + } // Here "nearest" points to the JsonValue(null-type now) corresponding to the last field - LeafJsonNodeAssignValue(*nearest, inType, inValue); + if (isAppend || nearest->type() == Json::ValueType::arrayValue) { + LeafJsonNodeAppendValue(*nearest, inType, inValue); + } else { + LeafJsonNodeAssignValue(*nearest, inType, inValue); + } return E_OK; } @@ -607,9 +711,51 @@ int JsonObject::LocateJsonValueByFieldPath(const FieldPath &inPath, Json::Value } exact = &((*exact)[eachPathSegment]); // "exact" go deeper } + if (exact->type() == Json::ValueType::arrayValue) { + return -E_NOT_FOUND; // could append value if path is an array field. + } // Here, JsonValue exist at exact path, "nearest" is "exact" parent. return E_OK; } + +int JsonObject::GetObjectArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const +{ + if (!isValid_) { + LOGE("[Json][GetValue] Not Valid Yet."); + return -E_NOT_PERMIT; + } + int errCode = E_OK; + const Json::Value &valueNode = GetJsonValueByFieldPath(inPath, errCode); + if (errCode != E_OK) { + return errCode; + } + + if (!valueNode.isArray()) { + LOGE("[Json][GetValue] Not Array type."); + return -E_NOT_PERMIT; + } + outArray.resize(valueNode.size()); + for (Json::ArrayIndex i = 0; i < valueNode.size(); ++i) { + outArray.emplace_back(JsonObject(valueNode[i])); + } + return E_OK; +} + +int JsonObject::GetStringArrayByFieldPath(const FieldPath &inPath, std::vector &outArray) const +{ + if (!isValid_) { + LOGE("[Json][GetValue] Not Valid Yet."); + return -E_NOT_PERMIT; + } + int errCode = E_OK; + const Json::Value &valueNode = GetJsonValueByFieldPath(inPath, errCode); + if (errCode != E_OK) { + return errCode; + } + + return GetStringArrayContentByJsonValue(valueNode, outArray); +} + #else // OMIT_JSON uint32_t JsonObject::SetMaxNestDepth(uint32_t nestDepth) { @@ -710,9 +856,19 @@ int JsonObject::InsertField(const FieldPath &inPath, FieldType inType, const Fie return -E_NOT_PERMIT; } +int JsonObject::InsertField(const FieldPath &inPath, const JsonObject &inValue, bool isAppend = false) +{ + return -E_NOT_PERMIT; +} + int JsonObject::DeleteField(const FieldPath &inPath) { return -E_NOT_PERMIT; } + +int JsonObject::GetArrayValueByFieldPath(const FieldPath &inPath, JsonObject &outArray) const +{ + return -E_NOT_PERMIT; +} #endif // OMIT_JSON } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/log_print.cpp b/services/distributeddataservice/libs/distributeddb/common/src/log_print.cpp index b6491dfd3b8662324e54e121ae19be9cff068254..1a4a63f1fa6b50906732939329af79f03971e9c2 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/log_print.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/log_print.cpp @@ -35,7 +35,6 @@ public: if (msg.empty()) { return; } - const std::string format = "%{public}s"; OHOS::HiviewDFX::HiLogLabel label = { LOG_CORE, 0xD001630, tag.c_str() }; // log module id. switch (level) { diff --git a/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp b/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp index 0086834b9c6c8f04ac6d12b3c3b1ebc978765db6..e313bd4b4d2eee781f667311a564023288bcc777 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/param_check_utils.cpp @@ -177,4 +177,14 @@ int ParamCheckUtils::CheckAndTransferAutoLaunchParam(const AutoLaunchParam ¶ } return E_OK; } + +uint8_t ParamCheckUtils::GetValidCompressionRate(uint8_t compressionRate) +{ + // Valid when between 1 and 100. When compressionRate is invalid, change it to default rate. + if (compressionRate < 1 || compressionRate > DBConstant::DEFAULT_COMPTRESS_RATE) { + LOGD("Invalid compression rate:%u.", compressionRate); + compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + } + return compressionRate; +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp b/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp index 08292ccd1d2dc8afb0b28bf26543ef8d40c5edc8..e02f8685d6066707a921bea88557c6c335a73ba3 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/parcel.cpp @@ -46,94 +46,105 @@ bool Parcel::IsError() const return isError_; } -int Parcel::WriteInt(int32_t data) +int Parcel::WriteBool(bool data) { - int32_t inData = HostToNet(data); - if (isError_ || parcelLen_ + sizeof(int32_t) > totalLen_) { + const uint32_t boolLen = GetBoolLen(); + if (isError_ || parcelLen_ + boolLen > totalLen_) { isError_ = true; return -E_PARSE_FAIL; } - errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &inData, sizeof(int32_t)); + errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &data, sizeof(bool)); if (errCode != EOK) { isError_ = true; return -E_SECUREC_ERROR; } - bufPtr_ += sizeof(int32_t); - parcelLen_ += sizeof(int32_t); - return errCode; + bufPtr_ += boolLen; + parcelLen_ += boolLen; + return E_OK; } -uint32_t Parcel::ReadInt(int32_t &val) +uint32_t Parcel::ReadBool(bool &val) { - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(int32_t) > totalLen_) { + const uint32_t boolLen = GetBoolLen(); + if (isError_ || parcelLen_ + boolLen > totalLen_) { isError_ = true; - return 0; + return -E_PARSE_FAIL; } - val = *(reinterpret_cast(bufPtr_)); - bufPtr_ += sizeof(int32_t); - parcelLen_ += sizeof(int32_t); - val = NetToHost(val); - return sizeof(int32_t); + val = *(reinterpret_cast(bufPtr_)); + bufPtr_ += boolLen; + parcelLen_ += boolLen; + return boolLen; } -int Parcel::WriteUInt32(uint32_t data) +int Parcel::WriteInt(int32_t data) { - uint32_t inData = HostToNet(data); - if (isError_ || parcelLen_ + sizeof(uint32_t) > totalLen_) { + return WriteInteger(data); +} + +uint32_t Parcel::ReadInt(int32_t &val) +{ + return ReadInteger(val); +} + + +int Parcel::WriteDouble(double data) +{ + double inData = HostToNet(data); + if (isError_ || parcelLen_ + sizeof(double) > totalLen_) { isError_ = true; return -E_PARSE_FAIL; } - errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &inData, sizeof(uint32_t)); + errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &inData, sizeof(double)); if (errCode != EOK) { isError_ = true; return -E_SECUREC_ERROR; } - bufPtr_ += sizeof(uint32_t); - parcelLen_ += sizeof(uint32_t); - return errCode; + bufPtr_ += sizeof(double); + parcelLen_ += sizeof(double); + return E_OK; } -uint32_t Parcel::ReadUInt32(uint32_t &val) +uint32_t Parcel::ReadDouble(double &val) { - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_) { + if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(double) > totalLen_) { isError_ = true; return 0; } - val = *(reinterpret_cast(bufPtr_)); - bufPtr_ += sizeof(uint32_t); - parcelLen_ += sizeof(uint32_t); + val = *(reinterpret_cast(bufPtr_)); + bufPtr_ += sizeof(double); + parcelLen_ += sizeof(double); val = NetToHost(val); - return sizeof(uint32_t); + return sizeof(double); +} + +int Parcel::WriteInt64(int64_t data) +{ + return WriteInteger(data); +} + +uint32_t Parcel::ReadInt64(int64_t &val) +{ + return ReadInteger(val); +} + +int Parcel::WriteUInt32(uint32_t data) +{ + return WriteInteger(data); +} + +uint32_t Parcel::ReadUInt32(uint32_t &val) +{ + return ReadInteger(val); } int Parcel::WriteUInt64(uint64_t data) { - uint64_t inData = HostToNet(data); - if (isError_ || parcelLen_ + sizeof(uint64_t) > totalLen_) { - isError_ = true; - return -E_PARSE_FAIL; - } - errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &inData, sizeof(uint64_t)); - if (errCode != EOK) { - isError_ = true; - return -E_SECUREC_ERROR; - } - bufPtr_ += sizeof(uint64_t); - parcelLen_ += sizeof(uint64_t); - return errCode; + return WriteInteger(data); } uint32_t Parcel::ReadUInt64(uint64_t &val) { - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(uint64_t) > totalLen_) { - isError_ = true; - return 0; - } - val = *(reinterpret_cast(bufPtr_)); - bufPtr_ += sizeof(uint64_t); - parcelLen_ += sizeof(uint64_t); - val = NetToHost(val); - return sizeof(uint64_t); + return ReadInteger(val); } int Parcel::WriteVectorChar(const std::vector& data) @@ -149,18 +160,24 @@ uint32_t Parcel::ReadVectorChar(std::vector& val) int Parcel::WriteString(const std::string &inVal) { if (inVal.size() > INT32_MAX) { + LOGE("[WriteString] Invalid string, size:%zu.", inVal.size()); isError_ = true; return -E_PARSE_FAIL; } + if (IsError()) { + return -E_PARSE_FAIL; + } uint32_t len = inVal.size(); uint64_t stepLen = sizeof(uint32_t) + static_cast(inVal.size()); len = HostToNet(len); - if (isError_ || stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + if (stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + LOGE("[WriteString] stepLen:%llu, totalLen:%llu, parcelLen:%llu", stepLen, totalLen_, parcelLen_); isError_ = true; return -E_PARSE_FAIL; } errno_t errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_, &len, sizeof(uint32_t)); if (errCode != EOK) { + LOGE("[WriteString] bufPtr:%d, totalLen:%llu, parcelLen:%llu", bufPtr_ != nullptr, totalLen_, parcelLen_); isError_ = true; return -E_SECUREC_ERROR; } @@ -172,18 +189,24 @@ int Parcel::WriteString(const std::string &inVal) } errCode = memcpy_s(bufPtr_, totalLen_ - parcelLen_ - sizeof(uint32_t), inVal.c_str(), inVal.size()); if (errCode != EOK) { + LOGE("[WriteString] totalLen:%llu, parcelLen:%llu, inVal.size:%zu.", + totalLen_, parcelLen_, inVal.size()); isError_ = true; return -E_SECUREC_ERROR; } bufPtr_ += inVal.size(); bufPtr_ += BYTE_8_ALIGN(stepLen) - stepLen; parcelLen_ += BYTE_8_ALIGN(stepLen); - return errCode; + return E_OK; } uint32_t Parcel::ReadString(std::string &outVal) { - if (isError_ || bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_) { + if (IsError()) { + return 0; + } + if (bufPtr_ == nullptr || parcelLen_ + sizeof(uint32_t) > totalLen_) { + LOGE("[ReadString] bufPtr:%d, totalLen:%llu, parcelLen:%llu", bufPtr_ != nullptr, totalLen_, parcelLen_); isError_ = true; return 0; } @@ -191,6 +214,7 @@ uint32_t Parcel::ReadString(std::string &outVal) len = NetToHost(len); uint64_t stepLen = static_cast(len) + sizeof(uint32_t); if (stepLen > INT32_MAX || parcelLen_ + BYTE_8_ALIGN(stepLen) > totalLen_) { + LOGE("[ReadString] stepLen:%llu, totalLen:%llu, parcelLen:%llu", stepLen, totalLen_, parcelLen_); isError_ = true; return 0; } @@ -202,6 +226,11 @@ uint32_t Parcel::ReadString(std::string &outVal) return static_cast(stepLen); } +bool Parcel::IsContinueRead() +{ + return (parcelLen_ < totalLen_); +} + #ifndef OMIT_MULTI_VER int Parcel::WriteMultiVerCommit(const MultiVerCommitNode &commit) { @@ -271,8 +300,39 @@ int Parcel::WriteMultiVerCommits(const std::vector &commits) return errCode; } for (auto &iter : commits) { - errCode = WriteMultiVerCommit(iter); + errCode = WriteVectorChar(iter.commitId); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write commitId err!"); + return errCode; + } + errCode = WriteVectorChar(iter.leftParent); if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write leftParent err!"); + return errCode; + } + errCode = WriteVectorChar(iter.rightParent); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write rightParent err!"); + return errCode; + } + errCode = WriteUInt64(iter.timestamp); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write timestamp err!"); + return errCode; + } + errCode = WriteUInt64(iter.version); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write version err!"); + return errCode; + } + errCode = WriteUInt64(iter.isLocal); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write isLocal err!"); + return errCode; + } + errCode = WriteString(iter.deviceInfo); + if (errCode != E_OK) { + LOGE("Parcel::WriteMultiVerCommit write deviceInfo err!"); return errCode; } EightByteAlign(); @@ -296,7 +356,13 @@ uint32_t Parcel::ReadMultiVerCommits(std::vector &commits) } for (uint64_t i = 0; i < size; i++) { MultiVerCommitNode commit; - len += ReadMultiVerCommit(commit); + len += ReadVectorChar(commit.commitId); + len += ReadVectorChar(commit.leftParent); + len += ReadVectorChar(commit.rightParent); + len += ReadUInt64(commit.timestamp); + len += ReadUInt64(commit.version); + len += ReadUInt64(commit.isLocal); + len += ReadString(commit.deviceInfo); commits.push_back(commit); EightByteAlign(); len = BYTE_8_ALIGN(len); @@ -314,37 +380,49 @@ uint32_t Parcel::ReadMultiVerCommits(std::vector &commits) int Parcel::WriteBlob(const char *buffer, uint32_t bufLen) { if (buffer == nullptr) { + LOGE("[WriteBlob] Invalid buffer."); isError_ = true; return -E_INVALID_ARGS; } - if (isError_ || parcelLen_ + bufLen > totalLen_) { + if (IsError()) { + return -E_PARSE_FAIL; + } + if (parcelLen_ + bufLen > totalLen_) { + LOGE("[WriteBlob] bufLen:%u, totalLen:%llu, parcelLen:%llu", bufLen, totalLen_, parcelLen_); isError_ = true; return -E_PARSE_FAIL; } uint32_t leftLen = static_cast(totalLen_ - parcelLen_); int errCode = memcpy_s(bufPtr_, leftLen, buffer, bufLen); if (errCode != EOK) { + LOGE("[WriteBlob] bufPtr:%d, leftLen:%u, buffer:%p, bufLen:%u", bufPtr_ != nullptr, leftLen, buffer, bufLen); isError_ = true; return -E_SECUREC_ERROR; } uint32_t length = (BYTE_8_ALIGN(bufLen) < leftLen) ? BYTE_8_ALIGN(bufLen) : leftLen; bufPtr_ += length; parcelLen_ += length; - return errCode; + return E_OK; } uint32_t Parcel::ReadBlob(char *buffer, uint32_t bufLen) { if (buffer == nullptr) { + LOGE("[ReadBlob] Invalid buffer."); isError_ = true; return 0; } + if (IsError()) { + return 0; + } uint32_t leftLen = static_cast(totalLen_ - parcelLen_); - if (isError_ || parcelLen_ + bufLen > totalLen_) { + if (parcelLen_ + bufLen > totalLen_) { + LOGE("[ReadBlob] bufLen:%u, totalLen:%llu, parcelLen:%llu", bufLen, totalLen_, parcelLen_); isError_ = true; return 0; } int errCode = memcpy_s(buffer, bufLen, bufPtr_, bufLen); if (errCode != EOK) { + LOGE("[ReadBlob] bufPtr:%d, buffer:%p, bufLen:%u", bufPtr_ != nullptr, buffer, bufLen); isError_ = true; return 0; } @@ -354,6 +432,11 @@ uint32_t Parcel::ReadBlob(char *buffer, uint32_t bufLen) return length; } +uint32_t Parcel::GetBoolLen() +{ + return GetEightByteAlign(sizeof(bool)); +} + uint32_t Parcel::GetIntLen() { return sizeof(int32_t); @@ -369,6 +452,16 @@ uint32_t Parcel::GetUInt64Len() return sizeof(uint64_t); } +uint32_t Parcel::GetInt64Len() +{ + return sizeof(int64_t); +} + +uint32_t Parcel::GetDoubleLen() +{ + return sizeof(double); +} + uint32_t Parcel::GetVectorCharLen(const std::vector &data) { return GetVectorLen(data); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/platform_specific.cpp b/services/distributeddataservice/libs/distributeddb/common/src/platform_specific.cpp index c56ba8f53e6b4edf177a65fa9f9ce3a7a36c2604..264842e879ff00187a26e9b72c5f728b5b2594e4 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/platform_specific.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/platform_specific.cpp @@ -54,6 +54,7 @@ int RenameFilePath(const std::string &oldFilePath, const std::string &newFilePat LOGE("[Rename] Rename file fail. err = %d", errno); return -E_SYSTEM_API_FAIL; } + LOGI("Rename file path successfully!"); return E_OK; } @@ -64,6 +65,7 @@ int RemoveFile(const std::string &filePath) LOGE("[RemoveFile] Remove file fail. err = %d", errno); return -E_SYSTEM_API_FAIL; } + LOGI("Remove file successfully!"); return E_OK; } @@ -71,8 +73,9 @@ int CalFileSize(const std::string &fileUrl, uint64_t &size) { struct stat fileStat; if (fileUrl.empty() || stat(fileUrl.c_str(), &fileStat) < 0 || fileStat.st_size < 0) { - LOGE("Get file[%zu] size failed, errno [%d].", fileUrl.size(), errno); - return -E_INVALID_ARGS; + int errCode = (errno == ENOENT) ? -E_NOT_FOUND : -E_INVALID_DB; + LOGD("Get file[%zu] size failed, errno [%d].", fileUrl.size(), errno); + return errCode; } size = fileStat.st_size; @@ -258,5 +261,67 @@ int SetFilePermissions(const std::string &fileName, uint32_t permissions) } return E_OK; } + +int OpenFile(const std::string &fileName, FileHandle &handle) +{ + handle.handle = open(fileName.c_str(), (O_WRONLY | O_CREAT), (S_IRUSR | S_IWUSR | S_IRGRP)); + if (handle.handle < 0) { + LOGE("[FileLock] can not open file when lock it:[%d]", errno); + return -E_SYSTEM_API_FAIL; + } + return E_OK; +} + +int CloseFile(FileHandle &handle) +{ + if (close(handle.handle) != 0) { + LOGE("close file failed, errno:%d", errno); + return -E_SYSTEM_API_FAIL; + } + handle.handle = -1; + return E_OK; +} + +int FileLock(const FileHandle &handle, bool isBlock) +{ + if (handle.handle < 0) { + LOGE("[FileLock] can not open file when lock it:[%d]", errno); + return -E_SYSTEM_API_FAIL; + } + + struct flock fileLockInfo; + (void)memset_s(&fileLockInfo, sizeof(fileLockInfo), 0, sizeof(fileLockInfo)); + fileLockInfo.l_type = F_WRLCK; + fileLockInfo.l_whence = SEEK_SET; + fileLockInfo.l_start = 0; + fileLockInfo.l_len = 0; + LOGD("Lock file isBlock[%d]", isBlock); + if (fcntl(handle.handle, isBlock ? F_SETLKW : F_SETLK, &fileLockInfo) == -1 && !isBlock) { + LOGD("Lock file is Blocked, please retry!"); + return -E_BUSY; + } + LOGI("file locked! errno:%d", errno); + return E_OK; +} + +int FileUnlock(FileHandle &handle) +{ + if (handle.handle == -1) { + LOGI("[FileUnlock] file handle is invalid!"); + return E_OK; + } + + struct flock fileLockInfo; + (void)memset_s(&fileLockInfo, sizeof(fileLockInfo), 0, sizeof(fileLockInfo)); + fileLockInfo.l_type = F_UNLCK; + fileLockInfo.l_whence = SEEK_SET; + fileLockInfo.l_start = 0; + fileLockInfo.l_len = 0; + if (fcntl(handle.handle, F_SETLK, &fileLockInfo) == -1) { + LOGE("Unlock file failed. errno:%d", errno); + return -E_SYSTEM_API_FAIL; + } + return CloseFile(handle); +} } // namespace OS } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/common/src/query.cpp b/services/distributeddataservice/libs/distributeddb/common/src/query.cpp index 462acd9604682d434cfe0a1a3e10c7148aa667ba..5204d1921dc91248fd520aab2ac5c37b1d1c166c 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/query.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/query.cpp @@ -14,6 +14,22 @@ */ #include "query.h" namespace DistributedDB { +Query::Query(const std::string &tableName) +{ + queryExpression_.SetTableName(tableName); +} +Query Query::Select() +{ + Query query; + return query; +} + +Query Query::Select(const std::string &tableName) +{ + Query query(tableName); + return query; +} + Query &Query::BeginGroup() { queryExpression_.BeginGroup(); @@ -44,43 +60,46 @@ Query &Query::SuggestIndex(const std::string &indexName) return *this; } -void Query::ExecuteLogicOperation(QueryObjType operType) -{ - switch (operType) { - case QueryObjType::OR: - queryExpression_.Or(); - break; - case QueryObjType::AND: - queryExpression_.And(); - break; - default: - return; - } -} - -void Query::ExecuteOrderBy(const std::string &field, bool isAsc) +Query &Query::OrderBy(const std::string &field, bool isAsc) { queryExpression_.OrderBy(field, isAsc); + return *this; } -void Query::ExecuteLimit(int number, int offset) +Query &Query::Limit(int number, int offset) { queryExpression_.Limit(number, offset); + return *this; } -void Query::ExecuteLike(const std::string &field, const std::string &value) +Query &Query::Like(const std::string &field, const std::string &value) { queryExpression_.Like(field, value); + return *this; } -void Query::ExecuteNotLike(const std::string &field, const std::string &value) +Query &Query::NotLike(const std::string &field, const std::string &value) { queryExpression_.NotLike(field, value); + return *this; } -void Query::ExecuteIsNull(const std::string &field) +Query &Query::IsNull(const std::string &field) { queryExpression_.IsNull(field); + return *this; +} + +Query &Query::And() +{ + queryExpression_.And(); + return *this; +} + +Query &Query::Or() +{ + queryExpression_.Or(); + return *this; } void Query::ExecuteCompareOperation(QueryObjType operType, const std::string &field, const QueryValueType type, diff --git a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp index 72aeeab76f1728b39c67ab25b7e503e2ff991b29..f8be6d9c6b7eefc686123aa833d3c78c3ec92bc4 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/query_expression.cpp @@ -49,7 +49,9 @@ void QueryExpression::AssemblyQueryInfo(const QueryObjType queryOperType, const } QueryExpression::QueryExpression() - : errFlag_(true) + : errFlag_(true), + tableName_("sync_data"), // default kv type store table name + isTableNameSpecified_(false) {} void QueryExpression::EqualTo(const std::string& field, const QueryValueType type, const FieldValue &value) @@ -191,16 +193,31 @@ const std::list &QueryExpression::GetQueryExpression() queryInfo_.emplace_back(QueryObjNode{QueryObjType::OPER_ILLEGAL}); LOGE("Query operate illegal!"); } - return queryInfo_; } -std::vector QueryExpression::GetPreFixKey() +std::vector QueryExpression::GetPreFixKey() const { return prefixKey_; } -std::string QueryExpression::GetSuggestIndex() +void QueryExpression::SetTableName(const std::string &tableName) +{ + tableName_ = tableName; + isTableNameSpecified_ = true; +} + +const std::string &QueryExpression::GetTableName() +{ + return tableName_; +} + +bool QueryExpression::IsTableNameSpacified() const +{ + return isTableNameSpecified_; +} + +std::string QueryExpression::GetSuggestIndex() const { return suggestIndex_; } @@ -217,6 +234,15 @@ void QueryExpression::EndGroup() QueryValueType::VALUE_TYPE_NULL, std::vector()}); } +void QueryExpression::Reset() +{ + errFlag_ = true; + queryInfo_.clear(); + prefixKey_.clear(); + prefixKey_.shrink_to_fit(); + suggestIndex_.clear(); +} + void QueryExpression::SetErrFlag(bool flag) { errFlag_ = flag; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2c1ecbf67e8503b617fcfb02335c93b56c6ca9f9 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/src/relational/relational_schema_object.cpp @@ -0,0 +1,689 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include +#include + +#include "json_object.h" +#include "relational_schema_object.h" + +namespace DistributedDB { +const std::string &FieldInfo::GetFieldName() const +{ + return fieldName_; +} + +void FieldInfo::SetFieldName(const std::string &fileName) +{ + fieldName_ = fileName; +} + +const std::string &FieldInfo::GetDataType() const +{ + return dataType_; +} + +static StorageType AffinityType(const std::string &dataType) +{ + return StorageType::STORAGE_TYPE_NULL; +} + +void FieldInfo::SetDataType(const std::string &dataType) +{ + dataType_ = dataType; + transform(dataType_.begin(), dataType_.end(), dataType_.begin(), ::tolower); + storageType_ = AffinityType(dataType_); +} + +bool FieldInfo::IsNotNull() const +{ + return isNotNull_; +} + +void FieldInfo::SetNotNull(bool isNotNull) +{ + isNotNull_ = isNotNull; +} + +bool FieldInfo::HasDefaultValue() const +{ + return hasDefaultValue_; +} + +const std::string &FieldInfo::GetDefaultValue() const +{ + return defaultValue_; +} + +void FieldInfo::SetDefaultValue(const std::string &value) +{ + hasDefaultValue_ = true; + defaultValue_ = value; +} + +// convert to StorageType according "Determination Of Column Affinity" +StorageType FieldInfo::GetStorageType() const +{ + return storageType_; +} + +void FieldInfo::SetStorageType(StorageType storageType) +{ + storageType_ = storageType; +} + +int FieldInfo::GetColumnId() const +{ + return cid_; +} + +void FieldInfo::SetColumnId(int cid) +{ + cid_ = cid; +} + +// return field define string like ("fieldName": "MY INT(21), NOT NULL, DEFAULT 123") +std::string FieldInfo::ToAttributeString() const +{ + std::string attrStr = "\"" + fieldName_ + "\": \""; + attrStr += dataType_; + if (isNotNull_) { + attrStr += ", NOT NULL"; + } + if (hasDefaultValue_) { + attrStr += ", " + defaultValue_; + } + attrStr += + "\""; + return attrStr; +} + +const std::string &TableInfo::GetTableName() const +{ + return tableName_; +} + +void TableInfo::SetTableName(const std::string &tableName) +{ + tableName_ = tableName; +} + +void TableInfo::SetAutoIncrement(bool autoInc) +{ + autoInc_ = autoInc; +} + +bool TableInfo::GetAutoIncrement() const +{ + return autoInc_; +} + +const std::string &TableInfo::GetCreateTableSql() const +{ + return sql_; +} + +void TableInfo::SetCreateTableSql(std::string sql) +{ + sql_ = sql; + for (auto &c : sql) { + c = static_cast(std::toupper(c)); + } + if (sql.find("AUTOINCREMENT") != std::string::npos) { + autoInc_ = true; + } +} + +const std::map &TableInfo::GetFields() const +{ + return fields_; +} + +void TableInfo::AddField(const FieldInfo &field) +{ + fields_[field.GetFieldName()] = field; +} + +const std::vector &TableInfo::GetUniqueDefine() const +{ + return uniqueDefines_; +} + +void TableInfo::AddUniqueDefine(const CompositeFields &uniqueDefine) +{ + uniqueDefines_.push_back(uniqueDefine); +} + +void TableInfo::SetUniqueDefines(const std::vector &uniqueDefines) +{ + uniqueDefines_ = uniqueDefines; +} + +const std::map &TableInfo::GetIndexDefine() const +{ + return indexDefines_; +} + +void TableInfo::AddIndexDefine(const std::string &indexName, const CompositeFields &indexDefine) +{ + indexDefines_[indexName] = indexDefine; +} + +const FieldName &TableInfo::GetPrimaryKey() const +{ + return primaryKey_; +} + +void TableInfo::SetPrimaryKey(const FieldName &fieldName) +{ + primaryKey_ = fieldName; +} + +void TableInfo::AddTrigger(const std::string &triggerName) +{ + triggers_.push_back(triggerName); +} + +const std::vector &TableInfo::GetTriggers() const +{ + return triggers_; +} + +void TableInfo::AddFieldDefineString(std::string &attrStr) const +{ + if (fields_.empty()) { + return; + } + attrStr += R"("DEFINE": {)"; + for (auto itField = fields_.begin(); itField != fields_.end(); ++itField) { + attrStr += itField->second.ToAttributeString(); + if (itField != std::prev(fields_.end(), 1)) { + attrStr += ","; + } + } + attrStr += "},"; +} + +void TableInfo::AddUniqueDefineString(std::string &attrStr) const +{ + if (uniqueDefines_.empty()) { + return; + } + attrStr += R"("UNIQUE": [)"; + for (auto itUniqueDefine = uniqueDefines_.begin(); itUniqueDefine != uniqueDefines_.end(); ++itUniqueDefine) { + attrStr += "\""; + for (auto itField = (*itUniqueDefine).begin(); itField != (*itUniqueDefine).end(); ++itField) { + attrStr += *itField; + if (itField != (*itUniqueDefine).end() - 1) { + attrStr += ", "; + } + } + attrStr += "\""; + if (itUniqueDefine != uniqueDefines_.end() - 1) { + attrStr += ","; + } + } + attrStr += "],"; +} + +void TableInfo::AddIndexDefineString(std::string &attrStr) const +{ + if (indexDefines_.empty()) { + return; + } + attrStr += R"("INDEX": {)"; + for (auto itIndexDefine = indexDefines_.begin(); itIndexDefine != indexDefines_.end(); ++itIndexDefine) { + attrStr += "\"" + (*itIndexDefine).first + "\": \""; + for (auto itField = itIndexDefine->second.begin(); itField != itIndexDefine->second.end(); ++itField) { + attrStr += *itField; + if (itField != itIndexDefine->second.end() - 1) { + attrStr += ","; + } + } + attrStr += "\""; + if (itIndexDefine != std::prev(indexDefines_.end(), 1)) { + attrStr += ","; + } + } + attrStr += "}"; +} + +const std::string &TableInfo::GetDevId() const +{ + return devId_; +} + +void TableInfo::SetDevId(const std::string &devId) +{ + devId_ = devId; +} + +namespace { + std::string VectorJoin(const CompositeFields& fields, char con) + { + std::string res; + auto it = fields.begin(); + res += *it++; + for (; it != fields.end(); ++it) { + res += con + *it; + } + return res; + } +} + +JsonObject TableInfo::ToJsonObject() const +{ + FieldValue jsonField; + JsonObject tableJson; + jsonField.stringValue = tableName_; + tableJson.InsertField(FieldPath { "NAME" }, FieldType::LEAF_FIELD_STRING, jsonField); + jsonField.boolValue = autoInc_; + tableJson.InsertField(FieldPath { "AUTOINCREMENT" }, FieldType::LEAF_FIELD_BOOL, jsonField); + jsonField.stringValue = primaryKey_; + tableJson.InsertField(FieldPath { "PRIMARY_KEY" }, FieldType::LEAF_FIELD_STRING, jsonField); + for (const auto& it : fields_) { + jsonField.stringValue = it.second.ToAttributeString(); + tableJson.InsertField(FieldPath { "DEFINE", it.first }, FieldType::LEAF_FIELD_STRING, jsonField); + } + for (const auto& it : uniqueDefines_) { + jsonField.stringValue = VectorJoin(it, ','); + // TODO: add unique to tableJson + } + return tableJson; +} + +std::string TableInfo::ToTableInfoString() const +{ + std::string attrStr; + attrStr += R"("NAME": ")" + tableName_ + "\","; + AddFieldDefineString(attrStr); + attrStr += R"("AUTOINCREMENT": )"; + if (autoInc_) { + attrStr += "true,"; + } else { + attrStr += "false,"; + } + AddUniqueDefineString(attrStr); + if (!primaryKey_.empty()) { + attrStr += R"("PRIMARY_KEY": ")" + primaryKey_ + "\","; + } + AddIndexDefineString(attrStr); + return attrStr; +} + +int RelationalSyncOpinion::CalculateParcelLen(uint32_t softWareVersion) const +{ + return E_OK; +} + +int RelationalSyncOpinion::Serialize(Parcel &parcel, uint32_t softWareVersion) const +{ + return E_OK; +} + +int RelationalSyncOpinion::Deserialization(const Parcel &parcel) +{ + return E_OK; +} + +const SyncOpinion &RelationalSyncOpinion::GetTableOpinion(const std::string& tableName) const +{ + return opinions_.at(tableName); +} + +void RelationalSyncOpinion::AddSyncOpinion(const std::string &tableName, const SyncOpinion &opinion) +{ + opinions_[tableName] = opinion; +} + +const SyncStrategy &RelationalSyncStrategy::GetTableStrategy(const std::string &tableName) const +{ + return strategies_.at(tableName); +} + +void RelationalSyncStrategy::AddSyncStrategy(const std::string &tableName, const SyncStrategy &strategy) +{ + strategies_[tableName] = strategy; +} + +RelationalSyncOpinion RelationalSchemaObject::MakeLocalSyncOpinion( + const RelationalSchemaObject &localSchema, const std::string &remoteSchema, uint8_t remoteSchemaType) +{ + return RelationalSyncOpinion(); +} + +RelationalSyncStrategy RelationalSchemaObject::ConcludeSyncStrategy( + const RelationalSyncOpinion &localOpinion, const RelationalSyncOpinion &remoteOpinion) +{ + return RelationalSyncStrategy(); +} + +bool RelationalSchemaObject::IsSchemaValid() const +{ + return isValid_; +} + +SchemaType RelationalSchemaObject::GetSchemaType() const +{ + return schemaType_; +} + +std::string RelationalSchemaObject::ToSchemaString() const +{ + return schemaString_; +} + +int RelationalSchemaObject::ParseFromSchemaString(const std::string &inSchemaString) +{ + if (isValid_) { + return -E_NOT_PERMIT; + } + + if (inSchemaString.size() > SCHEMA_STRING_SIZE_LIMIT) { + LOGE("[RelationalSchema][Parse] SchemaSize=%zu Too Large.", inSchemaString.size()); + return -E_INVALID_ARGS; + } + JsonObject schemaObj; + int errCode = schemaObj.Parse(inSchemaString); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Schema json string parse failed: %d.", errCode); + return errCode; + } + + errCode = ParseRelationalSchema(schemaObj); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Parse to relational schema failed: %d.", errCode); + return errCode; + } + + schemaType_ = SchemaType::RELATIVE; + schemaString_ = inSchemaString; + isValid_ = true; + return E_OK; +} + +void RelationalSchemaObject::AddRelationalTable(const TableInfo& tb) +{ + tables_[tb.GetTableName()] = tb; +} + +const std::map &RelationalSchemaObject::GetTables() const +{ + return tables_; +} + +const TableInfo &RelationalSchemaObject::GetTable(const std::string& tableName) const +{ + return tables_.at(tableName); +} + +int RelationalSchemaObject::CompareAgainstSchemaObject(const std::string &inSchemaString, + std::map &cmpRst) const +{ + return E_OK; +} + +int RelationalSchemaObject::CompareAgainstSchemaObject(const RelationalSchemaObject &inSchemaObject, + std::map &cmpRst) const +{ + return E_OK; +} + +namespace { +int GetMemberFromJsonObject(const JsonObject &inJsonObject, const std::string &fieldName, FieldType expectType, + bool isNecessary, FieldValue &fieldValue) +{ + if (!inJsonObject.IsFieldPathExist(FieldPath {fieldName}) && !isNecessary) { + return E_OK; + } + + FieldType fieldType; + int errCode = inJsonObject.GetFieldTypeByFieldPath(FieldPath {fieldName}, fieldType); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Get schema %s fieldType failed: %d.", fieldName.c_str(), errCode); + return -E_SCHEMA_PARSE_FAIL; + } + + if (fieldType != expectType) { + LOGE("[RelationalSchema][Parse] Expect %s fieldType %d but : %d.", fieldName.c_str(), expectType, fieldType); + return -E_SCHEMA_PARSE_FAIL; + } + + errCode = inJsonObject.GetFieldValueByFieldPath(FieldPath {"NAME"}, fieldValue); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Get schema %s value failed: %d.", fieldName.c_str(), errCode); + return -E_SCHEMA_PARSE_FAIL; + } + return E_OK; +} +} + +int RelationalSchemaObject::ParseRelationalSchema(const JsonObject &inJsonObject) +{ + int errCode = ParseCheckSchemaVersionMode(inJsonObject); + if (errCode != E_OK) { + return errCode; + } + errCode = ParseCheckSchemaType(inJsonObject); + if (errCode != E_OK) { + return errCode; + } + errCode = ParseCheckSchemaTableDefine(inJsonObject); + if (errCode != E_OK) { + return errCode; + } + return E_OK; +} + +int RelationalSchemaObject::ParseCheckSchemaVersionMode(const JsonObject &inJsonObject) +{ + FieldValue fieldValue; + int errCode = GetMemberFromJsonObject(inJsonObject, KEYWORD_SCHEMA_VERSION, FieldType::LEAF_FIELD_STRING, + true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + + if (SchemaUtils::Strip(fieldValue.stringValue) != SCHEMA_SUPPORT_VERSION_V2) { + LOGE("[RelationalSchema][Parse] Unexpected SCHEMA_VERSION=%s.", fieldValue.stringValue.c_str()); + return -E_SCHEMA_PARSE_FAIL; + } + schemaVersion_ = SCHEMA_SUPPORT_VERSION_V2; + return E_OK; +} + +int RelationalSchemaObject::ParseCheckSchemaType(const JsonObject &inJsonObject) +{ + FieldValue fieldValue; + int errCode = GetMemberFromJsonObject(inJsonObject, KEYWORD_SCHEMA_TYPE, FieldType::LEAF_FIELD_STRING, + true, fieldValue); + if (errCode != E_OK) { + return errCode; + } + + if (SchemaUtils::Strip(fieldValue.stringValue) != KEYWORD_TYPE_RELATIVE) { + LOGE("[RelationalSchema][Parse] Unexpected SCHEMA_TYPE=%s.", fieldValue.stringValue.c_str()); + return -E_SCHEMA_PARSE_FAIL; + } + schemaType_ = SchemaType::RELATIVE; + return E_OK; +} + +int RelationalSchemaObject::ParseCheckSchemaTableDefine(const JsonObject &inJsonObject) +{ + FieldType fieldType; + int errCode = inJsonObject.GetFieldTypeByFieldPath(FieldPath {KEYWORD_SCHEMA_TABLE}, fieldType); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Get schema TABLES fieldType failed: %d.", errCode); + return -E_SCHEMA_PARSE_FAIL; + } + if (FieldType::LEAF_FIELD_ARRAY != fieldType) { + LOGE("[RelationalSchema][Parse] Expect TABLES fieldType ARRAY but %s.", + SchemaUtils::FieldTypeString(fieldType).c_str()); + return -E_SCHEMA_PARSE_FAIL; + } + std::vector tables; + errCode = inJsonObject.GetObjectArrayByFieldPath(FieldPath{KEYWORD_SCHEMA_TABLE}, tables); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Get schema TABLES value failed: %d.", errCode); + return -E_SCHEMA_PARSE_FAIL; + } + for (const JsonObject &table : tables) { + errCode = ParseCheckTableInfo(table); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Parse schema TABLES failed: %d.", errCode); + return errCode; + } + } + return E_OK; +} + +int RelationalSchemaObject::ParseCheckTableInfo(const JsonObject &inJsonObject) +{ + TableInfo resultTable; + int errCode = ParseCheckTableName(inJsonObject, resultTable); + if (errCode != E_OK) { + return errCode; + } + errCode = ParseCheckTableDefine(inJsonObject, resultTable); + if (errCode != E_OK) { + return errCode; + } + errCode = ParseCheckTableAutoInc(inJsonObject, resultTable); + if (errCode != E_OK) { + return errCode; + } + errCode = ParseCheckTableUnique(inJsonObject, resultTable); + if (errCode != E_OK) { + return errCode; + } + errCode = ParseCheckTablePrimaryKey(inJsonObject, resultTable); + if (errCode != E_OK) { + return errCode; + } + return ParseCheckTableIndex(inJsonObject, resultTable); +} + +int RelationalSchemaObject::ParseCheckTableName(const JsonObject &inJsonObject, TableInfo &resultTable) +{ + FieldValue fieldValue; + int errCode = GetMemberFromJsonObject(inJsonObject, "NAME", FieldType::LEAF_FIELD_STRING, + true, fieldValue); + if (errCode == E_OK) { + resultTable.SetTableName(fieldValue.stringValue); + } + return errCode; +} + +int RelationalSchemaObject::ParseCheckTableDefine(const JsonObject &inJsonObject, TableInfo &resultTable) +{ + std::map tableFields; + int errCode = inJsonObject.GetSubFieldPathAndType(FieldPath {"DEFINE"}, tableFields); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Get schema TABLES DEFINE failed: %d.", errCode); + return -E_SCHEMA_PARSE_FAIL; + } + + for (const auto &field : tableFields) { + if (field.second != FieldType::LEAF_FIELD_STRING) { + LOGE("[RelationalSchema][Parse] Expect schema TABLES DEFINE fieldType STRING but : %s.", + SchemaUtils::FieldTypeString(field.second).c_str()); + return -E_SCHEMA_PARSE_FAIL; + } + FieldValue fieldValue; + errCode = inJsonObject.GetFieldValueByFieldPath(field.first, fieldValue); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Get schema TABLES DEFINE field value failed: %d.", errCode); + return -E_SCHEMA_PARSE_FAIL; + } + + SchemaAttribute outAttr; + errCode = SchemaUtils::ParseAndCheckSchemaAttribute(fieldValue.stringValue, outAttr, false); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Parse schema TABLES DEFINE attribute failed: %d.", errCode); + return errCode; + } + + FieldInfo fieldInfo; + fieldInfo.SetFieldName(field.first[1]); + fieldInfo.SetDataType(outAttr.customFieldType); + fieldInfo.SetNotNull(outAttr.hasNotNullConstraint); + if (outAttr.hasDefaultValue) { + fieldInfo.SetDefaultValue(outAttr.defaultValue.stringValue); + } + resultTable.AddField(fieldInfo); + } + return E_OK; +} + +int RelationalSchemaObject::ParseCheckTableAutoInc(const JsonObject &inJsonObject, TableInfo &resultTable) +{ + FieldValue fieldValue; + int errCode = GetMemberFromJsonObject(inJsonObject, "AUTOINCREMENT", FieldType::LEAF_FIELD_BOOL, + false, fieldValue); + if (errCode == E_OK) { + resultTable.SetAutoIncrement(fieldValue.boolValue); + } + return errCode; +} + +int RelationalSchemaObject::ParseCheckTableUnique(const JsonObject &inJsonObject, TableInfo &resultTable) +{ + std::vector uniqueArray; + int errCode = inJsonObject.GetArrayContentOfStringOrStringArray(FieldPath {"UNIQUE"}, uniqueArray); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Get unique array failed: %d.", errCode); + return -E_SCHEMA_PARSE_FAIL; + } + resultTable.SetUniqueDefines(uniqueArray); + return E_OK; +} + +int RelationalSchemaObject::ParseCheckTablePrimaryKey(const JsonObject &inJsonObject, TableInfo &resultTable) +{ + FieldValue fieldValue; + int errCode = GetMemberFromJsonObject(inJsonObject, "PRIMARY_KEY", FieldType::LEAF_FIELD_STRING, false, fieldValue); + if (errCode == E_OK) { + resultTable.SetPrimaryKey(fieldValue.stringValue); + } + return errCode; +} + +int RelationalSchemaObject::ParseCheckTableIndex(const JsonObject &inJsonObject, TableInfo &resultTable) +{ + std::map tableFields; + int errCode = inJsonObject.GetSubFieldPathAndType(FieldPath {"INDEX"}, tableFields); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Get schema TABLES INDEX failed: %d.", errCode); + return -E_SCHEMA_PARSE_FAIL; + } + + for (const auto &field : tableFields) { + if (field.second != FieldType::LEAF_FIELD_ARRAY) { + LOGE("[RelationalSchema][Parse] Expect schema TABLES INDEX fieldType ARRAY but : %s.", + SchemaUtils::FieldTypeString(field.second).c_str()); + return -E_SCHEMA_PARSE_FAIL; + } + CompositeFields indexDefine; + errCode = inJsonObject.GetStringArrayByFieldPath(field.first, indexDefine); + if (errCode != E_OK) { + LOGE("[RelationalSchema][Parse] Get schema TABLES INDEX field value failed: %d.", errCode); + return -E_SCHEMA_PARSE_FAIL; + } + resultTable.AddIndexDefine(field.first[1], indexDefine); + } + return E_OK; +} +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp index 445de2cc216a2055b84423e39421c6ecd389b809..faaa671e6eef7b97f307b16696a0bf68beb9d934 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.cpp @@ -382,9 +382,9 @@ int RuntimeContextImpl::RunPermissionCheck(const std::string &userId, const std: } int RuntimeContextImpl::EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier) + const AutoLaunchOption &option) { - return autoLaunch_.EnableKvStoreAutoLaunch(properties, notifier, observer, conflictType, conflictNotifier); + return autoLaunch_.EnableKvStoreAutoLaunch(properties, notifier, option); } int RuntimeContextImpl::DisableKvStoreAutoLaunch(const std::string &identifier) @@ -408,7 +408,7 @@ NotificationChain::Listener *RuntimeContextImpl::RegisterLockStatusLister(const { std::lock(lockStatusLock_, systemApiAdapterLock_); std::lock_guard lockStatusLock(lockStatusLock_, std::adopt_lock); - std::lock_guard systemApiAdapterLock(systemApiAdapterLock_, std::adopt_lock); + std::lock_guard systemApiAdapterLock(systemApiAdapterLock_, std::adopt_lock); if (lockStatusObserver_ == nullptr) { lockStatusObserver_ = new (std::nothrow) LockStatusObserver(); if (lockStatusObserver_ == nullptr) { @@ -452,7 +452,7 @@ NotificationChain::Listener *RuntimeContextImpl::RegisterLockStatusLister(const bool RuntimeContextImpl::IsAccessControlled() const { - std::lock_guard autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr) { return false; } @@ -461,7 +461,7 @@ bool RuntimeContextImpl::IsAccessControlled() const int RuntimeContextImpl::SetSecurityOption(const std::string &filePath, const SecurityOption &option) const { - std::lock_guard autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr || !OS::CheckPathExistence(filePath)) { LOGI("Adapter is not set, or path not existed, not support set security option!"); return -E_NOT_SUPPORT; @@ -492,7 +492,7 @@ int RuntimeContextImpl::SetSecurityOption(const std::string &filePath, const Sec int RuntimeContextImpl::GetSecurityOption(const std::string &filePath, SecurityOption &option) const { - std::lock_guard autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr) { LOGI("Get Security option, but not set system api adapter!"); return -E_NOT_SUPPORT; @@ -521,7 +521,7 @@ int RuntimeContextImpl::GetSecurityOption(const std::string &filePath, SecurityO bool RuntimeContextImpl::CheckDeviceSecurityAbility(const std::string &devId, const SecurityOption &option) const { - std::lock_guard autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr) { return true; } @@ -532,7 +532,7 @@ int RuntimeContextImpl::SetProcessSystemApiAdapter(const std::shared_ptr lockStatusLock(lockStatusLock_, std::adopt_lock); - std::lock_guard systemApiAdapterLock(systemApiAdapterLock_, std::adopt_lock); + std::lock_guard systemApiAdapterLock(systemApiAdapterLock_, std::adopt_lock); systemApiAdapter_ = adapter; if (systemApiAdapter_ != nullptr && lockStatusObserver_ != nullptr && lockStatusObserver_->IsStarted()) { auto callback = std::bind(&LockStatusObserver::OnStatusChange, @@ -550,7 +550,7 @@ int RuntimeContextImpl::SetProcessSystemApiAdapter(const std::shared_ptr autoLock(systemApiAdapterLock_); + std::lock_guard autoLock(systemApiAdapterLock_); if (systemApiAdapter_ == nullptr) { return false; } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h index 43f077785ca27076cef86ac477335e3d3f694694..4d78e9261b8baafc926e4b6af8b144b8e9c5ce3c 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/runtime_context_impl.h @@ -70,7 +70,7 @@ public: const std::string &deviceId, uint8_t flag) const override; int EnableKvStoreAutoLaunch(const KvDBProperties &properties, AutoLaunchNotifier notifier, - KvStoreObserver *observer, int conflictType, KvStoreNbConflictNotifier conflictNotifier) override; + const AutoLaunchOption &option) override; int DisableKvStoreAutoLaunch(const std::string &identifier) override; @@ -78,7 +78,7 @@ public: void SetAutoLaunchRequestCallback(const AutoLaunchRequestCallback &callback) override; - NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errorCode) override; + NotificationChain::Listener *RegisterLockStatusLister(const LockStatusNotifier &action, int &errCode) override; bool IsAccessControlled() const override; @@ -93,6 +93,7 @@ public: bool IsProcessSystemApiAdapterValid() const override; bool IsCommunicatorAggregatorValid() const override; + // Notify TIME_CHANGE_EVENT. void NotifyTimeStampChanged(TimeOffset offset) const override; @@ -137,7 +138,7 @@ private: AutoLaunch autoLaunch_; // System api - mutable std::mutex systemApiAdapterLock_; + mutable std::recursive_mutex systemApiAdapterLock_; std::shared_ptr systemApiAdapter_; mutable std::mutex lockStatusLock_; // Mutex for lockStatusObserver_. LockStatusObserver *lockStatusObserver_; diff --git a/services/distributeddataservice/libs/distributeddb/common/src/schema_object.cpp b/services/distributeddataservice/libs/distributeddb/common/src/schema_object.cpp index 3029bd4d4eeb5af831c1dc92d4886f309b7ea36a..6470c54ca8594c5c98a2122c70ef0566abe2c473 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/schema_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/schema_object.cpp @@ -51,7 +51,7 @@ std::string SchemaObject::GetExtractFuncName(SchemaType inSchemaType) } std::string SchemaObject::GenerateExtractSQL(SchemaType inSchemaType, const FieldPath &inFieldpath, - FieldType inFieldType, uint32_t skipSize) + FieldType inFieldType, uint32_t skipSize, const std::string &accessStr) { static std::map fieldTypeMapSQLiteType { {FieldType::LEAF_FIELD_BOOL, "INT"}, @@ -70,7 +70,7 @@ std::string SchemaObject::GenerateExtractSQL(SchemaType inSchemaType, const Fiel } std::string resultSql = " CAST("; // Reserve blank at begin for convenience. resultSql += GetExtractFuncName(inSchemaType); - resultSql += "(value, '"; + resultSql += "(" + accessStr + "value, '"; resultSql += SchemaUtils::FieldPathString(inFieldpath); resultSql += "', "; resultSql += std::to_string(skipSize); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/schema_utils.cpp b/services/distributeddataservice/libs/distributeddb/common/src/schema_utils.cpp index 8f90faa881dd4768c772b20bdc16bc73ba281195..c1fe1fe961308bdbe95808690deed0bdf3af82c1 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/schema_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/schema_utils.cpp @@ -276,7 +276,8 @@ int SchemaUtils::TransformDefaultValue(std::string &defaultContent, SchemaAttrib return errCode; } -int SchemaUtils::ParseAndCheckSchemaAttribute(const std::string &inAttrString, SchemaAttribute &outAttr) +int SchemaUtils::ParseAndCheckSchemaAttribute(const std::string &inAttrString, SchemaAttribute &outAttr, + bool useAffinity) { if (inAttrString.empty()) { return -E_SCHEMA_PARSE_FAIL; @@ -290,7 +291,7 @@ int SchemaUtils::ParseAndCheckSchemaAttribute(const std::string &inAttrString, S LOGD("Syntax error, please check!"); return errCode; } - errCode = ParseSchemaAttribute(attrContext, outAttr); + errCode = ParseSchemaAttribute(attrContext, outAttr, useAffinity); if (errCode != E_OK) { LOGD("Grammatical error, please check!"); return errCode; @@ -299,7 +300,7 @@ int SchemaUtils::ParseAndCheckSchemaAttribute(const std::string &inAttrString, S return E_OK; } -int SchemaUtils::ParseSchemaAttribute(std::vector &attrContext, SchemaAttribute &outAttr) +int SchemaUtils::ParseSchemaAttribute(std::vector &attrContext, SchemaAttribute &outAttr, bool useAffinity) { // After split attribute? attrContext include 3 type field if (attrContext.size() < 3) { @@ -307,11 +308,16 @@ int SchemaUtils::ParseSchemaAttribute(std::vector &attrContext, Sch return -E_SCHEMA_PARSE_FAIL; } TrimFiled(attrContext[0]); - if (FIELD_TYPE_DIC.find(attrContext[0]) == FIELD_TYPE_DIC.end()) { - LOGE("Errno schema field type [%s]!!", attrContext[0].c_str()); - return -E_SCHEMA_PARSE_FAIL; + if (!useAffinity) { + if (FIELD_TYPE_DIC.find(attrContext[0]) == FIELD_TYPE_DIC.end()) { + LOGE("Errno schema field type [%s]!!", attrContext[0].c_str()); + return -E_SCHEMA_PARSE_FAIL; + } else { + outAttr.type = FIELD_TYPE_DIC.at(attrContext[0]); + } } else { - outAttr.type = FIELD_TYPE_DIC.at(attrContext[0]); + outAttr.type = FieldType::LEAF_FIELD_NULL; + outAttr.customFieldType = attrContext[0]; } outAttr.hasNotNullConstraint = !attrContext[1].empty(); diff --git a/services/distributeddataservice/libs/distributeddb/common/src/semaphore_utils.cpp b/services/distributeddataservice/libs/distributeddb/common/src/semaphore_utils.cpp old mode 100755 new mode 100644 index f2ab5388defdb3cf5221277860dfac620c240716..fad1df052faf422321bcc70c31fa8443ce0c997f --- a/services/distributeddataservice/libs/distributeddb/common/src/semaphore_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/common/src/semaphore_utils.cpp @@ -19,6 +19,7 @@ namespace DistributedDB { using std::unique_lock; +using std::lock_guard; using std::mutex; using std::condition_variable; @@ -49,7 +50,7 @@ void SemaphoreUtils::WaitSemaphore() void SemaphoreUtils::SendSemaphore() { - unique_lock lock(lockMutex_); + lock_guard lock(lockMutex_); count_++; cv_.notify_one(); } diff --git a/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.h b/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.h index 64d02004814383deee193c0e163fb8a101aebb91..35b409208670becf3d8f9ffdb551455ddc2e19a1 100755 --- a/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.h +++ b/services/distributeddataservice/libs/distributeddb/common/src/time_tick_monitor.h @@ -39,7 +39,7 @@ public: NotificationChain::Listener *RegisterTimeChangedLister(const TimeChangedAction &action, int &errCode); // Notify TIME_CHANGE_EVENT. - void NotifyTimeChange(TimeOffset timeChangeOffset) const; + void NotifyTimeChange(TimeOffset offset) const; private: static constexpr uint64_t MONITOR_INTERVAL = 1 * 1000; // 1s static constexpr int64_t MAX_NOISE = 9 * 100 * 1000; // 900ms diff --git a/services/distributeddataservice/libs/distributeddb/common/src/zlib_compression.cpp b/services/distributeddataservice/libs/distributeddb/common/src/zlib_compression.cpp new file mode 100644 index 0000000000000000000000000000000000000000..aeef6b3eafc44a182b028eb8f270fb82281d8230 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/common/src/zlib_compression.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "zlib_compression.h" +#ifndef OMIT_ZLIB +#include + +#include "db_constant.h" +#include "db_errno.h" +#include "log_print.h" +#include "types_export.h" + +namespace DistributedDB { +static ZlibCompression g_zlibInstance; + +ZlibCompression::ZlibCompression() +{ + DataCompression::Register(CompressAlgorithm::ZLIB, this); +} + +int ZlibCompression::Compress(const std::vector &srcData, std::vector &destData) const +{ + auto srcLen = srcData.size(); + auto destLen = compressBound(srcLen); + if (srcLen > DBConstant::MAX_SYNC_BLOCK_SIZE || destLen > DBConstant::MAX_SYNC_BLOCK_SIZE) { + LOGE("Too long to compress, srcLen:%u, destLen:%u.", srcLen, destLen); + return -E_INVALID_ARGS; + } + + // Alloc memory. + destData.resize(destLen); + + // Compress. + int errCode = compress(destData.data(), &destLen, srcData.data(), srcLen); + if (errCode != Z_OK) { + LOGE("Compress parcel failed, errCode = %d", errCode); + return -E_SYSTEM_API_FAIL; + } + + destData.resize(destLen); + destData.shrink_to_fit(); + return E_OK; +} + +int ZlibCompression::Uncompress(const std::vector &srcData, std::vector &destData, + unsigned long destLen) const +{ + auto srcLen = srcData.size(); + if (srcLen > DBConstant::MAX_SYNC_BLOCK_SIZE || destLen > DBConstant::MAX_SYNC_BLOCK_SIZE) { + LOGE("Too long to uncompress, srcLen:%u, destLen:%u.", srcLen, destLen); + return -E_INVALID_ARGS; + } + + // Alloc dest memory. + destData.resize(destLen); + + // Uncompress. + int errCode = uncompress(destData.data(), &destLen, srcData.data(), srcData.size()); + if (errCode != Z_OK) { + LOGE("Uncompress failed, errCode = %d", errCode); + return -E_SYSTEM_API_FAIL; + } + + destData.resize(destLen); + destData.shrink_to_fit(); + return E_OK; +} +} // namespace DistributedDB +#endif // OMIT_ZLIB diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/communicator_aggregator.h b/services/distributeddataservice/libs/distributeddb/communicator/include/communicator_aggregator.h index c91cd42b6b8d5e0c8e85b1841ec2f3a410cd20df..ae308a7b6d5037b282748d4d10bc6be6f0ecde7f 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/communicator_aggregator.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/communicator_aggregator.h @@ -55,6 +55,7 @@ public: // See ICommunicatorAggregator for detail int Initialize(IAdapter *inAdapter) override; + // Must not call any other functions if Finalize had been called. In fact, Finalize has no chance to be called. void Finalize() override; @@ -69,7 +70,13 @@ public: // return optimal allowed data size(Some header is taken into account and subtract) uint32_t GetCommunicatorAggregatorMtuSize() const; uint32_t GetCommunicatorAggregatorMtuSize(const std::string &target) const; + + // return timeout in range [5s, 60s] + uint32_t GetCommunicatorAggregatorTimeout() const; + uint32_t GetCommunicatorAggregatorTimeout(const std::string &target) const; + bool IsDeviceOnline(const std::string &device) const; int GetLocalIdentity(std::string &outTarget) const; + // Get the protocol version of remote target. Return -E_NOT_FOUND if no record. int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const; @@ -104,6 +111,7 @@ private: int OnAppLayerFrameReceive(const std::string &srcTarget, const uint8_t *bytes, uint32_t length, const ParseResult &inResult); int OnAppLayerFrameReceive(const std::string &srcTarget, SerialBuffer *&inFrameBuffer, const ParseResult &inResult); + // Function with suffix NoMutex should be called with mutex in the caller int TryDeliverAppLayerFrameToCommunicatorNoMutex(const std::string &srcTarget, SerialBuffer *&inFrameBuffer, const LabelType &toLabel); @@ -130,6 +138,7 @@ private: std::atomic shutdown_; std::atomic incFrameId_; std::atomic localSourceId_; + // Handle related mutable std::mutex commMapMutex_; std::map> commMap_; // bool true indicate communicator activated @@ -138,21 +147,26 @@ private: SendTaskScheduler scheduler_; IAdapter *adapterHandle_ = nullptr; CommunicatorLinker *commLinker_ = nullptr; + // Thread related std::thread exclusiveThread_; bool wakingSignal_ = false; mutable std::mutex wakingMutex_; std::condition_variable wakingCv_; + // RetryCreateTask related mutable std::mutex retryMutex_; std::condition_variable retryCv_; + // Remote target version related mutable std::mutex versionMapMutex_; std::map versionMap_; + // CommLack Callback related CommunicatorLackCallback onCommLackHandle_; Finalizer onCommLackFinalizer_; mutable std::mutex onCommLackMutex_; + // Connect Callback related OnConnectCallback onConnectHandle_; Finalizer onConnectFinalizer_; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/frame_combiner.h b/services/distributeddataservice/libs/distributeddb/communicator/include/frame_combiner.h index 0fe738698db488a8ad92af4244202c0a2cef0516..33124c497c406cbd976f569d8614838d7af6e688 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/frame_combiner.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/frame_combiner.h @@ -42,6 +42,7 @@ public: // Start the timer to supervise the progress void Initialize(); + // Clear the CombineWorkPool and stop the timer void Finalize(); @@ -68,7 +69,7 @@ private: TimerId timerId_ = 0; // 0 is invalid timerId bool isTimerWork_ = false; - SemaphoreUtils timerRemovedIndicator_{0}; + SemaphoreUtils timerRemovedIndicator_ {0}; uint64_t incProgressId_ = 0; uint64_t totalSizeByByte_ = 0; std::map> combineWorkPool_; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/frame_retainer.h b/services/distributeddataservice/libs/distributeddb/communicator/include/frame_retainer.h index 5ea246874788f391b8ca60b5881c9f4a92a75236..259007369b33411447f08540796833d1b75c028e 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/frame_retainer.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/frame_retainer.h @@ -47,6 +47,7 @@ public: // Start the timer to clear up overtime frames void Initialize(); + // Stop the timer and clear the RetainWorkPool void Finalize(); diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/iadapter.h b/services/distributeddataservice/libs/distributeddb/communicator/include/iadapter.h index 4bdab0b765a92a5c9171af19cee7e7a426497639..c6d3b0078850501dd94b0134ec44ad7026b2225c 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/iadapter.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/iadapter.h @@ -42,6 +42,10 @@ public: virtual uint32_t GetMtuSize() = 0; virtual uint32_t GetMtuSize(const std::string &target) = 0; + // Should returns timeout in range [5s, 60s] + virtual uint32_t GetTimeout() = 0; + virtual uint32_t GetTimeout(const std::string &target) = 0; + // Get local target name for identify self virtual int GetLocalIdentity(std::string &outTarget) = 0; @@ -61,6 +65,8 @@ public: // Return 0 as success. Return negative as error virtual int RegSendableCallback(const SendableCallback &onSendable, const Finalizer &inOper) = 0; + virtual bool IsDeviceOnline(const std::string &device) = 0; + virtual ~IAdapter() {}; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/icommunicator.h b/services/distributeddataservice/libs/distributeddb/communicator/include/icommunicator.h index 30060655bde2c6f8fb88302f49557e012afe618c..a7e0500622ab9fae7f5f67f6a67d3ad047a3cdf1 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/icommunicator.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/icommunicator.h @@ -39,8 +39,16 @@ public: // return optimal allowed data size(Some header is taken into account and subtract) virtual uint32_t GetCommunicatorMtuSize() const = 0; virtual uint32_t GetCommunicatorMtuSize(const std::string &target) const = 0; + + // return timeout in range [5s, 60s] + virtual uint32_t GetTimeout() const = 0; + virtual uint32_t GetTimeout(const std::string &target) const = 0; + + virtual bool IsDeviceOnline(const std::string &device) const = 0; + // Get local target name for identify self virtual int GetLocalIdentity(std::string &outTarget) const = 0; + // Get the protocol version of remote target. Return -E_NOT_FOUND if no record. virtual int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const = 0; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/message.h b/services/distributeddataservice/libs/distributeddb/communicator/include/message.h index 791c55e8f4ae640d99040a0482c7e7c0f94df6ad..7c99c9def64ebb9a69c1eb99a2a91127c2ee3ff8 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/message.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/message.h @@ -194,6 +194,12 @@ public: { return version_; } + + bool IsFeedbackError() const + { + return (errorNo_ == E_FEEDBACK_UNKNOWN_MESSAGE || errorNo_ == E_FEEDBACK_COMMUNICATOR_NOT_FOUND); + } + private: // Field or content that will be serialized for bytes transfer uint16_t version_ = MSG_VERSION_BASE; @@ -203,6 +209,7 @@ private: uint32_t sequenceId_ = 0; // Distinguish different message even in same session with same content in retry case uint32_t errorNo_ = NO_ERROR; ObjectHolder *holderPtr_ = nullptr; + // Field carry supplemental info std::string target_; Priority prio_ = Priority::LOW; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/network_adapter.h b/services/distributeddataservice/libs/distributeddb/communicator/include/network_adapter.h index 1116bf74df00ef63c6b0cb17287fbe36b0c0ae27..9954b651f153e88e65bd9ceb8bdc70a0a667a5c6 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/network_adapter.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/network_adapter.h @@ -39,6 +39,9 @@ public: uint32_t GetMtuSize() override; uint32_t GetMtuSize(const std::string &target) override; + + uint32_t GetTimeout() override; + uint32_t GetTimeout(const std::string &target) override; int GetLocalIdentity(std::string &outTarget) override; int SendBytes(const std::string &dstTarget, const uint8_t *bytes, uint32_t length) override; @@ -46,6 +49,9 @@ public: int RegBytesReceiveCallback(const BytesReceiveCallback &onReceive, const Finalizer &inOper) override; int RegTargetChangeCallback(const TargetChangeCallback &onChange, const Finalizer &inOper) override; int RegSendableCallback(const SendableCallback &onSendable, const Finalizer &inOper) override; + + bool IsDeviceOnline(const std::string &device) override; + private: void OnDataReceiveHandler(const DeviceInfos &srcDevInfo, const uint8_t *data, uint32_t length); void OnDeviceChangeHandler(const DeviceInfos &devInfo, bool isOnline); @@ -59,7 +65,7 @@ private: // For protecting "LocalIdentity" and "MtuSize", these info only need to get from peripheral interface once mutable std::mutex identityMutex_; - bool isLocalIdentityValid_ = false; + std::string localIdentity_; mutable std::mutex mtuSizeMutex_; bool isMtuSizeValid_ = false; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/parse_result.h b/services/distributeddataservice/libs/distributeddb/communicator/include/parse_result.h index e6b1db26c07b33e44613fa99daf54386eea7f1c8..0e1278821a1492a77cf03e5780ac876fcb4a60ec 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/parse_result.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/parse_result.h @@ -154,13 +154,16 @@ private: uint8_t paddingLen_ = 0; bool isFragment_ = false; FrameType frameType_ = FrameType::INVALID_MAX_FRAME_TYPE; + // For CommPhyOptHeader uint32_t frameLen_ = 0; uint16_t fragCount_ = 0; uint16_t fragNo_ = 0; + // For Application Layer Frame uint32_t payloadLen_ = 0; LabelType commLabel_; + // For Communication Layer Frame uint64_t labelExchangeDistinctValue_ = 0; // For Both LabelExchange And LabelExchangeAck Frame uint64_t labelExchangeSequenceId_ = 0; // For Both LabelExchange And LabelExchangeAck Frame diff --git a/services/distributeddataservice/libs/distributeddb/communicator/include/send_task_scheduler.h b/services/distributeddataservice/libs/distributeddb/communicator/include/send_task_scheduler.h index a4be7923e2e593226f830a905f710146a771e36c..8c1e3e12ec32d62d6dd55ad06ba06e00473b05fa 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/include/send_task_scheduler.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/include/send_task_scheduler.h @@ -54,14 +54,17 @@ public: DISABLE_COPY_ASSIGN_MOVE(SendTaskScheduler); void Initialize(); + // This method for consumer void Finalize(); // This method for producer, support multiple thread int AddSendTaskIntoSchedule(const SendTask &inTask, Priority inPrio); + // This method for consumer, not recommend for multiple thread int ScheduleOutSendTask(SendTask &outTask); int ScheduleOutSendTask(SendTask &outTask, SendTaskInfo &outTaskInfo); + // This method for consumer, call ScheduleOutSendTask at least one time before each calling this int FinalizeLastScheduleTask(); diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.cpp index 394c6621f48317df6255c5b55bd7c9f22b890a62..b1b4e50090dc7d87fb1f5155aecc30cedd382364 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.cpp @@ -80,6 +80,21 @@ int Communicator::GetLocalIdentity(std::string &outTarget) const return commAggrHandle_->GetLocalIdentity(outTarget); } +uint32_t Communicator::GetTimeout() const +{ + return commAggrHandle_->GetCommunicatorAggregatorTimeout(); +} + +uint32_t Communicator::GetTimeout(const std::string &target) const +{ + return commAggrHandle_->GetCommunicatorAggregatorTimeout(target); +} + +bool Communicator::IsDeviceOnline(const std::string &device) const +{ + return commAggrHandle_->IsDeviceOnline(device); +} + int Communicator::SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout) { return SendMessage(dstTarget, inMsg, nonBlock, timeout, nullptr); diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h index e4668978f316fe839009021f8ff09b815698e74a..f0e929e6f750d3adfcb77aea29d358245e912b49 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator.h @@ -43,6 +43,10 @@ public: uint32_t GetCommunicatorMtuSize() const override; uint32_t GetCommunicatorMtuSize(const std::string &target) const override; + + uint32_t GetTimeout() const override; + uint32_t GetTimeout(const std::string &target) const override; + bool IsDeviceOnline(const std::string &device) const override; int GetLocalIdentity(std::string &outTarget) const override; // Get the protocol version of remote target. Return -E_NOT_FOUND if no record. int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const override; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_aggregator.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_aggregator.cpp index 9bc59ee4e8b05f510bb8487b68133411a8ccd263..cbd976f7c18ea36c146b4b6d4282e9748f7b75be 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_aggregator.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_aggregator.cpp @@ -211,6 +211,21 @@ uint32_t CommunicatorAggregator::GetCommunicatorAggregatorMtuSize(const std::str return adapterHandle_->GetMtuSize(target) - ProtocolProto::GetLengthBeforeSerializedData(); } +uint32_t CommunicatorAggregator::GetCommunicatorAggregatorTimeout() const +{ + return adapterHandle_->GetTimeout(); +} + +uint32_t CommunicatorAggregator::GetCommunicatorAggregatorTimeout(const std::string &target) const +{ + return adapterHandle_->GetTimeout(target); +} + +bool CommunicatorAggregator::IsDeviceOnline(const std::string &device) const +{ + return adapterHandle_->IsDeviceOnline(device); +} + int CommunicatorAggregator::GetLocalIdentity(std::string &outTarget) const { return adapterHandle_->GetLocalIdentity(outTarget); @@ -506,8 +521,8 @@ void CommunicatorAggregator::OnTargetChange(const std::string &target, bool isCo LOGI("[CommAggr][OnTarget] ConnectHandle invalid currently."); } } - // For communicator level target change std::set relatedLabels; + // For communicator level target change if (isConnect) { int errCode = commLinker_->TargetOnline(target, relatedLabels); if (errCode != E_OK) { @@ -523,7 +538,7 @@ void CommunicatorAggregator::OnTargetChange(const std::string &target, bool isCo std::lock_guard commMapLockGuard(commMapMutex_); for (auto &entry : commMap_) { // Ignore nonactivated communicator - if (relatedLabels.count(entry.first) != 0 && entry.second.second) { + if (entry.second.second && (!isConnect || (relatedLabels.count(entry.first) != 0))) { entry.second.first->OnConnectChange(target, isConnect); } } @@ -759,19 +774,20 @@ void CommunicatorAggregator::GenerateLocalSourceId() // When GetLocalIdentity fail, the identity be an empty string, the localSourceId be zero, need regenerate // The localSourceId is std::atomic, so there is no concurrency risk uint64_t identityHash = Hash::HashFunc(identity); + if (identityHash != localSourceId_) { + LOGI("[CommAggr][GenSrcId] identity=%s{private}, localSourceId=%llu.", identity.c_str(), ULL(identityHash)); + } localSourceId_ = identityHash; - LOGI("[CommAggr][GenSrcId] identity=%s{private}, localSourceId=%llu.", identity.c_str(), ULL(identityHash)); } bool CommunicatorAggregator::ReGenerateLocalSourceIdIfNeed() { - // If localSourceId is zero, pre-generate must have used an empty identity, re-fetch the identity and generate. + // The deviceId will change when switch user from A to B + // We can't listen to the user change, because it's hard to ensure the timing is correct. + // So we regenerate to make sure the deviceId and localSourceId is correct when we create send task. // The localSourceId is std::atomic, so there is no concurrency risk, no need lockguard here. - if (localSourceId_ == 0) { - GenerateLocalSourceId(); - return (localSourceId_ != 0); - } - return true; + GenerateLocalSourceId(); + return (localSourceId_ != 0); } void CommunicatorAggregator::TriggerVersionNegotiation(const std::string &dstTarget) diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.cpp index b3461bb77670a92e7e8053d3ad2c5a7b43ee27a0..87b29ec238e43df9e3899058599cede46e5fddc9 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.cpp @@ -319,11 +319,11 @@ int CommunicatorLinker::TriggerLabelExchangeEvent(const std::string &toTarget) int CommunicatorLinker::TriggerLabelExchangeAckEvent(const std::string &toTarget, uint64_t inSequenceId) { // Build LabelExchangeAck Frame - int error = E_OK; - SerialBuffer *buffer = ProtocolProto::BuildLabelExchangeAck(localDistinctValue_, inSequenceId, errno); - if (error != E_OK) { - LOGE("[Linker][TriggerAck] BuildAck fail, error=%d", error); - return error; + int errCode = E_OK; + SerialBuffer *buffer = ProtocolProto::BuildLabelExchangeAck(localDistinctValue_, inSequenceId, errCode); + if (errCode != E_OK) { + LOGE("[Linker][TriggerAck] BuildAck fail, error=%d", errCode); + return errCode; } // Apply for a latest ackId and update ackTriggerId_ uint64_t ackId; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.h b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.h index 58065f40ab84be6830eee1d252c7bd7d1acad950..d8d0740f5b14b2f63b0192b7cf587a2cb092d6a2 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/communicator_linker.h @@ -94,19 +94,26 @@ private: CommunicatorAggregator *aggregator_ = nullptr; mutable std::mutex entireInfoMutex_; + // Point out the distinctValue for each target in order to detect malfunctioning "target offline" std::map targetDistinctValue_; + // Point out the largest sequenceId of LabelExchange that ever received for each target std::map topRecvLabelSeq_; + // Point out currently which sequenceId of ack is being waited for each target std::map waitAckSeq_; + // Point out the largest sequenceId of LabelExchangeAck that ever received for each target std::map recvAckSeq_; + // Point out the latest ackTriggerId for each target in order to abort outdated triggered event std::map ackTriggerId_; + // Core Info : Online Labels std::set localOnlineLabels_; std::set remoteOnlineTarget_; + // remember the opened labels no matter target now online or offline std::map> targetMapOnlineLabels_; }; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/network_adapter.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/network_adapter.cpp index b521f9a56fb2fbeba50215fb0e9c35088f57c408..efb297927e176527daed2e5acfed15097cbf1e44 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/network_adapter.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/network_adapter.cpp @@ -14,6 +14,7 @@ */ #include "network_adapter.h" +#include "db_constant.h" #include "db_errno.h" #include "log_print.h" #include "runtime_context.h" @@ -22,8 +23,6 @@ namespace DistributedDB { namespace { const std::string DEFAULT_PROCESS_LABEL = "Distributeddb_Anonymous_Process"; const std::string SCHEDULE_QUEUE_TAG = "NetworkAdapter"; -constexpr uint32_t MIN_MTU_SIZE = 1024; // 1KB -constexpr uint32_t MAX_MTU_SIZE = 5242880; // 5MB } NetworkAdapter::NetworkAdapter() @@ -121,14 +120,25 @@ void NetworkAdapter::StopAdapter() namespace { uint32_t CheckAndAdjustMtuSize(uint32_t inMtuSize) { - if (inMtuSize < MIN_MTU_SIZE) { - return MIN_MTU_SIZE; - } else if (inMtuSize > MAX_MTU_SIZE) { - return MAX_MTU_SIZE; + if (inMtuSize < DBConstant::MIN_MTU_SIZE) { + return DBConstant::MIN_MTU_SIZE; + } else if (inMtuSize > DBConstant::MAX_MTU_SIZE) { + return DBConstant::MAX_MTU_SIZE; } else { return (inMtuSize - (inMtuSize % sizeof(uint64_t))); // Octet alignment } } + +uint32_t CheckAndAdjustTimeout(uint32_t inTimeout) +{ + if (inTimeout < DBConstant::MIN_TIMEOUT) { + return DBConstant::MIN_TIMEOUT; + } else if (inTimeout > DBConstant::MAX_TIMEOUT) { + return DBConstant::MAX_TIMEOUT; + } else { + return inTimeout; + } +} } uint32_t NetworkAdapter::GetMtuSize() @@ -145,6 +155,12 @@ uint32_t NetworkAdapter::GetMtuSize() uint32_t NetworkAdapter::GetMtuSize(const std::string &target) { +#ifndef OMIT_MTU_CACHE + DeviceInfos devInfo; + devInfo.identifier = target; + uint32_t oriMtuSize = processCommunicator_->GetMtuSize(devInfo); + return CheckAndAdjustMtuSize(oriMtuSize); +#else std::lock_guard mtuSizeLockGuard(mtuSizeMutex_); if (devMapMtuSize_.count(target) == 0) { DeviceInfos devInfo; @@ -154,20 +170,36 @@ uint32_t NetworkAdapter::GetMtuSize(const std::string &target) devMapMtuSize_[target] = CheckAndAdjustMtuSize(oriMtuSize); } return devMapMtuSize_[target]; +#endif +} + +uint32_t NetworkAdapter::GetTimeout() +{ + uint32_t timeout = processCommunicator_->GetTimeout(); + LOGI("[NAdapt][GetTimeout] timeout_=%u ms.", timeout); + return CheckAndAdjustTimeout(timeout); +} + +uint32_t NetworkAdapter::GetTimeout(const std::string &target) +{ + DeviceInfos devInfos; + devInfos.identifier = target; + uint32_t timeout = processCommunicator_->GetTimeout(devInfos); + LOGI("[NAdapt][GetTimeout] timeout=%u ms of target=%s{private}.", timeout, target.c_str()); + return CheckAndAdjustTimeout(timeout); } int NetworkAdapter::GetLocalIdentity(std::string &outTarget) { std::lock_guard identityLockGuard(identityMutex_); - if (!isLocalIdentityValid_) { - DeviceInfos devInfo = processCommunicator_->GetLocalDeviceInfos(); + DeviceInfos devInfo = processCommunicator_->GetLocalDeviceInfos(); + if (devInfo.identifier.empty()) { + return -E_PERIPHERAL_INTERFACE_FAIL; + } + if (devInfo.identifier != localIdentity_) { LOGI("[NAdapt][GetLocal] localIdentity=%s{private}.", devInfo.identifier.c_str()); - if (devInfo.identifier.empty()) { - return -E_PERIPHERAL_INTERFACE_FAIL; - } - localIdentity_ = devInfo.identifier; - isLocalIdentityValid_ = true; } + localIdentity_ = devInfo.identifier; outTarget = localIdentity_; return E_OK; } @@ -348,4 +380,13 @@ void NetworkAdapter::CheckDeviceOfflineAfterSendFail(const DeviceInfos &devInfo) } } } + +bool NetworkAdapter::IsDeviceOnline(const std::string &device) +{ + std::lock_guard onlineRemoteDevLockGuard(onlineRemoteDevMutex_); + if (onlineRemoteDev_.find(device) != onlineRemoteDev_.end()) { + return true; + } + return false; +} } diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.cpp b/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.cpp index df5d93fc830a0670ae6494e6936093f7f2c70bee..73c99c0c1feb88a892fbdf6c1175ebb99e4f4842 100755 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.cpp +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.cpp @@ -18,6 +18,7 @@ #include #include "hash.h" #include "securec.h" +#include "version.h" #include "db_common.h" #include "log_print.h" #include "macro_utils.h" @@ -28,7 +29,8 @@ namespace DistributedDB { namespace { const uint16_t MAGIC_CODE = 0xAAAA; const uint16_t PROTOCOL_VERSION = 0; -const uint16_t DB_GLOBAL_VERSION = 2; // Compatibility Final Method. 2 Correspond To Version 1.1.3(103) +// Compatibility Final Method. 3 Correspond To Version 1.1.4(104) +const uint16_t DB_GLOBAL_VERSION = SOFTWARE_VERSION_CURRENT - SOFTWARE_VERSION_EARLIEST; const uint8_t PACKET_TYPE_FRAGMENTED = BITX(0); // Use bit 0 const uint8_t PACKET_TYPE_NOT_FRAGMENTED = 0; const uint8_t MAX_PADDING_LEN = 7; @@ -622,7 +624,6 @@ int ProtocolProto::DeSerializeMessage(const SerialBuffer *inBuff, Message *inMsg LOGE("[Proto][DeSerialize] dataLen=%u, msgDataLen=%u.", dataLen, messageHdr.dataLen); return -E_LENGTH_ERROR; } - // It is better to check FeedbackMessage first and check onlyMsgHeader flag later if (IsFeedbackErrorMessage(messageHdr.errorNo)) { LOGI("[Proto][DeSerialize] Feedback Message with errorNo=%u.", messageHdr.errorNo); @@ -900,6 +901,9 @@ int ProtocolProto::FrameFragmentation(const uint8_t *splitStartBytes, uint32_t s const CommPhyHeader &framePhyHeader, std::vector> &outPieces) { // It can be guaranteed that fragCount >= 2 and also won't be too large + if (fragCount < 2) { + return -E_INVALID_ARGS; + } outPieces.resize(fragCount); // Note: should use resize other than reserve uint32_t quotient = splitLength / fragCount; uint16_t remainder = splitLength % fragCount; diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.h b/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.h index 2e28b8c1f72fc67ab390bed82ffa1e3a323bf050..0a1e6211b88b9c4a3e224e3f5ce82d94fd43ff0d 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/protocol_proto.h @@ -36,6 +36,7 @@ public: // For application layer frame static uint32_t GetAppLayerFrameHeaderLength(); static uint32_t GetLengthBeforeSerializedData(); + // For communication layer frame static uint32_t GetCommLayerFrameHeaderLength(); @@ -54,6 +55,7 @@ public: static int SplitFrameIntoPacketsIfNeed(const SerialBuffer *inBuff, uint32_t inMtuSize, std::vector> &outPieces); static int AnalyzeSplitStructure(const ParseResult &inResult, uint32_t &outFragLen, uint32_t &outLastFragLen); + // inFrame is the destination, pktBytes and pktLength are the source, fragOffset and fragLength give the boundary static int CombinePacketIntoFrame(SerialBuffer *inFrame, const uint8_t *pktBytes, uint32_t pktLength, uint32_t fragOffset, uint32_t fragLength); @@ -65,12 +67,14 @@ public: // For application layer frame. In send case. Focus on frame. static int SetDivergeHeader(SerialBuffer *inBuff, const LabelType &inCommLabel); + // For both application and communication layer frame. In send case. Focus on frame. static int SetPhyHeader(SerialBuffer *inBuff, const PhyHeaderInfo &inInfo); // In receive case, return error if parse fail. static int CheckAndParsePacket(const std::string &srcTarget, const uint8_t *bytes, uint32_t length, ParseResult &outResult); + // The CommPhyHeader had already been parsed into outResult static int CheckAndParseFrame(const SerialBuffer *inBuff, ParseResult &outResult); diff --git a/services/distributeddataservice/libs/distributeddb/communicator/src/serial_buffer.h b/services/distributeddataservice/libs/distributeddb/communicator/src/serial_buffer.h index 135164744de89920ff14f78d844b5072c64481ba..f343053dcdad22b7b6834ce633cc186bc42cbb19 100644 --- a/services/distributeddataservice/libs/distributeddb/communicator/src/serial_buffer.h +++ b/services/distributeddataservice/libs/distributeddb/communicator/src/serial_buffer.h @@ -40,6 +40,7 @@ public: // Create a SerialBuffer that has a independent bytes_ and point to the same externalBytes_ SerialBuffer *Clone(int &outErrorNo); + // After return E_OK, this SerialBuffer can cross thread. Do nothing indeed if it already able to cross thread. int ConvertForCrossThread(); diff --git a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h index 2c9e1f1b7cb9d98957af77ccf8971c29936f9d5b..ea7376a1a1b723b7e90e26e992947027b3972f92 100755 --- a/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h +++ b/services/distributeddataservice/libs/distributeddb/include/auto_launch_export.h @@ -33,6 +33,11 @@ struct AutoLaunchOption { int conflictType = 0; KvStoreNbConflictNotifier notifier; SecurityOption secOption; + bool isNeedIntegrityCheck = false; + bool isNeedRmCorruptedDb = false; + bool isNeedCompressOnSync = false; + uint8_t compressionRate = 100; // valid in [1, 100]. + bool isAutoSync = true; }; struct AutoLaunchParam { diff --git a/services/distributeddataservice/libs/distributeddb/include/query.h b/services/distributeddataservice/libs/distributeddb/include/query.h index e8b90b2834ed2cd11252009620eb1b08aabc2acb..91271f6a3c281f647ae78394fc4108874e6e01bf 100755 --- a/services/distributeddataservice/libs/distributeddb/include/query.h +++ b/services/distributeddataservice/libs/distributeddb/include/query.h @@ -29,11 +29,8 @@ class Query { public: // Do not support concurrent use of query objects - DB_API static Query Select() - { - Query query; - return query; - } + DB_API static Query Select(); + DB_API static Query Select(const std::string &tableName); template DB_API Query &EqualTo(const std::string &field, const T &value) @@ -89,29 +86,13 @@ public: return *this; } - DB_API Query &OrderBy(const std::string &field, bool isAsc = true) - { - ExecuteOrderBy(field, isAsc); - return *this; - } + DB_API Query &OrderBy(const std::string &field, bool isAsc = true); - DB_API Query &Limit(int number, int offset = 0) - { - ExecuteLimit(number, offset); - return *this; - } + DB_API Query &Limit(int number, int offset = 0); - DB_API Query &Like(const std::string &field, const std::string &value) - { - ExecuteLike(field, value); - return *this; - } + DB_API Query &Like(const std::string &field, const std::string &value); - DB_API Query &NotLike(const std::string &field, const std::string &value) - { - ExecuteNotLike(field, value); - return *this; - } + DB_API Query &NotLike(const std::string &field, const std::string &value); template DB_API Query &In(const std::string &field, const std::vector &values) @@ -143,23 +124,11 @@ public: return *this; } - DB_API Query &IsNull(const std::string &field) - { - ExecuteIsNull(field); - return *this; - } + DB_API Query &IsNull(const std::string &field); - DB_API Query &And() - { - ExecuteLogicOperation(QueryObjType::AND); - return *this; - } + DB_API Query &And(); - DB_API Query &Or() - { - ExecuteLogicOperation(QueryObjType::OR); - return *this; - } + DB_API Query &Or(); DB_API Query &IsNotNull(const std::string &field); @@ -172,23 +141,17 @@ public: DB_API Query &SuggestIndex(const std::string &indexName); friend class GetQueryInfo; - ~Query() {} + ~Query() = default; private: - Query() {} + Query() = default; + Query(const std::string &tableName); DB_SYMBOL void ExecuteCompareOperation(QueryObjType operType, const std::string &field, const QueryValueType type, const FieldValue &fieldValue); DB_SYMBOL void ExecuteCompareOperation(QueryObjType operType, const std::string &field, const QueryValueType type, const std::vector &fieldValue); - DB_SYMBOL void ExecuteLogicOperation(QueryObjType operType); - DB_SYMBOL void ExecuteOrderBy(const std::string &field, bool isAsc); - DB_SYMBOL void ExecuteLimit(int number, int offset); - DB_SYMBOL void ExecuteLike(const std::string &field, const std::string &value); - DB_SYMBOL void ExecuteNotLike(const std::string &field, const std::string &value); - DB_SYMBOL void ExecuteIsNull(const std::string &field); - template QueryValueType GetFieldTypeAndValue(const T &queryValue, FieldValue &fieldValue) { diff --git a/services/distributeddataservice/libs/distributeddb/include/query_expression.h b/services/distributeddataservice/libs/distributeddb/include/query_expression.h index f1788d0c4307a7c6474432f785e025ed1c06e482..48084148c535dee01e8856da4ecf0c95ae9a1ca4 100755 --- a/services/distributeddataservice/libs/distributeddb/include/query_expression.h +++ b/services/distributeddataservice/libs/distributeddb/include/query_expression.h @@ -23,7 +23,7 @@ #include "types_export.h" namespace DistributedDB { -enum class QueryValueType { +enum class QueryValueType: int32_t { VALUE_TYPE_INVALID = -1, VALUE_TYPE_NULL, VALUE_TYPE_BOOL, @@ -33,34 +33,35 @@ enum class QueryValueType { VALUE_TYPE_STRING, }; -enum class QueryObjType { - OPER_ILLEGAL = -1, - QUERY_VALUE, - EQUALTO, +// value will const, it will influence query object id +// use high pos bit to distinguish operator type +enum class QueryObjType : uint32_t { + OPER_ILLEGAL = 0x0000, + EQUALTO = 0x0101, NOT_EQUALTO, GREATER_THAN, LESS_THAN, GREATER_THAN_OR_EQUALTO, LESS_THAN_OR_EQUALTO, - LIKE, + LIKE = 0x0201, NOT_LIKE, IS_NULL, IS_NOT_NULL, - IN, + IN = 0x0301, NOT_IN, - QUERY_BY_KEY_PREFIX, - BEGIN_GROUP, + QUERY_BY_KEY_PREFIX = 0x0401, + BEGIN_GROUP = 0x0501, END_GROUP, - AND, + AND = 0x0601, OR, - LIMIT, + LIMIT = 0x0701, ORDERBY, - SUGGEST_INDEX, + SUGGEST_INDEX = 0x0801, }; struct QueryObjNode { QueryObjType operFlag = QueryObjType::OPER_ILLEGAL; - std::string fieldName = ""; + std::string fieldName {}; QueryValueType type = QueryValueType::VALUE_TYPE_INVALID; std::vector fieldValue = {}; }; @@ -109,9 +110,13 @@ public: void QueryBySuggestIndex(const std::string &indexName); - std::vector GetPreFixKey(); + std::vector GetPreFixKey() const; - std::string GetSuggestIndex(); + void SetTableName(const std::string &tableName); + const std::string &GetTableName(); + bool IsTableNameSpacified() const; + + std::string GetSuggestIndex() const; const std::list &GetQueryExpression(); @@ -119,13 +124,15 @@ public: bool GetErrFlag(); private: - void AssemblyQueryInfo(const QueryObjType querrOperType, const std::string &field, + void AssemblyQueryInfo(const QueryObjType queryOperType, const std::string &field, const QueryValueType type, const std::vector &value, bool isNeedFieldPath); std::list queryInfo_; bool errFlag_ = true; std::vector prefixKey_; std::string suggestIndex_; + std::string tableName_; + bool isTableNameSpecified_; }; // specialize for double diff --git a/services/distributeddataservice/libs/distributeddb/include/types_export.h b/services/distributeddataservice/libs/distributeddb/include/types_export.h index b8063b1f33958016dc57a4e92b0d14b5b87b9117..6e474e73408006ca43080e5547e87961be9f6e98 100755 --- a/services/distributeddataservice/libs/distributeddb/include/types_export.h +++ b/services/distributeddataservice/libs/distributeddb/include/types_export.h @@ -109,6 +109,8 @@ struct FieldValue { enum PermissionCheckFlag { CHECK_FLAG_SEND = 1, // send CHECK_FLAG_RECEIVE = 2, // receive + CHECK_FLAG_AUTOSYNC = 4, // autosync flag + CHECK_FLAG_SPONSOR = 8, // sync sponsor }; using PermissionCheckCallback = std::function; using RemotePushFinisheNotifier = RemotePushFinishedNotifier; // To correct spelling errors in the previous version + +enum class CompressAlgorithm : uint8_t { + NONE = 0, + ZLIB = 1 +}; +} // namespace DistributedDB #endif // DISTRIBUTEDDB_TYPES_EXPORT_H diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h new file mode 100644 index 0000000000000000000000000000000000000000..bd50513ae27efc01e8275eeb3cc8747b1277f078 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/data_value.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DISTRIBUTED_DB_DATA_VALUE_H +#define DISTRIBUTED_DB_DATA_VALUE_H + +#include +#include +#include +#include +namespace DistributedDB { +// field types stored in sqlite +enum class StorageType { + STORAGE_TYPE_NONE = 0, + STORAGE_TYPE_NULL, + STORAGE_TYPE_BOOL, + STORAGE_TYPE_INTEGER, + STORAGE_TYPE_REAL, + STORAGE_TYPE_TEXT, + STORAGE_TYPE_BLOB +}; + +class Blob { +public: + Blob(); + ~Blob(); + + DISABLE_COPY_ASSIGN_MOVE(Blob); + + const uint8_t* GetData() const; + uint32_t GetSize() const; + + int WriteBlob(const uint8_t *ptrArray, const uint32_t &size); + +private: + uint8_t* ptr_; + uint32_t size_; +}; + +class DataValue { +public: + DataValue(); + ~DataValue(); + + // copy constructor + DataValue(const DataValue &dataValue); + DataValue &operator=(const DataValue &dataValue); + // move constructor + DataValue(DataValue &&dataValue) noexcept; + DataValue &operator=(DataValue &&dataValue) noexcept; + DataValue &operator=(const bool &boolVal); + DataValue &operator=(const int64_t &intVal); + DataValue &operator=(const double &doubleVal); + DataValue &operator=(const Blob &blob); + DataValue &operator=(const std::string &string); + // equals + bool operator==(const DataValue &dataValue) const; + bool operator!=(const DataValue &dataValue) const; + + StorageType GetType() const; + int GetBool(bool &outVal) const; + int GetInt64(int64_t &outVal) const; + int GetDouble(double &outVal) const; + int GetBlob(Blob *&outVal) const; + int GetBlob(Blob &outVal) const; + int GetText(std::string &outVal) const; + void ResetValue(); + int GetBlobLength(uint32_t &length) const; + std::string ToString() const; + +private: + StorageType type_ = StorageType::STORAGE_TYPE_NULL; + union { + void* zeroMem; + bool bValue; + Blob* blobPtr; + double dValue; + int64_t iValue; + } value_{}; +}; + +class ObjectData { +public: + int GetBool(const std::string &fieldName, bool &outValue) const; + int GetInt64(const std::string &fieldName, int64_t &outValue) const; + int GetDouble(const std::string &fieldName, double &outValue) const; + int GetString(const std::string &fieldName, std::string &outValue) const; + int GetBlob(const std::string &fieldName, Blob &blob) const; + void PutDataValue(const std::string &fieldName, const DataValue &value); +private: + mutable std::map fieldData; +}; +} +#endif // DISTRIBUTED_DB_DATA_VALUE_H diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/intercepted_data.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/intercepted_data.h new file mode 100644 index 0000000000000000000000000000000000000000..85c49cef5e79e5b5fa698945fe3a74847b924f29 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/intercepted_data.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INTERCEPTED_DATA_H +#define INTERCEPTED_DATA_H + +#include +#include "types.h" +#include "types_export.h" + +namespace DistributedDB { +struct KVEntry { + const Key &key; + const Value &value; +}; + +class InterceptedData { +public: + InterceptedData() {} + DB_API virtual ~InterceptedData() {} + + // Interface for getting the intercepted entries. + DB_API virtual std::vector GetEntries() = 0; + + // Interface for modifying key. Index is the index of GetEntries(). + DB_API virtual DBStatus ModifyKey(size_t index, const Key &newKey) = 0; + + // Interface for modifying value. Index is the index of GetEntries(). + DB_API virtual DBStatus ModifyValue(size_t index, const Value &newValue) = 0; +}; + +// The callback function works on the send data from device "sourceID" to device "targetID". +using PushDataInterceptor = std::function; +} // namespace DistributedDB +#endif // INTERCEPTED_DATA_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/iprocess_communicator.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/iprocess_communicator.h index cdeac53a5dd87ff498961e2cd787c7231e9d333b..94228dbab33c191d5d22e875cf568d57c21ef6d9 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/iprocess_communicator.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/iprocess_communicator.h @@ -97,6 +97,22 @@ public: } return GetMtuSize(); } + + // The valid timeout range from 5s to 60s, value beyond this range will be set to the upper or lower limit. + virtual uint32_t GetTimeout() + { + return 5 * 1000; // 5 * 1000ms + }; + + // The valid timeout range from 5s to 60s, value beyond this range will be set to the upper or lower limit. + virtual uint32_t GetTimeout(const DeviceInfos &devInfo) + { + if (devInfo.identifier.empty()) { + // Error case(would never happen actually) to avoid "unused-parameter" warning. + return 5 * 1000; // 5 * 1000ms + } + return GetTimeout(); + } }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_delegate_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_delegate_manager.h index 638a49660436bea5df85254889a746dfd87cf47f..e1613e077368482d9992193c09d402c8ed6b8a60 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_delegate_manager.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_delegate_manager.h @@ -41,7 +41,7 @@ public: KvStoreDelegateManager &operator=(const KvStoreDelegateManager &) = delete; KvStoreDelegateManager &operator=(KvStoreDelegateManager &&) = delete; - // Used to set global config of the KvStores, such dataDir, return OK if set config susccess. + // Used to set global config of the KvStores, such dataDir, return OK if set config success. DB_API DBStatus SetKvStoreConfig(const KvStoreConfig &kvStoreConfig); #ifndef OMIT_MULTI_VER @@ -58,19 +58,19 @@ public: const std::function &callback); #ifndef OMIT_MULTI_VER - // Close a KvStore, return OK if close susccess. + // Close a KvStore, return OK if close success. DB_API DBStatus CloseKvStore(KvStoreDelegate *kvStore); #endif DB_API DBStatus CloseKvStore(KvStoreNbDelegate *kvStore); - // Used to delete a KvStore, return OK if delete susccess. + // Used to delete a KvStore, return OK if delete success. DB_API DBStatus DeleteKvStore(const std::string &storeId); // Get the database size. DB_API DBStatus GetKvStoreDiskSize(const std::string &storeId, uint64_t &size); - // Used to set the process userid and appid + // Used to set the process userid and appId DB_API static DBStatus SetProcessLabel(const std::string &appId, const std::string &userId); // Set process communicator. diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h index 02a0a900562f85541b5db766b88b080e7bcecebb..cae441e22b54b321ff9d59c73f122d855adbf66b 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/kv_store_nb_delegate.h @@ -26,25 +26,9 @@ #include "kv_store_result_set.h" #include "query.h" #include "iprocess_system_api_adapter.h" +#include "intercepted_data.h" namespace DistributedDB { -enum ObserverMode { - OBSERVER_CHANGES_NATIVE = 1, - OBSERVER_CHANGES_FOREIGN = 2, - OBSERVER_CHANGES_LOCAL_ONLY = 4, -}; - -enum SyncMode { - SYNC_MODE_PUSH_ONLY, - SYNC_MODE_PULL_ONLY, - SYNC_MODE_PUSH_PULL, -}; - -enum ConflictResolvePolicy { - LAST_WIN = 0, - DEVICE_COLLABORATION, -}; - using KvStoreNbPublishOnConflict = std::function; using KvStoreNbConflictNotifier = std::function; @@ -60,11 +44,15 @@ public: bool createDirByStoreIdOnly = false; SecurityOption secOption; // Add data security level parameter KvStoreObserver *observer = nullptr; - Key key; // The key that needs to be subscribed on obsever, emptye means full subscription + Key key; // The key that needs to be subscribed on obsever, empty means full subscription unsigned int mode = 0; // obsever mode int conflictType = 0; KvStoreNbConflictNotifier notifier = nullptr; int conflictResolvePolicy = LAST_WIN; + bool isNeedIntegrityCheck = false; + bool isNeedRmCorruptedDb = false; + bool isNeedCompressOnSync = false; + uint8_t compressionRate = 100; // Valid in [1, 100]. }; DB_API virtual ~KvStoreNbDelegate() {} @@ -195,6 +183,37 @@ public: // If Repeat set, subject to the last time. // If set nullptr, means unregister the notify. DB_API virtual DBStatus SetRemotePushFinishedNotify(const RemotePushFinishedNotifier ¬ifier) = 0; + + // Sync function interface, if wait set true, this function will be blocked until sync finished. + // Param query used to filter the records to be synchronized. + // Now just support push mode and query by prefixKey. + // If Query.limit is used, its query cache will not be recorded, In the same way below, + // the synchronization will still take the full amount. + DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) = 0; + + // Check the integrity of this kvStore. + DB_API virtual DBStatus CheckIntegrity() const = 0; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + DB_API virtual DBStatus SetEqualIdentifier(const std::string &identifier, + const std::vector &targets) = 0; + + // This API is not recommended. Before using this API, you need to understand the API usage rules. + // Set pushdatainterceptor. The interceptor works when send data. + DB_API virtual DBStatus SetPushDataInterceptor(const PushDataInterceptor &interceptor) = 0; + + // Register a subscriber query on peer devices. The data in the peer device meets the subscriber query condition + // will automatically push to the local device when it's changed. + DB_API virtual DBStatus SubscribeRemoteQuery(const std::vector &devices, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) = 0; + + // Unregister a subscriber query on peer devices. + DB_API virtual DBStatus UnSubscribeRemoteQuery(const std::vector &devices, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h new file mode 100644 index 0000000000000000000000000000000000000000..3e2923b77680b6f0b0bbd3b4eb638fb4b33078f9 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_delegate.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RELATIONAL_STORE_DELEGATE_H +#define RELATIONAL_STORE_DELEGATE_H + +#include + +#include "query.h" +#include "types.h" + +namespace DistributedDB { +class RelationalStoreDelegate { +public: + DB_API virtual ~RelationalStoreDelegate() = default; + + struct Option { + bool createIfNecessary = true; + }; + + DB_API virtual DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) = 0; + + DB_API virtual DBStatus RemoveDeviceData(const std::string &device) = 0; + + struct TableOption { + }; + + DB_API virtual DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) = 0; + + DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, + SyncStatusCallback &onComplete, bool wait) = 0; + + DB_API virtual DBStatus Sync(const std::vector &devices, SyncMode mode, + SyncStatusCallback &onComplete, const Query &query, bool wait) = 0; + + DB_API virtual DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) = 0; +}; +} // namespace DistributedDB +#endif // RELATIONAL_STORE_DELEGATE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h new file mode 100644 index 0000000000000000000000000000000000000000..17f669b0033f50eb6e232edfffdfd8709601a8c0 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_manager.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_STORE_MANAGER_H +#define RELATIONAL_STORE_MANAGER_H +#ifdef RELATIONAL_STORE +#include +#include +#include + +#include "auto_launch_export.h" +#include "relational_store_delegate.h" +#include "irelational_store.h" +#include "kvdb_properties.h" +#include "types.h" + +namespace DistributedDB { +class RelationalStoreManager final { +public: + DB_API RelationalStoreManager(const std::string &appId, const std::string &userId); + DB_API ~RelationalStoreManager() = default; + + RelationalStoreManager(const RelationalStoreManager &) = delete; + RelationalStoreManager(RelationalStoreManager &&) = delete; + RelationalStoreManager &operator=(const RelationalStoreManager &) = delete; + RelationalStoreManager &operator=(RelationalStoreManager &&) = delete; + + // PRAGMA journal_mode=WAL + // PRAGMA synchronous=FULL + // PRAGMA synchronous=NORMAL + DB_API void OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, + const std::function &callback); + + DB_API DBStatus CloseStore(RelationalStoreDelegate *store); + + DB_API DBStatus DeleteStore(const std::string &path); + +private: + std::string appId_; + std::string userId_; +}; +} // namespace DistributedDB +#endif // RELATIONAL_STORE +#endif // RELATIONAL_STORE_MANAGER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h new file mode 100644 index 0000000000000000000000000000000000000000..5f548ae818f985db3a78d170a38087729c845852 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/relational_store_sqlite_ext.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_STORE_EXT_H +#define RELATIONAL_STORE_EXT_H +#ifdef RELATIONAL_STORE + +#define SQLITE3_HW_EXPORT_SYMBOLS + +#include "sqlite3.h" +#include "types.h" + +// We extend the original purpose of the "sqlite3ext.h". +struct sqlite3_api_routines_relational { + int (*close)(sqlite3*); + int (*close_v2)(sqlite3*); + int (*open)(const char *, sqlite3 **); + int (*open16)(const void *, sqlite3 **); + int (*open_v2)(const char *, sqlite3 **, int, const char *); +}; + +extern const struct sqlite3_api_routines_relational *sqlite3_export_relational_symbols; +#define sqlite3_close sqlite3_export_relational_symbols->close +#define sqlite3_close_v2 sqlite3_export_relational_symbols->close_v2 +#define sqlite3_open sqlite3_export_relational_symbols->open +#define sqlite3_open16 sqlite3_export_relational_symbols->open16 +#define sqlite3_open_v2 sqlite3_export_relational_symbols->open_v2 + +#endif +#endif // RELATIONAL_STORE_EXT_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/runtime_config.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/runtime_config.h new file mode 100644 index 0000000000000000000000000000000000000000..b21c7dba80724fde3813fcfc037ae6e3c5072f19 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/relational/runtime_config.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_CONFIG_H +#define RUNTIME_CONFIG_H + +#include +#include + +#include "iprocess_communicator.h" +#include "iprocess_system_api_adapter.h" +#include "types.h" +namespace DistributedDB { +class RuntimeConfig final { +public: + DB_API RuntimeConfig() = default; + DB_API ~RuntimeConfig() = default; + + DB_API static DBStatus SetProcessLabel(const std::string &appId, const std::string &userId); + + DB_API static DBStatus SetProcessCommunicator(const std::shared_ptr &inCommunicator); + + DB_API static DBStatus SetPermissionCheckCallback(const PermissionCheckCallbackV2 &callback); + + DB_API static DBStatus SetProcessSystemAPIAdapter(const std::shared_ptr &adapter); + +private: + static std::mutex communicatorMutex_; + static std::shared_ptr processCommunicator_; +}; +} // namespace DistributedDB + +#endif // RUNTIME_CONFIG_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h index 3a1d93fb6f3d007c76314ab9349cc5323bce31fb..fb12ab170a7a1f206c361a3c0d3c0119485dbc13 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/include/types.h @@ -18,7 +18,7 @@ #include #include -#include +#include #include "types_export.h" @@ -43,10 +43,10 @@ enum DBStatus { INVALID_FIELD_TYPE, // invalid put value field type for json schema. CONSTRAIN_VIOLATION, // invalid put value constrain for json schema. INVALID_FORMAT, // invalid put value format for json schema. - STALE, // new record is staler compared to the same key exist in db + STALE, // new record is staler compared to the same key existed in db. LOCAL_DELETED, // local data is deleted by the unpublish. LOCAL_DEFEAT, // local data defeat the sync data while unpublish. - LOCAL_COVERED, // local data is coverd by the sync data while unpublish. + LOCAL_COVERED, // local data is covered by the sync data while unpublish. INVALID_QUERY_FORMAT, INVALID_QUERY_FIELD, PERMISSION_CHECK_FORBID_SYNC, // permission check result , forbid sync. @@ -55,6 +55,7 @@ enum DBStatus { EKEYREVOKED_ERROR, // EKEYREVOKED error when operating db file SECURITY_OPTION_CHECK_ERROR, // such as remote device's SecurityOption not equal to local SCHEMA_VIOLATE_VALUE, // Values already exist in dbFile do not match new schema + INTERCEPT_DATA_FAIL, // Interceptor push data failed. }; struct KvStoreConfig { @@ -77,6 +78,7 @@ enum PragmaCmd { SET_WIPE_POLICY, // set the policy of wipe remote stale data RESULT_SET_CACHE_MODE, // Accept ResultSetCacheMode Type As PragmaData RESULT_SET_CACHE_MAX_SIZE, // Allowed Int Type Range [1,16], Unit MB + SET_SYNC_RETRY, }; enum ResolutionPolicyType { @@ -84,8 +86,31 @@ enum ResolutionPolicyType { CUSTOMER_RESOLUTION = 1 // resolve conflicts by user }; +enum ObserverMode { + OBSERVER_CHANGES_NATIVE = 1, + OBSERVER_CHANGES_FOREIGN = 2, + OBSERVER_CHANGES_LOCAL_ONLY = 4, +}; + +enum SyncMode { + SYNC_MODE_PUSH_ONLY, + SYNC_MODE_PULL_ONLY, + SYNC_MODE_PUSH_PULL, +}; + +enum ConflictResolvePolicy { + LAST_WIN = 0, + DEVICE_COLLABORATION, +}; + +struct TableStatus { + std::string tableName; + DBStatus status; +}; using KvStoreCorruptionHandler = std::function; + const std::string &storeId)>; +using StoreCorruptionHandler = std::function; +using SyncStatusCallback = std::function> &devicesMap)>; } // namespace DistributedDB - #endif // KV_STORE_TYPE_H diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/intercepted_data_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/intercepted_data_impl.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3bfb06d1571e25693a8ef4ef25beae0f3f4063b3 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/intercepted_data_impl.cpp @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "intercepted_data_impl.h" +#include "db_common.h" +#include "db_constant.h" +#include "generic_single_ver_kv_entry.h" +#include "parcel.h" +#include "version.h" + +namespace DistributedDB { +namespace { +bool CheckKey(const Key &key) +{ + if (key.empty() || key.size() > DBConstant::MAX_KEY_SIZE) { + LOGE("Key is too large:%zu.", key.size()); + return false; + } + return true; +} + +bool CheckValue(const Value &value, const std::function &checkSchema) +{ + if (value.size() > DBConstant::MAX_VALUE_SIZE) { + LOGE("Value is too large:%zu.", value.size()); + return false; + } + + if (checkSchema == nullptr) { + LOGE("Check schema failed, no check func."); + return false; + } + + int errCode = checkSchema(value); + if (errCode != E_OK) { + LOGE("Check schema failed, value is invalid:%d.", errCode); + return false; + } + return true; +} + +bool CheckLength(size_t len, size_t maxPacketSize) +{ + if (len > maxPacketSize) { + LOGE("Packet is too large:%zu.", len); + return false; + } + return true; +} +} // anonymous namespace + +InterceptedDataImpl::InterceptedDataImpl(std::vector dataItems, + const std::function &checkSchema) + : kvEntriesReady_(false), + isError_(false), + totalLength_(), + maxPacketSize_(), + checkSchema_(checkSchema), + dataItems_(dataItems), + kvEntries_(), + indexes_() +{ + totalLength_ = GenericSingleVerKvEntry::CalculateLens(dataItems, SOFTWARE_VERSION_CURRENT); + // New packet cannot exceed both twice the MTU and twice the original size. + // Besides, it cannot exceed 30 MB. + maxPacketSize_ = std::min(DBConstant::MAX_SYNC_BLOCK_SIZE, + std::max(totalLength_, static_cast(DBConstant::MAX_MTU_SIZE)) * 2); +} + +InterceptedDataImpl::~InterceptedDataImpl() +{} + +std::vector InterceptedDataImpl::GetEntries() +{ + if (!kvEntriesReady_) { + GetKvEntries(); + } + return kvEntries_; +} + +bool InterceptedDataImpl::CheckIndex(size_t index) +{ + if (!kvEntriesReady_) { + GetKvEntries(); + } + + if (index >= kvEntries_.size()) { + LOGE("Index is too large:%zu, size:%zu.", index, kvEntries_.size()); + return false; + } + return true; +} + +DBStatus InterceptedDataImpl::ModifyKey(size_t index, const Key &newKey) +{ + // Check index. + if (!CheckIndex(index)) { + isError_ = true; + return INVALID_ARGS; + } + + // Check key. + if (!CheckKey(newKey)) { + isError_ = true; + return INVALID_ARGS; + } + + // Check length. + const auto &oldKey = dataItems_[indexes_[index]]->GetKey(); + size_t newLength = totalLength_ - Parcel::GetVectorCharLen(oldKey) + Parcel::GetVectorCharLen(newKey); + if (!CheckLength(newLength, maxPacketSize_)) { + isError_ = true; + return INVALID_ARGS; + } + totalLength_ = newLength; + + // Modify data + auto entry = dataItems_[indexes_[index]]; + entry->SetKey(newKey); + Key hashKey; + int errCode = DBCommon::CalcValueHash(newKey, hashKey); + if (errCode != E_OK) { + LOGE("Calc hashkey failed."); + isError_ = true; + return INVALID_ARGS; + } + entry->SetHashKey(hashKey); + return OK; +} + +DBStatus InterceptedDataImpl::ModifyValue(size_t index, const Value &newValue) +{ + // Check index. + if (!CheckIndex(index)) { + isError_ = true; + return INVALID_ARGS; + } + + // Check value. + if (!CheckValue(newValue, checkSchema_)) { + isError_ = true; + return INVALID_ARGS; + } + + // Check length. + const auto &oldValue = dataItems_[indexes_[index]]->GetValue(); + size_t newLength = totalLength_ - Parcel::GetVectorCharLen(oldValue) + Parcel::GetVectorCharLen(newValue); + if (!CheckLength(newLength, maxPacketSize_)) { + isError_ = true; + return INVALID_ARGS; + } + totalLength_ = newLength; + + // Modify data + auto entry = dataItems_[indexes_[index]]; + entry->SetValue(newValue); + return OK; +} + +bool InterceptedDataImpl::IsError() const +{ + return isError_; +} + +void InterceptedDataImpl::GetKvEntries() +{ + for (size_t i = 0; i < dataItems_.size(); ++i) { + const auto &kvEntry = dataItems_[i]; + if ((kvEntry->GetFlag() & DataItem::DELETE_FLAG) == 0) { // For deleted data, do not modify. + kvEntries_.push_back({ kvEntry->GetKey(), kvEntry->GetValue() }); + indexes_.push_back(i); + } + } + kvEntriesReady_ = true; +} +} // namespace DistributedDB + diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/intercepted_data_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/intercepted_data_impl.h new file mode 100644 index 0000000000000000000000000000000000000000..6e548d72e2ceac993588c9a8d446ff739b0b90a3 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/intercepted_data_impl.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INTERCEPTED_DATA_IMPL_H +#define INTERCEPTED_DATA_IMPL_H + +#include +#include +#include + +#include "intercepted_data.h" +#include "macro_utils.h" +#include "single_ver_kv_entry.h" +#include "types.h" +#include "types_export.h" + +namespace DistributedDB { +class InterceptedDataImpl : public InterceptedData { +public: + InterceptedDataImpl(std::vector dataItems, const std::function &checkSchema); + virtual ~InterceptedDataImpl(); + DISABLE_COPY_ASSIGN_MOVE(InterceptedDataImpl); + + std::vector GetEntries() override; + DBStatus ModifyKey(size_t index, const Key &newKey) override; + DBStatus ModifyValue(size_t index, const Value &newValue) override; + + bool IsError() const; + +private: + bool CheckIndex(size_t index); + void GetKvEntries(); + + bool kvEntriesReady_; + bool isError_; + size_t totalLength_; + size_t maxPacketSize_; + std::function checkSchema_; + std::vector dataItems_; + std::vector kvEntries_; + std::vector indexes_; +}; +} // namespace DistributedDB +#endif // INTERCEPTED_DATA_IMPL_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_impl.cpp index 987f92b401ef2143e3109d121c289b649d0819ac..d1a0be4449550aee11b767ec1df405e40c18e60a 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_impl.cpp @@ -434,7 +434,6 @@ DBStatus KvStoreDelegateImpl::Pragma(PragmaCmd cmd, PragmaData ¶mData) LOGE("%s", INVALID_CONNECTION.c_str()); return DB_ERROR; } - int errCode; switch (cmd) { case PERFORMANCE_ANALYSIS_GET_REPORT: diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp index d3493a05dc47b138ab9281a03935adb88d3b155d..aab890c2d8fcdb5949089a2b5910e1ee82a3d8f6 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_delegate_manager.cpp @@ -89,6 +89,8 @@ namespace { properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); properties.SetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, option.createDirByStoreIdOnly); properties.SetSchema(schema); + properties.SetBoolProp(KvDBProperties::CHECK_INTEGRITY, option.isNeedIntegrityCheck); + properties.SetBoolProp(KvDBProperties::RM_CORRUPTED_DB, option.isNeedRmCorruptedDb); if (RuntimeContext::GetInstance()->IsProcessSystemApiAdapterValid()) { properties.SetIntProp(KvDBProperties::SECURITY_LABEL, option.secOption.securityLabel); properties.SetIntProp(KvDBProperties::SECURITY_FLAG, option.secOption.securityFlag); @@ -98,6 +100,11 @@ namespace { if (option.isEncryptedDb) { properties.SetPassword(option.cipher, option.passwd); } + properties.SetBoolProp(KvDBProperties::COMPRESS_ON_SYNC, option.isNeedCompressOnSync); + if (option.isNeedCompressOnSync) { + properties.SetIntProp(KvDBProperties::COMPRESSION_RATE, + ParamCheckUtils::GetValidCompressionRate(option.compressionRate)); + } } bool CheckObserverConflictParam(const KvStoreNbDelegate::Option &option) @@ -524,8 +531,7 @@ DBStatus KvStoreDelegateManager::EnableKvStoreAutoLaunch(const std::string &user return TransferDBErrno(errCode); } - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(properties, notifier, option.observer, - option.conflictType, option.notifier); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(properties, notifier, option); if (errCode != E_OK) { LOGE("[KvStoreManager] Enable auto launch failed:%d", errCode); return TransferDBErrno(errCode); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp index 6c7df9c341f46ff1f8039d52e8d7c39d92125e2e..b827ad0aecc9b18eb6578acf1de1a50459412ef9 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_errno.cpp @@ -50,7 +50,8 @@ namespace { { -E_INVALID_QUERY_FIELD, INVALID_QUERY_FIELD }, { -E_ALREADY_SET, ALREADY_SET }, { -E_EKEYREVOKED, EKEYREVOKED_ERROR }, - { -E_SECURITY_OPTION_CHECK_ERROR, SECURITY_OPTION_CHECK_ERROR}, + { -E_SECURITY_OPTION_CHECK_ERROR, SECURITY_OPTION_CHECK_ERROR }, + { -E_INTERCEPT_DATA_FAIL, INTERCEPT_DATA_FAIL }, }; } diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp index be459311f379610eb498aa53b13fb318970f262b..f1f57a3863545bb64ce95ef8ae51d51b94d74c28 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.cpp @@ -56,6 +56,7 @@ namespace { {SET_WIPE_POLICY, PRAGMA_SET_WIPE_POLICY}, {RESULT_SET_CACHE_MODE, PRAGMA_RESULT_SET_CACHE_MODE}, {RESULT_SET_CACHE_MAX_SIZE, PRAGMA_RESULT_SET_CACHE_MAX_SIZE}, + {SET_SYNC_RETRY, PRAGMA_SET_SYNC_RETRY}, }; const std::string INVALID_CONNECTION = "[KvStoreNbDelegate] Invalid connection for operation"; @@ -476,17 +477,29 @@ DBStatus KvStoreNbDelegateImpl::Sync(const std::vector &devices, Sy this, std::placeholders::_1, onComplete), wait); int errCode = conn_->Pragma(PRAGMA_SYNC_DEVICES, &pragmaData); if (errCode < E_OK) { - if (errCode == -E_BUSY) { - return BUSY; - } - - if (errCode == -E_INVALID_ARGS) { - return INVALID_ARGS; - } - LOGE("[KvStoreNbDelegate] Sync data failed:%d", errCode); + return TransferDBErrno(errCode); + } + return OK; +} + +DBStatus KvStoreNbDelegateImpl::Sync(const std::vector &devices, SyncMode mode, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION.c_str()); return DB_ERROR; } + + QuerySyncObject querySyncObj(query); + PragmaSync pragmaData(devices, mode, querySyncObj, std::bind(&KvStoreNbDelegateImpl::OnSyncComplete, + this, std::placeholders::_1, onComplete), wait); + int errCode = conn_->Pragma(PRAGMA_SYNC_DEVICES, &pragmaData); + if (errCode < E_OK) { + LOGE("[KvStoreNbDelegate] QuerySync data failed:%d", errCode); + return TransferDBErrno(errCode); + } return OK; } @@ -705,6 +718,16 @@ DBStatus KvStoreNbDelegateImpl::Close() return OK; } +DBStatus KvStoreNbDelegateImpl::CheckIntegrity() const +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION.c_str()); + return DB_ERROR; + } + + return TransferDBErrno(conn_->CheckIntegrity()); +} + DBStatus KvStoreNbDelegateImpl::GetSecurityOption(SecurityOption &option) const { if (conn_ == nullptr) { @@ -807,14 +830,20 @@ void KvStoreNbDelegateImpl::OnSyncComplete(const std::map &sta for (const auto &pair : statuses) { DBStatus status = DB_ERROR; static const std::map statusMap = { - { static_cast(SyncOperation::FINISHED_ALL), DBStatus::OK }, - { static_cast(SyncOperation::TIMEOUT), DBStatus::TIME_OUT }, - { static_cast(SyncOperation::PERMISSION_CHECK_FAILED), DBStatus::PERMISSION_CHECK_FORBID_SYNC }, - { static_cast(SyncOperation::COMM_ABNORMAL), DBStatus::COMM_FAILURE }, - { static_cast(SyncOperation::SECURITY_OPTION_CHECK_FAILURE), DBStatus::SECURITY_OPTION_CHECK_ERROR }, - { static_cast(SyncOperation::EKEYREVOKED_FAILURE), DBStatus::EKEYREVOKED_ERROR }, - { static_cast(SyncOperation::SCHEMA_INCOMPATIBLE), DBStatus::SCHEMA_MISMATCH }, - { static_cast(SyncOperation::BUSY_FAILURE), DBStatus::BUSY }, + { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, + { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, + { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, + { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, + { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, + { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, + { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, + { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, + { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, + { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, + { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, + { static_cast(SyncOperation::OP_INTERCEPT_DATA_FAIL), INTERCEPT_DATA_FAIL }, + { static_cast(SyncOperation::OP_MAX_LIMITS), OVER_MAX_LIMITS }, + { static_cast(SyncOperation::OP_INVALID_ARGS), INVALID_ARGS }, }; auto iter = statusMap.find(pair.second); if (iter != statusMap.end()) { @@ -826,4 +855,76 @@ void KvStoreNbDelegateImpl::OnSyncComplete(const std::map &sta onComplete(result); } } + +DBStatus KvStoreNbDelegateImpl::SetEqualIdentifier(const std::string &identifier, + const std::vector &targets) +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION.c_str()); + return DB_ERROR; + } + + PragmaSetEqualIdentifier pragma(identifier, targets); + int errCode = conn_->Pragma(PRAGMA_ADD_EQUAL_IDENTIFIER, reinterpret_cast(&pragma)); + if (errCode != E_OK) { + LOGE("[KvStoreNbDelegate] Set store equal identifier failed : %d", errCode); + } + + return TransferDBErrno(errCode); +} + +DBStatus KvStoreNbDelegateImpl::SetPushDataInterceptor(const PushDataInterceptor &interceptor) +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION.c_str()); + return DB_ERROR; + } + + PushDataInterceptor notify = interceptor; + int errCode = conn_->Pragma(PRAGMA_INTERCEPT_SYNC_DATA, static_cast(¬ify)); + if (errCode != E_OK) { + LOGE("[KvStoreNbDelegate] Set data interceptor notify failed : %d", errCode); + } + return TransferDBErrno(errCode); +} + +DBStatus KvStoreNbDelegateImpl::SubscribeRemoteQuery(const std::vector &devices, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION.c_str()); + return DB_ERROR; + } + + QuerySyncObject querySyncObj(query); + PragmaSync pragmaData(devices, SyncModeType::SUBSCRIBE_QUERY, querySyncObj, + std::bind(&KvStoreNbDelegateImpl::OnSyncComplete, this, std::placeholders::_1, onComplete), wait); + int errCode = conn_->Pragma(PRAGMA_SUBSCRIBE_QUERY, &pragmaData); + if (errCode < E_OK) { + LOGE("[KvStoreNbDelegate] Subscribe remote data with query failed:%d", errCode); + return TransferDBErrno(errCode); + } + return OK; +} + +DBStatus KvStoreNbDelegateImpl::UnSubscribeRemoteQuery(const std::vector &devices, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) +{ + if (conn_ == nullptr) { + LOGE("%s", INVALID_CONNECTION.c_str()); + return DB_ERROR; + } + + QuerySyncObject querySyncObj(query); + PragmaSync pragmaData(devices, SyncModeType::UNSUBSCRIBE_QUERY, querySyncObj, + std::bind(&KvStoreNbDelegateImpl::OnSyncComplete, this, std::placeholders::_1, onComplete), wait); + int errCode = conn_->Pragma(PRAGMA_SUBSCRIBE_QUERY, &pragmaData); + if (errCode < E_OK) { + LOGE("[KvStoreNbDelegate] Unsubscribe remote data with query failed:%d", errCode); + return TransferDBErrno(errCode); + } + return OK; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h index 241ca620f9093501074ec67b6ac1173c0dd5feb9..35606772fd9120ef6ece995eba3b5a3ae3f1a4cd 100755 --- a/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/kv_store_nb_delegate_impl.h @@ -83,15 +83,15 @@ public: std::string GetStoreId() const override; // Sync function interface, if wait set true, this function will be blocked until sync finished - DB_API DBStatus Sync(const std::vector &devices, SyncMode mode, + DBStatus Sync(const std::vector &devices, SyncMode mode, const std::function &devicesMap)> &onComplete, bool wait) override; // Special pragma interface, see PragmaCmd and PragmaData, - DB_API DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) override; + DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) override; // Set the conflict notifier for getting the specified type conflict data. - DB_API DBStatus SetConflictNotifier(int conflictType, const KvStoreNbConflictNotifier ¬ifier) override; + DBStatus SetConflictNotifier(int conflictType, const KvStoreNbConflictNotifier ¬ifier) override; // Rekey the database. DBStatus Rekey(const CipherPassword &password) override; @@ -125,6 +125,31 @@ public: DBStatus Close(); + // Sync function interface, if wait set true, this function will be blocked until sync finished. + // Param query used to filter the records to be synchronized. + // Now just support push mode and query by prefixKey. + DBStatus Sync(const std::vector &devices, SyncMode mode, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) override; + + DBStatus CheckIntegrity() const override; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + DBStatus SetEqualIdentifier(const std::string &identifier, const std::vector &targets) override; + + DBStatus SetPushDataInterceptor(const PushDataInterceptor &interceptor) override; + + // Register a subscriber query on peer devices. The data in the peer device meets the subscriber query condition + // will automatically push to the local device when it's changed. + DBStatus SubscribeRemoteQuery(const std::vector &devices, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) override; + + // Unregister a subscriber query on peer devices. + DBStatus UnSubscribeRemoteQuery(const std::vector &devices, + const std::function &devicesMap)> &onComplete, + const Query &query, bool wait) override; + private: DBStatus GetInner(const IOption &option, const Key &key, Value &value) const; DBStatus PutInner(const IOption &option, const Key &key, const Value &value); diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ad5072181e94e7df2f36a12ac6d779775fc5fc78 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/data_value.cpp @@ -0,0 +1,392 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "data_value.h" + +#include "db_errno.h" +#include "relational_schema_object.h" +#include "securec.h" + +namespace DistributedDB { +Blob::Blob() : ptr_(nullptr), size_(0) +{ +} + +Blob::~Blob() +{ + if (ptr_ != nullptr) { + delete[] ptr_; + ptr_ = nullptr; + } + size_ = 0; +} + +const uint8_t *Blob::GetData() const +{ + return ptr_; +} + +uint32_t Blob::GetSize() const +{ + return size_; +} + +int Blob::WriteBlob(const uint8_t *ptrArray, const uint32_t &size) +{ + if (size == 0) return E_OK; + ptr_ = new(std::nothrow) uint8_t[size]; + if (ptr_ == nullptr) { + return -E_OUT_OF_MEMORY; + } + size_ = size; + errno_t errCode = memcpy_s(ptr_, size, ptrArray, size); + if (errCode != EOK) { + return -E_SECUREC_ERROR; + } + return E_OK; +} + +DataValue::DataValue() : type_(StorageType::STORAGE_TYPE_NULL) +{ + value_.zeroMem = nullptr; +} + +DataValue::~DataValue() +{ + ResetValue(); +} + +DataValue::DataValue(const DataValue &dataValue) +{ + *this = dataValue; +} + +DataValue::DataValue(DataValue &&dataValue) noexcept + :type_(dataValue.type_), value_(dataValue.value_) +{ + switch (type_) { + case StorageType::STORAGE_TYPE_BLOB: + case StorageType::STORAGE_TYPE_TEXT: + dataValue.value_.blobPtr = nullptr; + break; + default: + break; + } + dataValue.ResetValue(); +} + +DataValue &DataValue::operator=(const DataValue &dataValue) +{ + if (this == &dataValue) { + return *this; + } + ResetValue(); + switch (dataValue.type_) { + case StorageType::STORAGE_TYPE_BOOL: + dataValue.GetBool(this->value_.bValue); + break; + case StorageType::STORAGE_TYPE_INTEGER: + dataValue.GetInt64(this->value_.iValue); + break; + case StorageType::STORAGE_TYPE_REAL: + dataValue.GetDouble(this->value_.dValue); + break; + case StorageType::STORAGE_TYPE_BLOB: + case StorageType::STORAGE_TYPE_TEXT: + dataValue.GetBlob(this->value_.blobPtr); + break; + default: + break; + } + type_ = dataValue.type_; + return *this; +} + +DataValue &DataValue::operator=(DataValue &&dataValue) noexcept +{ + this->type_ = dataValue.type_; + this->value_ = dataValue.value_; + switch (type_) { + case StorageType::STORAGE_TYPE_BLOB: + case StorageType::STORAGE_TYPE_TEXT: + dataValue.value_.blobPtr = nullptr; + break; + default: + break; + } + return *this; +} + +DataValue &DataValue::operator=(const bool &boolVal) +{ + ResetValue(); + type_ = StorageType::STORAGE_TYPE_BOOL; + value_.bValue = boolVal; + return *this; +} + +DataValue &DataValue::operator=(const int64_t &intVal) +{ + ResetValue(); + type_ = StorageType::STORAGE_TYPE_INTEGER; + value_.iValue = intVal; + return *this; +} + +DataValue &DataValue::operator=(const double &doubleVal) +{ + ResetValue(); + type_ = StorageType::STORAGE_TYPE_REAL; + value_.dValue = doubleVal; + return *this; +} + +DataValue &DataValue::operator=(const Blob &blob) +{ + ResetValue(); + if (blob.GetSize() <= 0) { + return *this; + } + value_.blobPtr = new(std::nothrow) Blob(); + if (value_.blobPtr == nullptr) { + return *this; + } + type_ = StorageType::STORAGE_TYPE_BLOB; + if (blob.GetSize() > 0) { + value_.blobPtr->WriteBlob(blob.GetData(), blob.GetSize()); + } + return *this; +} + +DataValue &DataValue::operator=(const std::string &string) +{ + ResetValue(); + value_.blobPtr = new(std::nothrow) Blob(); + if (value_.blobPtr == nullptr) { + return *this; + } + type_ = StorageType::STORAGE_TYPE_TEXT; + value_.blobPtr->WriteBlob(reinterpret_cast(string.c_str()), string.size()); + return *this; +} + +bool DataValue::operator==(const DataValue &dataValue) const +{ + if (dataValue.type_ != type_) { + return false; + } + switch (type_) { + case StorageType::STORAGE_TYPE_BOOL: + return dataValue.value_.bValue == value_.bValue; + case StorageType::STORAGE_TYPE_INTEGER: + return dataValue.value_.iValue == value_.iValue; + case StorageType::STORAGE_TYPE_REAL: + return dataValue.value_.dValue == value_.dValue; + case StorageType::STORAGE_TYPE_BLOB: + case StorageType::STORAGE_TYPE_TEXT: + if (dataValue.value_.blobPtr->GetSize() != value_.blobPtr->GetSize()) { + return false; + } + for (uint32_t i = 0; i < dataValue.value_.blobPtr->GetSize(); ++i) { + if (dataValue.value_.blobPtr->GetData()[i] != value_.blobPtr->GetData()[i]) { + return false; + } + } + return true; + default: + return true; + } +} + +bool DataValue::operator!=(const DataValue &dataValue) const +{ + return !(*this==dataValue); +} + +int DataValue::GetBool(bool &outVal) const +{ + if (type_ != StorageType::STORAGE_TYPE_BOOL) { + return -E_NOT_SUPPORT; + } + outVal = value_.bValue; + return E_OK; +} + +int DataValue::GetDouble(double &outVal) const +{ + if (type_ != StorageType::STORAGE_TYPE_REAL) { + return -E_NOT_SUPPORT; + } + outVal = value_.dValue; + return E_OK; +} + +int DataValue::GetInt64(int64_t &outVal) const +{ + if (type_ != StorageType::STORAGE_TYPE_INTEGER) { + return -E_NOT_SUPPORT; + } + outVal = value_.iValue; + return E_OK; +} + +int DataValue::GetBlob(Blob *&outVal) const +{ + if (type_ != StorageType::STORAGE_TYPE_BLOB && type_ != StorageType::STORAGE_TYPE_TEXT) { + return -E_NOT_SUPPORT; + } + outVal = new(std::nothrow) Blob(); + if (outVal == nullptr) { + return -E_OUT_OF_MEMORY; + } + return outVal->WriteBlob(value_.blobPtr->GetData(), value_.blobPtr->GetSize()); +} + +int DataValue::GetBlob(Blob &outVal) const +{ + if (type_ != StorageType::STORAGE_TYPE_BLOB && type_ != StorageType::STORAGE_TYPE_TEXT) { + return -E_NOT_SUPPORT; + } + return outVal.WriteBlob(value_.blobPtr->GetData(), value_.blobPtr->GetSize()); +} + +int DataValue::GetText(std::string &outValue) const +{ + if (type_ != StorageType::STORAGE_TYPE_TEXT) { + return -E_NOT_SUPPORT; + } + const uint8_t *data = value_.blobPtr->GetData(); + uint32_t len = value_.blobPtr->GetSize(); + outValue.resize(len); + outValue.assign(data, data + len); + return E_OK; +} + +StorageType DataValue::GetType() const +{ + return type_; +} + +int DataValue::GetBlobLength(uint32_t &length) const +{ + if (type_ != StorageType::STORAGE_TYPE_BLOB && type_ != StorageType::STORAGE_TYPE_TEXT) { + return -E_NOT_SUPPORT; + } + length = value_.blobPtr->GetSize(); + return E_OK; +} + +void DataValue::ResetValue() +{ + switch (type_) { + case StorageType::STORAGE_TYPE_TEXT: + case StorageType::STORAGE_TYPE_BLOB: + delete value_.blobPtr; + value_.blobPtr = nullptr; + break; + case StorageType::STORAGE_TYPE_NULL: + case StorageType::STORAGE_TYPE_BOOL: + case StorageType::STORAGE_TYPE_INTEGER: + case StorageType::STORAGE_TYPE_REAL: + default: + break; + } + type_ = StorageType::STORAGE_TYPE_NULL; + value_.zeroMem = nullptr; +} + +std::string DataValue::ToString() const +{ + std::string res; + switch (type_) { + case StorageType::STORAGE_TYPE_TEXT: + (void)GetText(res); + break; + case StorageType::STORAGE_TYPE_BLOB: + res = "NOT SUPPORT"; + break; + case StorageType::STORAGE_TYPE_NULL: + res = "null"; + break; + case StorageType::STORAGE_TYPE_BOOL: + res = std::to_string(value_.bValue); + break; + case StorageType::STORAGE_TYPE_INTEGER: + res = std::to_string(value_.iValue); + break; + case StorageType::STORAGE_TYPE_REAL: + res = std::to_string(value_.dValue); + break; + default: + res = "default"; + break; + } + return "[" + res + "]"; +} + +int ObjectData::GetDouble(const std::string &fieldName, double &outValue) const +{ + if (fieldData.find(fieldName) == fieldData.end()) { + return -E_NOT_FOUND; + } + return fieldData[fieldName].GetDouble(outValue); +} + +int ObjectData::GetInt64(const std::string &fieldName, int64_t &outValue) const +{ + if (fieldData.find(fieldName) == fieldData.end()) { + return -E_NOT_FOUND; + } + return fieldData[fieldName].GetInt64(outValue); +} + +int ObjectData::GetBlob(const std::string &fieldName, Blob &blob) const +{ + if (fieldData.find(fieldName) == fieldData.end()) { + return -E_NOT_FOUND; + } + int errCode = fieldData[fieldName].GetBlob(blob); + return errCode; +} + +int ObjectData::GetString(const std::string &fieldName, std::string &outValue) const +{ + if (fieldData.find(fieldName) == fieldData.end()) { + return -E_NOT_FOUND; + } + Blob blob; + int errCode = fieldData[fieldName].GetBlob(blob); + if (errCode != E_OK) { + return errCode; + } + outValue.resize(blob.GetSize()); + outValue.assign(blob.GetData(), blob.GetData() + blob.GetSize()); + return errCode; +} + +int ObjectData::GetBool(const std::string &fieldName, bool &outValue) const +{ + if (fieldData.find(fieldName) == fieldData.end()) { + return -E_NOT_FOUND; + } + return fieldData[fieldName].GetBool(outValue); +} + +void ObjectData::PutDataValue(const std::string &fieldName, const DataValue &value) +{ + fieldData[fieldName] = value; +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fb4d08542149f6b884cfd04c2c6f95aa89023c56 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.cpp @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "relational_store_delegate_impl.h" + +#include "db_errno.h" +#include "log_print.h" +#include "kv_store_errno.h" +#include "sync_operation.h" + +namespace DistributedDB { +RelationalStoreDelegateImpl::RelationalStoreDelegateImpl(RelationalStoreConnection *conn, const std::string &path) + : conn_(conn), + storePath_(path) +{} + +RelationalStoreDelegateImpl::~RelationalStoreDelegateImpl() +{ + if (!releaseFlag_) { + LOGF("[KvStoreNbDelegate] Can't release directly"); + return; + } + + conn_ = nullptr; +}; + +DBStatus RelationalStoreDelegateImpl::Pragma(PragmaCmd cmd, PragmaData ¶mData) +{ + return NOT_SUPPORT; +} + +DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, + SyncStatusCallback &onComplete, bool wait) +{ + return NOT_SUPPORT; +} + +DBStatus RelationalStoreDelegateImpl::RemoveDeviceData(const std::string &device) +{ + return NOT_SUPPORT; +} + +DBStatus RelationalStoreDelegateImpl::CreateDistributedTable(const std::string &tableName, const TableOption &option) +{ + // check table Name and option + if (conn_ == nullptr) { + LOGE("[RelationalStore Delegate] Invalid connection for operation!"); + return DB_ERROR; + } + + int errCode = conn_->CreateDistributedTable(tableName, option); + if (errCode != E_OK) { + LOGW("[RelationalStore Delegate] Create Distributed table failed:%d", errCode); + return TransferDBErrno(errCode); + } + return OK; +} + +DBStatus RelationalStoreDelegateImpl::Sync(const std::vector &devices, SyncMode mode, + SyncStatusCallback &onComplete, const Query &query, bool wait) +{ + if (conn_ == nullptr) { + LOGE("Invalid connection for operation!"); + return DB_ERROR; + } + + RelationalStoreConnection::SyncInfo syncInfo{devices, mode, onComplete, query, wait}; + int errCode = conn_->SyncToDevice(syncInfo); + if (errCode != E_OK) { + LOGW("[RelationalStore Delegate] sync data to device failed:%d", errCode); + return TransferDBErrno(errCode); + } + return OK; +} + +DBStatus RelationalStoreDelegateImpl::RemoveDevicesData(const std::string &tableName, const std::string &device) +{ + return NOT_SUPPORT; +} + +DBStatus RelationalStoreDelegateImpl::Close() +{ + if (conn_ == nullptr) { + return OK; + } + + int errCode = conn_->Close(); + if (errCode == -E_BUSY) { + LOGW("[KvStoreDelegate] busy for close"); + return BUSY; + } + if (errCode != E_OK) { + LOGE("Release db connection error:%d", errCode); + return TransferDBErrno(errCode); + } + + LOGI("[KvStoreDelegate] Close"); + conn_ = nullptr; + return OK; +} + +void RelationalStoreDelegateImpl::SetReleaseFlag(bool flag) +{ + releaseFlag_ = flag; +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h new file mode 100644 index 0000000000000000000000000000000000000000..6dc5404c254ab98da64c2fe61ff2f078fcb349f6 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_delegate_impl.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_STORE_DELEGATE_IMPL_H +#define RELATIONAL_STORE_DELEGATE_IMPL_H +#ifdef RELATIONAL_STORE + +#include "macro_utils.h" +#include "relational_store_connection.h" + +namespace DistributedDB { +class RelationalStoreDelegateImpl final : public RelationalStoreDelegate { +public: + RelationalStoreDelegateImpl() = default; + ~RelationalStoreDelegateImpl() override; + + RelationalStoreDelegateImpl(RelationalStoreConnection *conn, const std::string &path); + + DISABLE_COPY_ASSIGN_MOVE(RelationalStoreDelegateImpl); + + DBStatus Pragma(PragmaCmd cmd, PragmaData ¶mData) override; + + DBStatus Sync(const std::vector &devices, SyncMode mode, + SyncStatusCallback &onComplete, bool wait) override; + DBStatus Sync(const std::vector &devices, SyncMode mode, + SyncStatusCallback &onComplete, const Query &query, bool wait) override; + + DBStatus RemoveDeviceData(const std::string &device) override; + + DBStatus CreateDistributedTable(const std::string &tableName, const TableOption &option) override; + DBStatus RemoveDevicesData(const std::string &tableName, const std::string &device) override; + + // For connection + DBStatus Close(); + void SetReleaseFlag(bool flag); + +private: + RelationalStoreConnection *conn_ = nullptr; + std::string storePath_; + std::atomic releaseFlag_ = false; +}; +} // namespace DistributedDB +#endif +#endif // RELATIONAL_STORE_DELEGATE_IMPL_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h new file mode 100644 index 0000000000000000000000000000000000000000..34316ef8260ab7d4c26fe4d75892f1d7c08742d1 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_instance.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_STORE_INSTANCE_H +#define RELATIONAL_STORE_INSTANCE_H +#ifdef RELATIONAL_STORE + +#include +#include + +#include "irelational_store.h" +#include "kvdb_properties.h" + +namespace DistributedDB { +class RelationalStoreInstance final { +public: + RelationalStoreInstance(); + ~RelationalStoreInstance() = default; + + static RelationalStoreConnection *GetDatabaseConnection(const DBProperties &properties, int &errCode); + static RelationalStoreInstance *GetInstance(); + + int CheckDatabaseFileStatus(const std::string &id); + + // public for test mock + static IRelationalStore *GetDataBase(const DBProperties &properties, int &errCode); +private: + + IRelationalStore *OpenDatabase(const DBProperties &properties, int &errCode); + + void RemoveKvDBFromCache(const DBProperties &properties); + void SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties); + + static RelationalStoreInstance *instance_; + static std::mutex instanceLock_; + + std::string appId_; + std::string userId_; +}; +} // namespace DistributedDB + +#endif +#endif // RELATIONAL_STORE_INSTANCE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9a4a60324bc34d0cbdf3c274cbb73157e836b645 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_manager.cpp @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "relational_store_manager.h" + +#include + +#include "relational_store_instance.h" +#include "db_common.h" +#include "param_check_utils.h" +#include "log_print.h" +#include "db_errno.h" +#include "kv_store_errno.h" +#include "relational_store_delegate_impl.h" +#include "platform_specific.h" + +namespace DistributedDB { +RelationalStoreManager::RelationalStoreManager(const std::string &appId, const std::string &userId) + : appId_(appId), + userId_(userId) +{} + +static void InitStoreProp(const RelationalStoreDelegate::Option &option, const std::string &storePath, + const std::string &appId, const std::string &userId, DBProperties &properties) +{ + properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, option.createIfNecessary); + properties.SetStringProp(KvDBProperties::DATA_DIR, storePath); + properties.SetStringProp(KvDBProperties::APP_ID, appId); + properties.SetStringProp(KvDBProperties::USER_ID, userId); + properties.SetStringProp(KvDBProperties::STORE_ID, storePath); // same as dir + std::string identifier = userId + "-" + appId + "-" + storePath; + std::string hashIdentifier = DBCommon::TransferHashString(identifier); + properties.SetStringProp(KvDBProperties::IDENTIFIER_DATA, hashIdentifier); +} + +static const int GET_CONNECT_RETRY = 3; +static const int RETRY_GET_CONN_INTER = 30; + +static RelationalStoreConnection *GetOneConnectionWithRetry(const DBProperties &properties, int &errCode) +{ + for (int i = 0; i < GET_CONNECT_RETRY; i++) { + auto conn = RelationalStoreInstance::GetDatabaseConnection(properties, errCode); + if (conn != nullptr) { + return conn; + } + if (errCode == -E_STALE) { + std::this_thread::sleep_for(std::chrono::milliseconds(RETRY_GET_CONN_INTER)); + } else { + return nullptr; + } + } + return nullptr; +} + +void RelationalStoreManager::OpenStore(const std::string &path, const RelationalStoreDelegate::Option &option, + const std::function &callback) +{ + if (!callback) { + LOGE("[KvStoreMgr] Invalid callback for kv store!"); + return; + } + + if (!ParamCheckUtils::CheckStoreParameter("Relational_default_id", appId_, userId_) || path.empty()) { + callback(INVALID_ARGS, nullptr); + return; + } + + DBProperties properties; + InitStoreProp(option, path, appId_, userId_, properties); + + int errCode = E_OK; + auto *conn = GetOneConnectionWithRetry(properties, errCode); + DBStatus status = TransferDBErrno(errCode); + if (conn == nullptr) { + callback(status, nullptr); + return; + } + + auto store = new (std::nothrow) RelationalStoreDelegateImpl(conn, path); + if (store == nullptr) { + conn->Close(); + callback(DB_ERROR, nullptr); + return; + } + + (void)conn->TriggerAutoSync(); + callback(OK, store); +} + +DBStatus RelationalStoreManager::CloseStore(RelationalStoreDelegate *store) +{ + if (store == nullptr) { + return INVALID_ARGS; + } + + auto storeImpl = static_cast(store); + DBStatus status = storeImpl->Close(); + if (status == BUSY) { + LOGD("NbDelegateImpl is busy now."); + return BUSY; + } + storeImpl->SetReleaseFlag(true); + delete store; + store = nullptr; + return OK; +} + +static int RemoveFile(const std::string &fileName) +{ + if (!OS::CheckPathExistence(fileName)) { + return E_OK; + } + + if (OS::RemoveFile(fileName.c_str()) != E_OK) { + LOGE("Remove file failed:%d", errno); + return -E_REMOVE_FILE; + } + return E_OK; +} + +static int RemoveDB(const std::string &path) +{ + if (RemoveFile(path) != E_OK) { + LOGE("Remove the db file failed:%d", errno); + return -E_REMOVE_FILE; + } + + std::string dbFile = path + "-wal"; + if (RemoveFile(dbFile) != E_OK) { + LOGE("Remove the wal file failed:%d", errno); + return -E_REMOVE_FILE; + } + + dbFile = path + "-shm"; + if (RemoveFile(dbFile) != E_OK) { + LOGE("Remove the shm file failed:%d", errno); + return -E_REMOVE_FILE; + } + return E_OK; +} + +DBStatus RelationalStoreManager::DeleteStore(const std::string &path) +{ + if (path.empty()) { + LOGE("Invalid store info for deleting"); + return INVALID_ARGS; + } + + std::string identifier = userId_ + "-" + appId_ + "-" + path; + std::string hashIdentifier = DBCommon::TransferHashString(identifier); + + auto *manager = RelationalStoreInstance::GetInstance(); + int errCode = manager->CheckDatabaseFileStatus(hashIdentifier); + if (errCode != E_OK) { + LOGE("The store is busy!"); + return BUSY; + } + + // RemoveDB + errCode = RemoveDB(path); + if (errCode == E_OK) { + LOGI("Database deleted successfully!"); + return OK; + } + LOGE("Delete the kv store error:%d", errCode); + return TransferDBErrno(errCode); +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp new file mode 100644 index 0000000000000000000000000000000000000000..70209f45d85909b8baf3fbd954161fa279189e9d --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_store_sqlite_ext.cpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "sqlite_utils.h" +#include "log_print.h" +using namespace DistributedDB; +// namespace DistributedDB { +// We extend the original purpose of the "sqlite3ext.h". +SQLITE_API int sqlite3_close_relational(sqlite3* db) +{ + ::LOGD("### my sqlite3_close_relational!"); + return sqlite3_close(db); +} + +SQLITE_API int sqlite3_close_v2_relational(sqlite3* db) +{ + ::LOGD("### my sqlite3_close_v2_relational!"); + return sqlite3_close_v2(db); +} + +SQLITE_API int sqlite3_open_relational(const char *filename, sqlite3 **ppDb) +{ + int err = sqlite3_open(filename, ppDb); + DistributedDB::SQLiteUtils::RegisterCalcHash(*ppDb); + DistributedDB::SQLiteUtils::RegisterGetSysTime(*ppDb); + ::LOGD("### my sqlite3_open!"); + return err; +} + +SQLITE_API int sqlite3_open16_relational(const void *filename, sqlite3 **ppDb) +{ + int err = sqlite3_open16(filename, ppDb); + DistributedDB::SQLiteUtils::RegisterCalcHash(*ppDb); + DistributedDB::SQLiteUtils::RegisterGetSysTime(*ppDb); + ::LOGD("### my sqlite3_open16!"); + return err; +} + +SQLITE_API int sqlite3_open_v2_relational(const char *filename, sqlite3 **ppDb, int flags, const char *zVfs) +{ + int err = sqlite3_open_v2(filename, ppDb, flags, zVfs); + DistributedDB::SQLiteUtils::RegisterCalcHash(*ppDb); + DistributedDB::SQLiteUtils::RegisterGetSysTime(*ppDb); + ::LOGD("### my sqlite3_open_v2!"); + return err; +} + +// hw export the symbols +#ifdef SQLITE_DISTRIBUTE_RELATIONAL +#if defined(__GNUC__) +# define EXPORT_SYMBOLS __attribute__ ((visibility ("default"))) +#elif defined(_MSC_VER) + # define EXPORT_SYMBOLS __declspec(dllexport) +#else +# define EXPORT_SYMBOLS +#endif + +struct sqlite3_api_routines_relational { + int (*close)(sqlite3*); + int (*close_v2)(sqlite3*); + int (*open)(const char *, sqlite3 **); + int (*open16)(const void *, sqlite3 **); + int (*open_v2)(const char *, sqlite3 **, int, const char *); +}; + +typedef struct sqlite3_api_routines_relational sqlite3_api_routines_relational; +static const sqlite3_api_routines_relational sqlite3HwApis = { +#ifdef SQLITE_DISTRIBUTE_RELATIONAL + sqlite3_close_relational, + sqlite3_close_v2_relational, + sqlite3_open_relational, + sqlite3_open16_relational, + sqlite3_open_v2_relational +#else + 0, + 0, + 0, + 0, + 0 +#endif +}; + +EXPORT_SYMBOLS const sqlite3_api_routines_relational *sqlite3_export_relational_symbols = &sqlite3HwApis; +#endif + +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h new file mode 100644 index 0000000000000000000000000000000000000000..37ef57ada6909c8456de6080e71f83d7cdc63a29 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/relational_sync_able_storage.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_SYNC_ABLE_STORAGE_H +#define RELATIONAL_SYNC_ABLE_STORAGE_H +#ifdef RELATIONAL_STORE + +#include "relational_db_sync_interface.h" +#include "sqlite_single_relational_storage_engine.h" + +#include "sqlite_single_ver_continue_token.h" + +namespace DistributedDB { +class RelationalSyncAbleStorage : public RelationalDBSyncInterface, public virtual RefObject { +public: + explicit RelationalSyncAbleStorage(StorageEngine *engine); + ~RelationalSyncAbleStorage() override; + + // Get interface type of this kvdb. + int GetInterfaceType() const override; + + // Get the interface ref-count, in order to access asynchronously. + void IncRefCount() override; + + // Drop the interface ref-count. + void DecRefCount() override; + + // Get the identifier of this kvdb. + std::vector GetIdentifier() const override; + + // Get the max timestamp of all entries in database. + void GetMaxTimeStamp(TimeStamp &stamp) const override; + + // Get meta data associated with the given key. + int GetMetaData(const Key &key, Value &value) const override; + + // Put meta data as a key-value entry. + int PutMetaData(const Key &key, const Value &value) override; + + // Delete multiple meta data records in a transaction. + int DeleteMetaData(const std::vector &keys) override; + + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + + // Get all meta data keys. + int GetAllMetaKeys(std::vector &keys) const override; + + const KvDBProperties &GetDbProperties() const override; + + // Get the data which would be synced with query condition + int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const override; + + int GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, + const DataSizeSpecInfo &dataSizeInfo) const override; + + // Release the continue token of getting data. + void ReleaseContinueToken(ContinueToken &continueStmtToken) const override; + + int PutSyncDataWithQuery(const QueryObject &object, const std::vector &entries, + const DeviceID &deviceName) override; + + int RemoveDeviceData(const std::string &deviceName, bool isNeedNotify) override; + + RelationalSchemaObject GetSchemaInfo() const override; + + int GetSecurityOption(SecurityOption &option) const override; + + void NotifyRemotePushFinished(const std::string &deviceId) const override; + + // Get the timestamp when database created or imported + int GetDatabaseCreateTimeStamp(TimeStamp &outTime) const override; + + // Get batch meta data associated with the given key. + int GetBatchMetaData(const std::vector &keys, std::vector &entries) const override; + // Put batch meta data as a key-value entry vector + int PutBatchMetaData(std::vector &entries) override; + + std::vector GetTablesQuery() override; + + int LocalDataChanged(int notifyEvent, std::vector &queryObj) override; + + int SchemaChanged(int notifyEvent) override; + + int InterceptData(std::vector &entries, + const std::string &sourceID, const std::string &targetID) const override + { + return E_OK; + } + + int CheckAndInitQueryCondition(QueryObject &query) const override + { + return E_OK; + } + +private: + SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode, OperatePerm perm) const; + void ReleaseHandle(SQLiteSingleVerRelationalStorageExecutor *&handle) const; + int SetMaxTimeStamp(TimeStamp timestamp); + + // get + int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerContinueToken *&continueStmtToken, + const DataSizeSpecInfo &dataSizeInfo) const; + + // put + int PutSyncData(const QueryObject &object, std::vector &dataItems, const std::string &deviceName); + int SaveSyncDataItems(const QueryObject &object, std::vector &dataItems, const std::string &deviceName); + + // data + SQLiteSingleRelationalStorageEngine *storageEngine_ = nullptr; + TimeStamp currentMaxTimeStamp_ = 0; + KvDBProperties properties; + mutable std::mutex maxTimeStampMutex_; +}; +} // namespace DistributedDB +#endif +#endif // RELATIONAL_SYNC_ABLE_STORAGE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/runtime_config.cpp b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/runtime_config.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5988576c7dfa1dafcc0d14eeac8f317827adf303 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/interfaces/src/relational/runtime_config.cpp @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "runtime_config.h" + +#include "db_constant.h" +#include "kvdb_manager.h" +#include "kv_store_errno.h" +#include "log_print.h" +#include "network_adapter.h" +#include "runtime_context.h" + +namespace DistributedDB { +std::mutex RuntimeConfig::communicatorMutex_; +std::shared_ptr RuntimeConfig::processCommunicator_ = nullptr; + +// Used to set the process userid and appId +DBStatus RuntimeConfig::SetProcessLabel(const std::string &appId, const std::string &userId) +{ + if (appId.size() > DBConstant::MAX_APP_ID_LENGTH || appId.empty() || + userId.size() > DBConstant::MAX_USER_ID_LENGTH || userId.empty()) { + LOGE("Invalid app or user info[%zu]-[%zu]", appId.length(), userId.length()); + return INVALID_ARGS; + } + + int errCode = KvDBManager::SetProcessLabel(appId, userId); + if (errCode != E_OK) { + LOGE("Failed to set the process label:%d", errCode); + return DB_ERROR; + } + return OK; +} + +// Set process communicator. +DBStatus RuntimeConfig::SetProcessCommunicator(const std::shared_ptr &inCommunicator) +{ + std::lock_guard lock(communicatorMutex_); + if (processCommunicator_ != nullptr) { + LOGE("processCommunicator_ is not null!"); + return DB_ERROR; + } + + std::string processLabel = RuntimeContext::GetInstance()->GetProcessLabel(); + if (processLabel.empty()) { + LOGE("ProcessLabel is not set!"); + return DB_ERROR; + } + + auto *adapter = new (std::nothrow) NetworkAdapter(processLabel, inCommunicator); + if (adapter == nullptr) { + LOGE("New NetworkAdapter failed!"); + return DB_ERROR; + } + processCommunicator_ = inCommunicator; + if (RuntimeContext::GetInstance()->SetCommunicatorAdapter(adapter) != E_OK) { + LOGE("SetProcessCommunicator not support!"); + delete adapter; + return DB_ERROR; + } + KvDBManager::RestoreSyncableKvStore(); + return OK; +} + +DBStatus RuntimeConfig::SetPermissionCheckCallback(const PermissionCheckCallbackV2 &callback) +{ + int errCode = RuntimeContext::GetInstance()->SetPermissionCheckCallback(callback); + return TransferDBErrno(errCode); +} + +DBStatus RuntimeConfig::SetProcessSystemAPIAdapter(const std::shared_ptr &adapter) +{ + return TransferDBErrno(RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(adapter)); +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb.h b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb.h index e21c0455e2e435376117dc4e56962320bbe217e3..1aa687b9800c3e1e7878a680bd2aeefd61cd21f0 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb.h @@ -55,6 +55,8 @@ public: virtual int GetKvDBSize(const KvDBProperties &properties, uint64_t &size) const = 0; virtual void EnableAutonomicUpgrade() = 0; + + virtual int CheckIntegrity() const = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h index 37ca17240370b44803540bbec673d13f725cc4b6..75406272927c240fa6a6cdba2aa97806c4339549 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_connection.h @@ -124,6 +124,8 @@ public: // Get the securityLabel and securityFlag virtual int GetSecurityOption(int &securityLabel, int &securityFlag) const = 0; + + virtual int CheckIntegrity() const = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_sync_interface.h index bd698dbe0f0ee044f7b19e13c4ad3efa8d3bd560..572add415c36ac39eb9a88cfba2c2d1135eda082 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/ikvdb_sync_interface.h @@ -20,44 +20,15 @@ #include "db_types.h" #include "kvdb_properties.h" +#include "sync_generic_interface.h" namespace DistributedDB { -class IKvDBSyncInterface { +class IKvDBSyncInterface : public SyncGenericInterface { public: - enum { - SYNC_SVD = 1, // Single version data - SYNC_MVD, // Multi version data - }; // Constructor/Destructor. IKvDBSyncInterface() = default; - virtual ~IKvDBSyncInterface() {} - - // Get interface type of this kvdb. - virtual int GetInterfaceType() const = 0; - - // Get the interface ref-count, in order to access asynchronously. - virtual void IncRefCount() = 0; - - // Drop the interface ref-count. - virtual void DecRefCount() = 0; - - // Get the identifier of this kvdb. - virtual std::vector GetIdentifier() const = 0; - - // Get the max timestamp of all entries in database. - virtual void GetMaxTimeStamp(TimeStamp &stamp) const = 0; - - // Get meta data associated with the given key. - virtual int GetMetaData(const Key &key, Value &value) const = 0; - - // Put meta data as a key-value entry. - virtual int PutMetaData(const Key &key, const Value &value) = 0; - - // Get all meta data keys. - virtual int GetAllMetaKeys(std::vector &keys) const = 0; - - virtual const KvDBProperties &GetDbProperties() const = 0; + ~IKvDBSyncInterface() override = default; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h new file mode 100644 index 0000000000000000000000000000000000000000..375823e39b9a306f1d624e693814b234ca52fda6 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/isync_interface.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef I_SYNC_INTERFACE_H +#define I_SYNC_INTERFACE_H + +#include + +#include "db_types.h" +#include "kvdb_properties.h" + +namespace DistributedDB { +struct SyncTimeRange { + TimeStamp beginTime = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp endTime = static_cast(INT64_MAX); + TimeStamp deleteEndTime = static_cast(INT64_MAX); + bool IsValid() const + { + return (beginTime <= endTime && deleteBeginTime <= deleteEndTime); + } +}; + +class ISyncInterface { +public: + enum { + SYNC_SVD = 1, // Single version data + SYNC_MVD, // Multi version data + SYNC_RELATION, // Relation version data + }; + + // Constructor/Destructor. + ISyncInterface() = default; + virtual ~ISyncInterface() = default; + + // Get interface type of this kvdb. + virtual int GetInterfaceType() const = 0; + + // Get the interface ref-count, in order to access asynchronously. + virtual void IncRefCount() = 0; + + // Drop the interface ref-count. + virtual void DecRefCount() = 0; + + // Get the identifier of this kvdb. + virtual std::vector GetIdentifier() const = 0; + + // Get the max timestamp of all entries in database. + virtual void GetMaxTimeStamp(TimeStamp &stamp) const = 0; + + // Get meta data associated with the given key. + virtual int GetMetaData(const Key &key, Value &value) const = 0; + + // Put meta data as a key-value entry. + virtual int PutMetaData(const Key &key, const Value &value) = 0; + + // Delete multiple meta data records in a transaction. + virtual int DeleteMetaData(const std::vector &keys) = 0; + + // Delete multiple meta data records with key prefix in a transaction. + virtual int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const = 0; + + // Get all meta data keys. + virtual int GetAllMetaKeys(std::vector &keys) const = 0; + + virtual const KvDBProperties &GetDbProperties() const = 0; +}; +} // namespace DistributedDB + +#endif // I_SYNC_INTERFACE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_manager.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_manager.h index af88322022e642bf180d04496c80290edc7db7dd..32cbd175e1bfee5d59ded2091816fa45aedd57e0 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_manager.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_manager.h @@ -25,6 +25,7 @@ #include "db_errno.h" #include "ikvdb.h" #include "ikvdb_factory.h" +#include "platform_specific.h" namespace DistributedDB { class KvDBManager final { @@ -89,6 +90,8 @@ private: void SetAllDatabaseCorruptionHander(const KvStoreCorruptionHandler &handler); + IKvDB *CreateDataBase(const KvDBProperties &property, int &errCode); + IKvDB *GetDataBase(const KvDBProperties &property, int &errCode, bool isNeedIfOpened); void DataBaseCorruptNotify(const std::string &appId, const std::string &userId, const std::string &storeId); @@ -117,9 +120,15 @@ private: IKvDB *GetKvDBFromCacheByIdentify(const std::string &identifier, const std::map &cache) const; - static KvDBManager *instance_; + static int CheckRemoveStateAndRetry(const KvDBProperties &property); + + static int TryLockDB(const KvDBProperties &kvDBProp, int retryTimes); + static int UnlockDB(const KvDBProperties &kvDBProp); + + static std::atomic instance_; static std::mutex kvDBLock_; static std::mutex instanceLock_; + static std::map locks_; std::map localKvDBs_; std::map multiVerNaturalStores_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h index c6ecc0464216cb1a15fe4684c267574dc078a758..f873b5b165280796b4e1338f68983bb3e5841ae9 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_pragma.h @@ -21,6 +21,7 @@ #include #include "types.h" +#include "query_sync_object.h" namespace DistributedDB { enum : int { @@ -44,18 +45,43 @@ enum : int { PRAGMA_RESULT_SET_CACHE_MAX_SIZE, PRAGMA_TRIGGER_TO_MIGRATE_DATA, PRAGMA_REMOTE_PUSH_FINISHED_NOTIFY, + PRAGMA_SET_SYNC_RETRY, + PRAGMA_ADD_EQUAL_IDENTIFIER, + PRAGMA_INTERCEPT_SYNC_DATA, + PRAGMA_SUBSCRIBE_QUERY, }; struct PragmaSync { + PragmaSync(const std::vector &devices, int mode, const QuerySyncObject &query, + const std::function &devicesMap)> &onComplete, + bool wait = false) + : devices_(devices), + mode_(mode), + onComplete_(onComplete), + wait_(wait), + isQuerySync_(true), + query_(query) + { + } + PragmaSync(const std::vector &devices, int mode, const std::function &devicesMap)> &onComplete, bool wait = false) - : devices_(devices), mode_(mode), onComplete_(onComplete), wait_(wait) {} + : devices_(devices), + mode_(mode), + onComplete_(onComplete), + wait_(wait), + isQuerySync_(false), + query_(Query::Select()) + { + } std::vector devices_; int mode_; std::function &devicesMap)> onComplete_; bool wait_; + bool isQuerySync_; + QuerySyncObject query_; }; struct PragmaRemotePushNotify { @@ -63,6 +89,15 @@ struct PragmaRemotePushNotify { RemotePushFinishedNotifier notifier_; }; + +struct PragmaSetEqualIdentifier { + PragmaSetEqualIdentifier(const std::string &identifier, const std::vector &targets) + : identifier_(identifier), + targets_(targets) {} + + std::string identifier_; + std::vector targets_; +}; } // namespace DistributedDB #endif // KV_DB_PRAGMA_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h index 9c8a663839b6a5949d3541f98a7e2414503ec96c..6337094af09f61f30f7e164874fea1a525700b92 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/kvdb_properties.h @@ -23,13 +23,10 @@ #include "schema_object.h" namespace DistributedDB { -class KvDBProperties final { +class DBProperties { public: - KvDBProperties(); - ~KvDBProperties(); - - // Get the sub directory for different type database. - static std::string GetStoreSubDirectory(int type); + DBProperties() = default; + ~DBProperties() = default; // Get the string property according the name std::string GetStringProp(const std::string &name, const std::string &defaultValue) const; @@ -48,6 +45,19 @@ public: // Set the integer property for the name void SetIntProp(const std::string &name, int value); +protected: + std::map stringProperties_; + std::map boolProperties_; + std::map intProperties_; +}; + +class KvDBProperties final : public DBProperties { +public: + KvDBProperties(); + ~KvDBProperties(); + + // Get the sub directory for different type database. + static std::string GetStoreSubDirectory(int type); // Get the password void GetPassword(CipherType &type, CipherPassword &password) const; @@ -68,6 +78,7 @@ public: int GetSecLabel() const; int GetSecFlag() const; + // Get schema const reference if you can guarantee the lifecycle of this KvDBProperties // The upper code will not change the schema if it is already set const SchemaObject &GetSchemaConstRef() const; @@ -89,15 +100,16 @@ public: static const std::string SECURITY_LABEL; static const std::string SECURITY_FLAG; static const std::string CONFLICT_RESOLVE_POLICY; + static const std::string CHECK_INTEGRITY; + static const std::string RM_CORRUPTED_DB; + static const std::string COMPRESS_ON_SYNC; + static const std::string COMPRESSION_RATE; static const int LOCAL_TYPE = 1; static const int MULTI_VER_TYPE = 2; static const int SINGLE_VER_TYPE = 3; private: - std::map stringProperties_; - std::map boolProperties_; - std::map intProperties_; CipherType cipherType_; CipherPassword password_; SchemaObject schema_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/multi_ver_vacuum.h b/services/distributeddataservice/libs/distributeddb/storage/include/multi_ver_vacuum.h index 613a3dc9f148cce3eb1fef9b3285c67bc91fbc30..fbc70e7b63913b614d192ec2535cee4139e593ee 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/multi_ver_vacuum.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/multi_ver_vacuum.h @@ -100,17 +100,21 @@ private: // Reducing duplicated code by merging similar code procedure of "DealLeftCommit" and "DealRightCommit" int DoDealCommitOfLeftOrRight(VacuumTaskContext &inTask, std::list &commitList, bool isLeft); + // Reducing duplicated code by merging similar code procedure of "DealLeftShadow" and "DealRightVacuumNeed" int DoDeleteRecordOfLeftShadowOrRightVacuumNeed(VacuumTaskContext &inTask, std::list &recordList); + // Only for reducing duplicated code void DoRollBackAndFinish(VacuumTaskContext &inTask); int DoCommitAndQuitIfWaitStatusObserved(VacuumTaskContext &inTask); // Return E_OK continue otherwise quit // Call this immediately before changing the database int StartTransactionIfNotYet(VacuumTaskContext &inTask); + // Call this immediately before normally quit int CommitTransactionIfNeed(VacuumTaskContext &inTask); + // Call this immediately before abnormally quit, return void since already in abnormal. void RollBackTransactionIfNeed(VacuumTaskContext &inTask); @@ -131,6 +135,7 @@ private: std::condition_variable vacuumTaskCv_; uint64_t incRunWaitOrder_ = 0; std::map dbMapVacuumTask_; + // the search of available vacuumtask, the change of isBackgroundVacuumTaskInExecution_, and the activation of // background execution, should all be protected by vacuumTaskMutex_, In order to avoid malfunction caused by // concurrency situation which is described below: diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h new file mode 100644 index 0000000000000000000000000000000000000000..94749dba30aee2f7f38472f548f733f02786599d --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_db_sync_interface.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_DB_SYNC_INTERFACE_H +#define RELATIONAL_DB_SYNC_INTERFACE_H +#ifdef RELATIONAL_STORE + +#include "query_sync_object.h" +#include "relational_schema_object.h" +#include "single_ver_kv_entry.h" +#include "sync_generic_interface.h" + +namespace DistributedDB { +class RelationalDBSyncInterface : public SyncGenericInterface { +public: + ~RelationalDBSyncInterface() override {}; + + virtual RelationalSchemaObject GetSchemaInfo() const = 0; + + // Get batch meta data associated with the given key. + virtual int GetBatchMetaData(const std::vector &keys, std::vector &entries) const = 0; + // Put batch meta data as a key-value entry vector + virtual int PutBatchMetaData(std::vector &entries) = 0; + + virtual std::vector GetTablesQuery() = 0; + + virtual int LocalDataChanged(int notifyEvent, std::vector &queryObj) = 0; + + virtual int SchemaChanged(int notifyEvent) = 0; +}; +} +#endif // RELATIONAL_STORE +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h new file mode 100644 index 0000000000000000000000000000000000000000..e502357a79c436dd4b25aea0853765289aa27cde --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/relational_store_connection.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_STORE_CONNECTION_H +#define RELATIONAL_STORE_CONNECTION_H +#ifdef RELATIONAL_STORE + +#include +#include +#include "macro_utils.h" +#include "relational_store_delegate.h" +#include "ref_object.h" + +namespace DistributedDB { +class IRelationalStore; + +class RelationalStoreConnection : public virtual RefObject { +public: + struct SyncInfo { + const std::vector &devices; + SyncMode mode = SYNC_MODE_PUSH_PULL; + SyncStatusCallback &onComplete; + const Query &query; + bool wait = true; + }; + RelationalStoreConnection() = default; + explicit RelationalStoreConnection(IRelationalStore *store) + { + store_ = store; + }; + + virtual ~RelationalStoreConnection() = default; + + DISABLE_COPY_ASSIGN_MOVE(RelationalStoreConnection); + + // Close and release the connection. + virtual int Close() = 0; + virtual int TriggerAutoSync() = 0; + virtual int SyncToDevice(SyncInfo &info) = 0; + virtual std::string GetIdentifier() = 0; + virtual int CreateDistributedTable(const std::string &tableName, + const RelationalStoreDelegate::TableOption &option) = 0; + +protected: + // Get the stashed 'KvDB_ pointer' without ref. + template + DerivedDBType *GetDB() const + { + return static_cast(store_); + } + + virtual int Pragma(int cmd, void *parameter); + IRelationalStore *store_ = nullptr; + std::atomic isExclusive_ = false; +}; +} // namespace DistributedDB +#endif +#endif // RELATIONAL_STORE_CONNECTION_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_kv_entry.h b/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kv_entry.h similarity index 76% rename from services/distributeddataservice/libs/distributeddb/storage/src/single_ver_kv_entry.h rename to services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kv_entry.h index 75ce5ff478fe6051af0f5af06e685bdb07c2ad58..f9a870017a9983bad29bf07635e6068ae73a8e05 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_kv_entry.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kv_entry.h @@ -33,6 +33,20 @@ public: virtual int SerializeData(Parcel &parcel, uint32_t softWareVersion) = 0; virtual int DeSerializeData(Parcel &parcel) = 0; virtual uint32_t CalculateLen(uint32_t softWareVersion) = 0; + virtual const Key &GetKey() const = 0; + virtual const Value &GetValue() const = 0; + virtual void SetKey(const Key &key) = 0; + virtual void SetValue(const Value &value) = 0; + virtual void SetHashKey(const Key &hashKey) = 0; + + static void Release(std::vector &entries) + { + for (auto &entry : entries) { + delete entry; + entry = nullptr; + } + entries.clear(); + }; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h index 828c4de2298ac294d1143b791d05fbcc8a499af5..1cca304b6ed674ff3f034566a190a2cab67c1965 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/single_ver_kvdb_sync_interface.h @@ -19,49 +19,24 @@ #include "ikvdb_sync_interface.h" #include "single_ver_kv_entry.h" #include "iprocess_system_api_adapter.h" +#include "query_object.h" +#include "intercepted_data.h" namespace DistributedDB { +using MulDevTimeRanges = std::map>; +using MulDevSinVerKvEntry = std::map>; +using MulDevDataItems = std::map>; + class SingleVerKvDBSyncInterface : public IKvDBSyncInterface { public: - ~SingleVerKvDBSyncInterface() override {}; - - // Get the data which would be synced to other devices according the timestamp. - // if the data size is over than the blockSize, It would alloc one token and assign to continueStmtToken, - // it should be released when the read operation terminate. - virtual int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &dataItems, - ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const = 0; - - virtual int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &entries, - ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const = 0; - - // Get the data using the token allocated by GetSyncData, the token would be release automatically when finished. - virtual int GetSyncDataNext(std::vector &dataItems, ContinueToken &continueStmtToken, - const DataSizeSpecInfo &dataSizeInfo) const = 0; - - virtual int GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, - const DataSizeSpecInfo &dataSizeInfo) const = 0; - - // Release the continue token of getting data. - virtual void ReleaseContinueToken(ContinueToken &continueStmtToken) const = 0; - - // Put synced data from remote devices. - virtual int PutSyncData(std::vector &dataItems, const std::string &deviceName) = 0; - - virtual int PutSyncData(const std::vector &entries, const std::string &deviceName) = 0; - - virtual void ReleaseKvEntry(const SingleVerKvEntry *entry) = 0; - - virtual int RemoveDeviceData(const std::string &deviceName, bool isNeedNotify) = 0; + SingleVerKvDBSyncInterface() = default; + ~SingleVerKvDBSyncInterface() override = default; virtual SchemaObject GetSchemaInfo() const = 0; virtual bool CheckCompatible(const std::string &schema) const = 0; - virtual int GetSecurityOption(SecurityOption &option) const = 0; - - virtual bool IsReadable() const = 0; - - virtual void NotifyRemotePushFinished(const std::string &targetId) const = 0; + virtual int GetCompressionAlgo(std::set &algorithmSet) const = 0; }; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/storage_engine_manager.h b/services/distributeddataservice/libs/distributeddb/storage/include/storage_engine_manager.h index ad11788439e4d3e325bf582a78d67ff4345afb4d..717cc0ed4363f9fa50b2d0045a58ad7bc74b7b0a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/include/storage_engine_manager.h +++ b/services/distributeddataservice/libs/distributeddb/storage/include/storage_engine_manager.h @@ -67,7 +67,7 @@ private: void ExitGetEngineProcess(const std::string &identifier); static std::mutex instanceLock_; - static StorageEngineManager *instance_; + static std::atomic instance_; static bool isRegLockStatusListener_; static std::mutex storageEnginesLock_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/include/sync_generic_interface.h b/services/distributeddataservice/libs/distributeddb/storage/include/sync_generic_interface.h new file mode 100644 index 0000000000000000000000000000000000000000..6068081113e839d5200c564017f1ae33f2dfb4d0 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/include/sync_generic_interface.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SYNC_GENERIC_INTERFACE_H +#define SYNC_GENERIC_INTERFACE_H + +#include "isync_interface.h" +#include "single_ver_kv_entry.h" +#include "query_object.h" + +namespace DistributedDB { +class SyncGenericInterface : public ISyncInterface { +public: + // Constructor/Destructor. + SyncGenericInterface() = default; + ~SyncGenericInterface() override = default; + + virtual int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &dataItems, + ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const + { + LOGE("GetSyncData not support!"); + return -E_NOT_SUPPORT; + } + + // Get the data which would be synced to other devices according the timestamp. + // if the data size is over than the blockSize, It would alloc one token and assign to continueStmtToken, + // it should be released when the read operation terminate. + virtual int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &entries, + ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const + { + LOGE("GetSyncData not support!"); + return -E_NOT_SUPPORT; + } + + // Get the data which would be synced with query condition + virtual int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const + { + return -E_NOT_SUPPORT; + } + + virtual int GetSyncDataNext(std::vector &dataItems, ContinueToken &continueStmtToken, + const DataSizeSpecInfo &dataSizeInfo) const + { + return -E_NOT_SUPPORT; + } + + virtual int GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, + const DataSizeSpecInfo &dataSizeInfo) const + { + return -E_NOT_SUPPORT; + } + + virtual int GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const + { + return -E_NOT_SUPPORT; + } + + // Release the continue token of getting data. + virtual void ReleaseContinueToken(ContinueToken &continueStmtToken) const + { + } + + virtual int RemoveDeviceData(const std::string &deviceName, bool isNeedNotify) + { + return -E_NOT_SUPPORT; + } + + virtual bool IsReadable() const + { + return true; + } + + virtual int GetSecurityOption(SecurityOption &option) const + { + return -E_NOT_SUPPORT; + } + + virtual void NotifyRemotePushFinished(const std::string &targetId) const + { + } + + // Get the timestamp when database created or imported + virtual int GetDatabaseCreateTimeStamp(TimeStamp &outTime) const + { + return -E_NOT_SUPPORT; + } + + virtual int PutSyncDataWithQuery(const QueryObject &query, const std::vector &entries, + const std::string &deviceName) + { + return -E_NOT_SUPPORT; + } + + virtual int CheckAndInitQueryCondition(QueryObject &query) const + { + return -E_NOT_SUPPORT; + } + + virtual int InterceptData(std::vector &entries, const std::string &sourceID, + const std::string &targetID) const + { + return -E_NOT_SUPPORT; + } + + virtual int AddSubscribe(const std::string &subscribeId, const QueryObject &query, bool needCacheSubscribe) + { + return -E_NOT_SUPPORT; + } + + virtual int RemoveSubscribe(const std::string &subscribeId) + { + return -E_NOT_SUPPORT; + } + + virtual int RemoveSubscribe(const std::vector &subscribeIds) + { + return -E_NOT_SUPPORT; + } +}; +} +#endif // SYNC_GENERIC_INTERFACE_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c4e020261e3acc597b205257197520cb772ff2a5 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.cpp @@ -0,0 +1,388 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "data_transformer.h" + +#include "db_errno.h" +#include "log_print.h" +#include "parcel.h" +#include "relational_schema_object.h" + +namespace DistributedDB { +int DataTransformer::TransformTableData(const TableDataWithLog &tableDataWithLog, + const std::vector &fieldInfoList, std::vector &dataItems) +{ + if (tableDataWithLog.dataList.empty()) { + return E_OK; + } + for (const RowDataWithLog& data : tableDataWithLog.dataList) { + DataItem dataItem; + int errCode = SerializeDataItem(data, fieldInfoList, dataItem); + if (errCode != E_OK) { + return errCode; + } + dataItems.push_back(std::move(dataItem)); + } + return E_OK; +} + +int DataTransformer::TransformDataItem(const std::vector &dataItems, + const std::vector &remoteFieldInfo, const std::vector &localFieldInfo, + OptTableDataWithLog &tableDataWithLog) +{ + if (dataItems.empty()) { + return E_OK; + } + std::vector indexMapping; + ReduceMapping(remoteFieldInfo, localFieldInfo, indexMapping); + for (const DataItem &dataItem : dataItems) { + OptRowDataWithLog dataWithLog; + int errCode = DeSerializeDataItem(dataItem, dataWithLog, remoteFieldInfo, indexMapping); + if (errCode != E_OK) { + return errCode; + } + tableDataWithLog.dataList.push_back(std::move(dataWithLog)); + } + return E_OK; +} + +int DataTransformer::SerializeDataItem(const RowDataWithLog &data, + const std::vector &fieldInfo, DataItem &dataItem) +{ + int errCode = SerializeValue(dataItem.value, data.rowData, fieldInfo); + if (errCode != E_OK) { + return errCode; + } + const LogInfo &logInfo = data.logInfo; + errCode = SerializeHashKey(dataItem.hashKey, logInfo.hashKey); + if (errCode != E_OK) { + return errCode; + } + dataItem.timeStamp = logInfo.timestamp; + dataItem.dev = logInfo.device; + dataItem.origDev = logInfo.originDev; + dataItem.writeTimeStamp = logInfo.wTimeStamp; + dataItem.flag = logInfo.flag; + return E_OK; +} + +int DataTransformer::DeSerializeDataItem(const DataItem &dataItem, OptRowDataWithLog &data, + const std::vector &remoteFieldInfo, std::vector &indexMapping) +{ + int errCode; + if ((dataItem.flag & DataItem::DELETE_FLAG) == 0) { + errCode = DeSerializeValue(dataItem.value, data.optionalData, remoteFieldInfo, indexMapping); + if (errCode != E_OK) { + return errCode; + } + } + + LogInfo &logInfo = data.logInfo; + errCode = DeSerializeHashKey(dataItem.hashKey, logInfo.hashKey); + if (errCode != E_OK) { + return errCode; + } + logInfo.timestamp = dataItem.timeStamp; + logInfo.device = dataItem.dev; + logInfo.originDev = dataItem.origDev; + logInfo.wTimeStamp = dataItem.writeTimeStamp; + logInfo.flag = dataItem.flag; + return E_OK; +} + +uint32_t DataTransformer::CalDataValueLength(const DataValue &dataValue) +{ + static std::map lengthMap = { + { StorageType::STORAGE_TYPE_NULL, Parcel::GetUInt32Len()}, + { StorageType::STORAGE_TYPE_BOOL, Parcel::GetBoolLen()}, + { StorageType::STORAGE_TYPE_INTEGER, Parcel::GetInt64Len()}, + { StorageType::STORAGE_TYPE_REAL, Parcel::GetDoubleLen()} + }; + if (lengthMap.find(dataValue.GetType()) != lengthMap.end()) { + return lengthMap[dataValue.GetType()]; + } + if (dataValue.GetType() != StorageType::STORAGE_TYPE_BLOB && + dataValue.GetType() != StorageType::STORAGE_TYPE_TEXT) { + return 0u; + } + uint32_t length = 0; + std::string str; + switch (dataValue.GetType()) { + case StorageType::STORAGE_TYPE_BLOB: + (void)dataValue.GetBlobLength(length); + length = Parcel::GetEightByteAlign(length); + length += Parcel::GetUInt32Len(); // record data length + break; + case StorageType::STORAGE_TYPE_TEXT: + (void)dataValue.GetText(str); + length = Parcel::GetStringLen(str); + break; + default: + break; + } + return length; +} + +void DataTransformer::ReduceMapping(const std::vector &remoteFieldInfo, + const std::vector &localFieldInfo, std::vector &indexMapping) +{ + std::map fieldMap; + for (int i = 0; i < (int)remoteFieldInfo.size(); ++i) { + const auto &fieldInfo = remoteFieldInfo[i]; + fieldMap[fieldInfo.GetFieldName()] = i; + } + for (const auto &fieldInfo : localFieldInfo) { + if (fieldMap.find(fieldInfo.GetFieldName()) == fieldMap.end()) { + indexMapping.push_back(-E_NOT_FOUND); + continue; + } + indexMapping.push_back(fieldMap[fieldInfo.GetFieldName()]); + } +} + +namespace { +int SerializeNullValue(const DataValue &dataValue, Parcel &parcel) +{ + return parcel.WriteUInt32(0u); +} + +int DeSerializeNullValue(DataValue &dataValue, Parcel &parcel) +{ + uint32_t dataLength = -1; + (void)parcel.ReadUInt32(dataLength); + if (parcel.IsError() || dataLength != 0) { + return -E_PARSE_FAIL; + } + dataValue.ResetValue(); + return E_OK; +} + +int SerializeBoolValue(const DataValue &dataValue, Parcel &parcel) +{ + bool val = false; + (void)dataValue.GetBool(val); + return parcel.WriteBool(val); +} + +int DeSerializeBoolValue(DataValue &dataValue, Parcel &parcel) +{ + bool val = false; + (void)parcel.ReadBool(val); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + dataValue = val; + return E_OK; +} + +int SerializeIntValue(const DataValue &dataValue, Parcel &parcel) +{ + int64_t val = 0; + (void)dataValue.GetInt64(val); + return parcel.WriteInt64(val); +} + +int DeSerializeIntValue(DataValue &dataValue, Parcel &parcel) +{ + int64_t val = 0; + (void)parcel.ReadInt64(val); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + dataValue = val; + return E_OK; +} + +int SerializeDoubleValue(const DataValue &dataValue, Parcel &parcel) +{ + double val = 0; + (void)dataValue.GetDouble(val); + return parcel.WriteDouble(val); +} + +int DeSerializeDoubleValue(DataValue &dataValue, Parcel &parcel) +{ + double val = 0; + (void)parcel.ReadDouble(val); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + dataValue = val; + return E_OK; +} + +int SerializeTextValue(const DataValue &dataValue, Parcel &parcel) +{ + std::string val; + (void)dataValue.GetText(val); + return parcel.WriteString(val); +} + +int DeSerializeTextValue(DataValue &dataValue, Parcel &parcel) +{ + std::string val; + (void)parcel.ReadString(val); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + dataValue = val; + return E_OK; +} + +int SerializeBlobValue(const DataValue &dataValue, Parcel &parcel) +{ + Blob val; + (void)dataValue.GetBlob(val); + const uint32_t &size = val.GetSize(); + if (size == 0) { + return SerializeNullValue(dataValue, parcel); + } + int errCode = parcel.WriteUInt32(size); + if (errCode != E_OK) { + return errCode; + } + return parcel.WriteBlob(reinterpret_cast(val.GetData()), size); +} + +int DeSerializeBlobValue(DataValue &dataValue, Parcel &parcel) +{ + Blob val; + uint32_t blobLength = 0; + (void)parcel.ReadUInt32(blobLength); + if (blobLength == 0) { + return E_OK; + } + char array[blobLength]; + (void)parcel.ReadBlob(array, blobLength); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + int errCode = val.WriteBlob(reinterpret_cast(array), blobLength); + if (errCode == E_OK) { + dataValue = val; + } + return errCode; +} + +struct FunctionEntry { + std::function serializeFunc; + std::function deSerializeFunc; +}; + +std::map typeFuncMap = { + { StorageType::STORAGE_TYPE_NULL, {SerializeNullValue, DeSerializeNullValue}}, + { StorageType::STORAGE_TYPE_BOOL, {SerializeBoolValue, DeSerializeBoolValue}}, + { StorageType::STORAGE_TYPE_INTEGER, {SerializeIntValue, DeSerializeIntValue}}, + { StorageType::STORAGE_TYPE_REAL, {SerializeDoubleValue, DeSerializeDoubleValue}}, + { StorageType::STORAGE_TYPE_TEXT, {SerializeTextValue, DeSerializeTextValue}}, + { StorageType::STORAGE_TYPE_BLOB, {SerializeBlobValue, DeSerializeBlobValue}}, +}; +} + +int DataTransformer::SerializeValue(Value &value, const RowData &rowData, const std::vector &fieldInfoList) +{ + if (rowData.size() != fieldInfoList.size()) { + LOGE("[DataTransformer][SerializeValue] unequal field counts!"); + return -E_INVALID_ARGS; + } + + uint32_t totalLength = Parcel::GetUInt64Len(); // first record field count + for (uint32_t i = 0; i < rowData.size(); ++i) { + const auto &dataValue = rowData[i]; + const auto &fieldInfo = fieldInfoList[i]; + if (dataValue.GetType() != StorageType::STORAGE_TYPE_NULL && + dataValue.GetType() != fieldInfo.GetStorageType()) { + return -E_INVALID_DATA; + } + if (typeFuncMap.find(dataValue.GetType()) == typeFuncMap.end()) { + return -E_NOT_SUPPORT; + } + uint32_t dataLength = CalDataValueLength(dataValue); + totalLength += dataLength; + } + value.resize(totalLength); + Parcel parcel(value.data(), value.size()); + (void)parcel.WriteUInt64(rowData.size()); + int index = 0; + for (const auto &dataValue : rowData) { + const auto &fieldInfo = fieldInfoList[index++]; + StorageType type = dataValue.GetType(); + if (dataValue.GetType() == StorageType::STORAGE_TYPE_NULL) { + type = fieldInfo.GetStorageType(); + } + int errCode = typeFuncMap[type].serializeFunc(dataValue, parcel); + if (errCode != E_OK) { + value.clear(); + return errCode; + } + } + return E_OK; +} + +int DataTransformer::DeSerializeValue(const Value &value, OptRowData &optionalData, + const std::vector &remoteFieldInfo, std::vector &indexMapping) +{ + Parcel parcel(const_cast(value.data()), value.size()); + uint64_t fieldCount = 0; + (void)parcel.ReadUInt64(fieldCount); + if (fieldCount != remoteFieldInfo.size()) { + LOGE("[DataTransformer][DeSerializeValue] unequal field counts!"); + return -E_INVALID_ARGS; + } + std::vector valueList; + for (const auto &fieldInfo : remoteFieldInfo) { + DataValue dataValue; + LOGD("[DataTransformer][DeSerializeValue] start deSerialize %s type %d", + fieldInfo.GetFieldName().c_str(), fieldInfo.GetStorageType()); + int errCode = typeFuncMap[fieldInfo.GetStorageType()].deSerializeFunc(dataValue, parcel); + if (errCode != E_OK) { + LOGD("[DataTransformer][DeSerializeValue] deSerialize %s failed", fieldInfo.GetFieldName().c_str()); + return errCode; + } + valueList.push_back(std::move(dataValue)); + } + for (const auto &index : indexMapping) { + if (index < 0) { + optionalData.push_back(std::nullopt); + continue; + } + if ((uint32_t)index >= valueList.size()) { + return -E_INTERNAL_ERROR; // should not happen + } + if (valueList[index].GetType() == StorageType::STORAGE_TYPE_NULL) { + optionalData.push_back(std::nullopt); + } else { + optionalData.push_back(valueList[index]); + } + } + return E_OK; +} + +int DataTransformer::SerializeHashKey(Key &key, const std::string &hashKey) +{ + key.resize(Parcel::GetStringLen(hashKey), 0); + Parcel parcel(key.data(), Parcel::GetStringLen(hashKey)); + int errCode = parcel.WriteString(hashKey); + return errCode; +} + +int DataTransformer::DeSerializeHashKey(const Key &key, std::string &hashKey) +{ + Parcel parcel(const_cast(key.data()), key.size()); + (void)parcel.ReadString(hashKey); + return parcel.IsError() ? -E_PARSE_FAIL : E_OK; +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.h b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.h new file mode 100644 index 0000000000000000000000000000000000000000..51a7d8510e7ca0cb36ec2486abb46916c3b2eab8 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/data_transformer.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DATA_TRANSFORMER_H +#define DATA_TRANSFORMER_H +#ifdef RELATIONAL_STORE + +#include +#include +#include "data_value.h" +#include "db_types.h" +#include "relational_schema_object.h" + +namespace DistributedDB { +using RowData = std::vector; +using OptRowData = std::vector>; + +struct LogInfo { + int dataKey = -1; + std::string device; + std::string originDev; + TimeStamp timestamp = 0; + TimeStamp wTimeStamp = 0; + uint64_t flag = 0; + std::string hashKey; // primary key hash value +}; + +struct RowDataWithLog { + LogInfo logInfo; + RowData rowData; +}; + +struct OptRowDataWithLog { + LogInfo logInfo; + OptRowData optionalData; +}; + +struct TableDataWithLog { + std::string tableName; + std::vector dataList; +}; + +struct OptTableDataWithLog { + std::string tableName; + std::vector dataList; +}; + +class DataTransformer { +public: + + static int TransformTableData(const TableDataWithLog &tableDataWithLog, + const std::vector &fieldInfoList, std::vector &dataItems); + static int TransformDataItem(const std::vector &dataItems, const std::vector &remoteFieldInfo, + const std::vector &localFieldInfo, OptTableDataWithLog &tableDataWithLog); + + static int SerializeDataItem(const RowDataWithLog &data, const std::vector &fieldInfo, + DataItem &dataItem); + static int DeSerializeDataItem(const DataItem &dataItem, OptRowDataWithLog &data, + const std::vector &remoteFieldInfo, std::vector &indexMapping); + static void ReduceMapping(const std::vector &remoteFieldInfo, + const std::vector &localFieldInfo, std::vector &indexMapping); + +private: + static int SerializeHashKey(Key &key, const std::string &hashKey); + static int SerializeValue(Value &value, const RowData &rowData, const std::vector &fieldInfoList); + + static int DeSerializeHashKey(const Key &key, std::string &hashKey); + static int DeSerializeValue(const Value &value, OptRowData &optionalData, + const std::vector &remoteFieldInfo, std::vector &indexMapping); + + static uint32_t CalDataValueLength(const DataValue &dataValue); +}; +} + +#endif +#endif // DATA_TRANSFORMER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.cpp index 158a435d794ff962aa0d88436e30f7956963cec6..9802edf55ac5dc0c0f6e41b35e1cccffc6887e2c 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.cpp @@ -374,6 +374,11 @@ void GenericKvDB::SetConnectionFlag(bool isExisted) const return; } +int GenericKvDB::CheckIntegrity() const +{ + return E_OK; +} + void GenericKvDB::GetStoreDirectory(const KvDBProperties &properties, int dbType, std::string &storeDir, std::string &storeOnlyDir) const { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.h b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.h index 17b9585b37faaf9d7e4425172c59e050980ab51e..5cacf01892d305b7b7ab2128f3e9873ee11f0c51 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb.h @@ -126,6 +126,8 @@ public: virtual void SetConnectionFlag(bool isExisted) const; + int CheckIntegrity() const override; + protected: // Create a connection object, no DB ref increased. virtual GenericKvDBConnection *NewConnection(int &errCode) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.cpp index 19935e922829f80ccab577bb524f22e342ecd57a..8bc0c90269ec110691681b6d1b70cf562ceb996b 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.cpp @@ -321,5 +321,9 @@ int GenericKvDBConnection::GetEventType(unsigned mode, std::list &eventType return TranslateObserverModeToEventTypes(mode, eventTypes); } -} +int GenericKvDBConnection::CheckIntegrity() const +{ + return E_OK; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.h index 91bac5720d1c0418b99b6d974feeb77db6119547..e5e2378e1683cdf0f7a924d60f6d5a41043d2d51 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_kvdb_connection.h @@ -72,6 +72,8 @@ public: int RegisterLifeCycleCallback(const DatabaseLifeCycleNotifier ¬ifier) override; int GetSecurityOption(int &securityLabel, int &securityFlag) const override; + + int CheckIntegrity() const override; protected: // Get the stashed 'KvDB_ pointer' without ref. template diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.cpp index da289231d7fe81948958107b01a0526b5b32f976..b1359bf7c1dc05686e1d1530ad8c83659240cf1c 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.cpp @@ -16,6 +16,7 @@ #include "generic_single_ver_kv_entry.h" #include +#include "data_compression.h" #include "db_errno.h" #include "parcel.h" #include "version.h" @@ -77,16 +78,46 @@ void GenericSingleVerKvEntry::GetKey(Key &key) const key = dataItem_.key; } +void GenericSingleVerKvEntry::GetHashKey(Key &key) const +{ + key = dataItem_.hashKey; +} + +const Key &GenericSingleVerKvEntry::GetKey() const +{ + return dataItem_.key; +} + void GenericSingleVerKvEntry::GetValue(Value &value) const { value = dataItem_.value; } +const Value &GenericSingleVerKvEntry::GetValue() const +{ + return dataItem_.value; +} + uint64_t GenericSingleVerKvEntry::GetFlag() const { return dataItem_.flag; } +void GenericSingleVerKvEntry::SetKey(const Key &key) +{ + dataItem_.key = key; +} + +void GenericSingleVerKvEntry::SetValue(const Value &value) +{ + dataItem_.value = value; +} + +void GenericSingleVerKvEntry::SetHashKey(const Key &hashKey) +{ + dataItem_.hashKey = hashKey; +} + // this func should do compatible int GenericSingleVerKvEntry::SerializeData(Parcel &parcel, uint32_t targetVersion) { @@ -105,10 +136,10 @@ int GenericSingleVerKvEntry::SerializeData(Parcel &parcel, uint32_t targetVersio int GenericSingleVerKvEntry::SerializeDatas(const std::vector &kvEntries, Parcel &parcel, uint32_t targetVersion) { - LOGD("GenericSingleVerKvEntry::SerialDatas targetVersion:%d", targetVersion); uint32_t size = kvEntries.size(); int errCode = parcel.WriteUInt32(size); if (errCode != E_OK) { + LOGE("[SerializeDatas] write entries size failed, errCode=%d.", errCode); return errCode; } parcel.EightByteAlign(); @@ -118,6 +149,7 @@ int GenericSingleVerKvEntry::SerializeDatas(const std::vectorSerializeData(parcel, targetVersion); if (errCode != E_OK) { + LOGE("[SerializeDatas] write kvEntry failed, errCode=%d.", errCode); return errCode; } } @@ -138,7 +170,6 @@ uint32_t GenericSingleVerKvEntry::CalculateLen(uint32_t targetVersion) uint32_t GenericSingleVerKvEntry::CalculateLens(const std::vector &kvEntries, uint32_t targetVersion) { - LOGD("GenericSingleVerKvEntry::CalculateLen targetVersion:%d", targetVersion); uint64_t len = 0; len += Parcel::GetUInt32Len(); len = BYTE_8_ALIGN(len); @@ -256,14 +287,20 @@ int GenericSingleVerKvEntry::SerializeDataByFirstVersion(Parcel &parcel) const return parcel.WriteString(dataItem_.origDev); } -int GenericSingleVerKvEntry::SerializeDataByLaterVersion(Parcel &parcel) const +int GenericSingleVerKvEntry::SerializeDataByLaterVersion(Parcel &parcel, uint32_t targetVersion) const { TimeStamp writeTimeStamp = dataItem_.writeTimeStamp; if (writeTimeStamp == 0) { writeTimeStamp = dataItem_.timeStamp; } - - return parcel.WriteUInt64(writeTimeStamp); + int errCode = parcel.WriteUInt64(writeTimeStamp); + if (errCode != E_OK) { + return errCode; + } + if (targetVersion >= SOFTWARE_VERSION_RELEASE_6_0) { + errCode = parcel.WriteVector(dataItem_.hashKey); + } + return errCode; } int GenericSingleVerKvEntry::SerializeDataByVersion(uint32_t targetVersion, Parcel &parcel) const @@ -272,7 +309,7 @@ int GenericSingleVerKvEntry::SerializeDataByVersion(uint32_t targetVersion, Parc if (targetVersion == SOFTWARE_VERSION_EARLIEST || errCode != E_OK) { return errCode; } - return SerializeDataByLaterVersion(parcel); + return SerializeDataByLaterVersion(parcel, targetVersion); } void GenericSingleVerKvEntry::CalLenByFirstVersion(uint64_t &len) const @@ -285,9 +322,12 @@ void GenericSingleVerKvEntry::CalLenByFirstVersion(uint64_t &len) const len += Parcel::GetStringLen(dataItem_.origDev); } -void GenericSingleVerKvEntry::CalLenByLaterVersion(uint64_t &len) const +void GenericSingleVerKvEntry::CalLenByLaterVersion(uint64_t &len, uint32_t targetVersion) const { len += Parcel::GetUInt64Len(); + if (targetVersion >= SOFTWARE_VERSION_RELEASE_6_0) { + len += Parcel::GetVectorLen(dataItem_.hashKey); + } } int GenericSingleVerKvEntry::CalLenByVersion(uint32_t targetVersion, uint64_t &len) const @@ -296,7 +336,7 @@ int GenericSingleVerKvEntry::CalLenByVersion(uint32_t targetVersion, uint64_t &l if (targetVersion == SOFTWARE_VERSION_EARLIEST) { return E_OK; } - CalLenByLaterVersion(len); + CalLenByLaterVersion(len, targetVersion); return E_OK; } @@ -310,9 +350,12 @@ void GenericSingleVerKvEntry::DeSerializeByFirstVersion(uint64_t &len, Parcel &p dataItem_.writeTimeStamp = dataItem_.timeStamp; } -void GenericSingleVerKvEntry::DeSerializeByLaterVersion(uint64_t &len, Parcel &parcel) +void GenericSingleVerKvEntry::DeSerializeByLaterVersion(uint64_t &len, Parcel &parcel, uint32_t targetVersion) { len += parcel.ReadUInt64(dataItem_.writeTimeStamp); + if (targetVersion >= SOFTWARE_VERSION_RELEASE_6_0) { + len += parcel.ReadVector(dataItem_.hashKey); + } } int GenericSingleVerKvEntry::DeSerializeByVersion(uint32_t targetVersion, Parcel &parcel, uint64_t &len) @@ -321,7 +364,106 @@ int GenericSingleVerKvEntry::DeSerializeByVersion(uint32_t targetVersion, Parcel if (targetVersion == SOFTWARE_VERSION_EARLIEST) { return E_OK; } - DeSerializeByLaterVersion(len, parcel); + DeSerializeByLaterVersion(len, parcel, targetVersion); return E_OK; } + +uint32_t GenericSingleVerKvEntry::CalculateCompressedLens(const std::vector &compressedData) +{ + // No compressed data in sync. + if (compressedData.empty()) { + return 0; + } + + // Calculate compressed data length. + uint64_t len = 0; + len += Parcel::GetUInt32Len(); // srcLen. + len += Parcel::GetUInt32Len(); // compression algorithm type. + len += Parcel::GetVectorLen(compressedData); // compressed data. + return (len > INT32_MAX) ? 0 : len; +} + +int GenericSingleVerKvEntry::Compress(const std::vector &kvEntries, std::vector &destData, + const CompressInfo &compressInfo) +{ + // Calculate length. + auto srcLen = CalculateLens(kvEntries, compressInfo.targetVersion); + if (srcLen == 0) { + LOGE("Over limit size, cannot compress."); + return -E_INVALID_ARGS; + } + + // Serialize data. + std::vector srcData(srcLen, 0); + Parcel parcel(srcData.data(), srcData.size()); + int errCode = SerializeDatas(kvEntries, parcel, compressInfo.targetVersion); + if (errCode != E_OK) { + return errCode; + } + + // Compress data. + auto inst = DataCompression::GetInstance(compressInfo.compressAlgo); + if (inst == nullptr) { + return -E_INVALID_COMPRESS_ALGO; + } + return inst->Compress(srcData, destData); +} + +int GenericSingleVerKvEntry::Uncompress(const std::vector &srcData, std::vector &kvEntries, + unsigned long destLen, CompressAlgorithm algo) +{ + // Uncompress data. + std::vector destData(destLen, 0); + auto inst = DataCompression::GetInstance(algo); + if (inst == nullptr) { + return -E_INVALID_COMPRESS_ALGO; + } + int errCode = inst->Uncompress(srcData, destData, destLen); + if (errCode != E_OK) { + return errCode; + } + + // Deserialize data. + Parcel parcel(destData.data(), destData.size()); + if (DeSerializeDatas(kvEntries, parcel) == 0) { + return -E_PARSE_FAIL; + } + return E_OK; +} + +int GenericSingleVerKvEntry::SerializeCompressedDatas(const std::vector &kvEntries, + const std::vector &compressedEntries, Parcel &parcel, uint32_t targetVersion, CompressAlgorithm algo) +{ + uint32_t srcLen = CalculateLens(kvEntries, targetVersion); + (void)parcel.WriteUInt32(static_cast(algo)); + (void)parcel.WriteUInt32(srcLen); + (void)parcel.WriteVector(compressedEntries); + return parcel.IsError() ? -E_PARSE_FAIL : E_OK; +} + +int GenericSingleVerKvEntry::DeSerializeCompressedDatas(std::vector &kvEntries, Parcel &parcel) +{ + // Get compression algo type. + uint32_t algoType = 0; + (void)parcel.ReadUInt32(algoType); + CompressAlgorithm compressAlgo = CompressAlgorithm::NONE; + int errCode = DataCompression::TransferCompressionAlgo(algoType, compressAlgo); + if (errCode != E_OK) { + return errCode; + } + + // Get buffer length. + uint32_t destLen = 0; + (void)parcel.ReadUInt32(destLen); + + // Get compressed data. + std::vector srcData; + (void)parcel.ReadVector(srcData); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + + // Uncompress data. + return GenericSingleVerKvEntry::Uncompress(srcData, kvEntries, destLen, compressAlgo); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.h b/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.h index 0dce405e35d1ec55519fda9680c26d83adb26616..67c036e17baf91552a9b932f8251723bf8559cc6 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/generic_single_ver_kv_entry.h @@ -24,6 +24,11 @@ #include "single_ver_kv_entry.h" namespace DistributedDB { +struct CompressInfo { + CompressAlgorithm compressAlgo; + uint32_t targetVersion; +}; + class GenericSingleVerKvEntry : public SingleVerKvEntry { public: GenericSingleVerKvEntry(); @@ -43,12 +48,16 @@ public: void SetWriteTimestamp(TimeStamp time) override; void GetKey(Key &key) const; + const Key &GetKey() const override; + + void GetHashKey(Key &key) const; void GetValue(Value &value) const; + const Value &GetValue() const override; uint64_t GetFlag() const override; - void SetEntryData(DataItem &&dateItem); + void SetEntryData(DataItem &&dataItem); int SerializeData(Parcel &parcel, uint32_t targetVersion) override; @@ -60,7 +69,19 @@ public: static int DeSerializeDatas(std::vector &kvEntries, Parcel &parcel); + void SetKey(const Key &key) override; + void SetValue(const Value &value) override; + void SetHashKey(const Key &hashKey) override; + static uint32_t CalculateLens(const std::vector &kvEntries, uint32_t targetVersion); + static uint32_t CalculateCompressedLens(const std::vector &compressedData); + static int Compress(const std::vector &kvEntries, std::vector &destData, + const CompressInfo &compressInfo); + static int Uncompress(const std::vector &srcData, std::vector &kvEntries, + unsigned long destLen, CompressAlgorithm algo); + static int SerializeCompressedDatas(const std::vector &kvEntries, + const std::vector &compressedEntries, Parcel &parcel, uint32_t targetVersion, CompressAlgorithm algo); + static int DeSerializeCompressedDatas(std::vector &kvEntries, Parcel &parcel); private: int AdaptToVersion(int operType, uint32_t targetVersion, Parcel &parcel, uint64_t &datalen); @@ -68,15 +89,15 @@ private: int SerializeDataByVersion(uint32_t targetVersion, Parcel &parcel) const; int SerializeDataByFirstVersion(Parcel &parcel) const; - int SerializeDataByLaterVersion(Parcel &parcel) const; + int SerializeDataByLaterVersion(Parcel &parcel, uint32_t targetVersion) const; int CalLenByVersion(uint32_t targetVersion, uint64_t &len) const; void CalLenByFirstVersion(uint64_t &len) const; - void CalLenByLaterVersion(uint64_t &len) const; + void CalLenByLaterVersion(uint64_t &len, uint32_t targetVersion) const; int DeSerializeByVersion(uint32_t targetVersion, Parcel &parcel, uint64_t &len); void DeSerializeByFirstVersion(uint64_t &len, Parcel &parcel); - void DeSerializeByLaterVersion(uint64_t &len, Parcel &parcel); + void DeSerializeByLaterVersion(uint64_t &len, Parcel &parcel, uint32_t targetVersion); DataItem dataItem_; }; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h new file mode 100644 index 0000000000000000000000000000000000000000..2d8cf45e0f67e7d606522b3f9efb210e5d315e04 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/irelational_store.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef I_RELATIONAL_STORE_H +#define I_RELATIONAL_STORE_H +#ifdef RELATIONAL_STORE + +#include +#include + +#include "ref_object.h" +#include "kvdb_properties.h" +#include "relational_store_connection.h" + +namespace DistributedDB { +class IRelationalStore : public virtual RefObject { +public: + IRelationalStore() = default; + ~IRelationalStore() override = default; + DISABLE_COPY_ASSIGN_MOVE(IRelationalStore); + + // Open the database. + virtual int Open(const DBProperties &kvDBProp) = 0; + + virtual void WakeUpSyncer() = 0; + + // Create a db connection. + virtual RelationalStoreConnection *GetDBConnection(int &errCode) = 0; +}; +} // namespace DistributedDB + +#endif +#endif // I_RELATIONAL_STORE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_commit_notify_filterable_data.h b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_commit_notify_filterable_data.h index d53d06b781c31be09e4521149573078cb619d40a..4db1c8a8d47953ec1c8fa13f6c943aa5f8585b6f 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_commit_notify_filterable_data.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_commit_notify_filterable_data.h @@ -65,4 +65,4 @@ private: }; } // namespace DistributedDB -#endif // KVDB_COMMIT_NOTIFY_FILTERABLE_DATA_H \ No newline at end of file +#endif // KVDB_COMMIT_NOTIFY_FILTERABLE_DATA_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_manager.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_manager.cpp index 47fa951eb8866a8874df40dd718b2be52ed9bc88..8d8d560741bc49127ce09c8f7d7807ec2259fd4a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_manager.cpp @@ -14,7 +14,6 @@ */ #include "kvdb_manager.h" -#include "platform_specific.h" #include "log_print.h" #include "db_common.h" #include "runtime_context.h" @@ -22,17 +21,19 @@ #include "default_factory.h" #include "generic_kvdb.h" #include "db_constant.h" +#include "res_finalizer.h" namespace DistributedDB { const std::string KvDBManager::PROCESS_LABEL_CONNECTOR = "-"; -KvDBManager *KvDBManager::instance_ = nullptr; +std::atomic KvDBManager::instance_{nullptr}; std::mutex KvDBManager::kvDBLock_; std::mutex KvDBManager::instanceLock_; +std::map KvDBManager::locks_; namespace { DefaultFactory g_defaultFactory; - int CreateDataBase(const KvDBProperties &property, IKvDB *&kvDB) + int CreateDataBaseInstance(const KvDBProperties &property, IKvDB *&kvDB) { IKvDBFactory *factory = IKvDBFactory::GetCurrent(); if (factory == nullptr) { @@ -66,38 +67,41 @@ namespace { int errCode = OS::CreateFileByFileName(storeDir); if (errCode != E_OK) { LOGE("Create remove state flag file failed:%d.", errCode); - return errCode; } return errCode; } +} - int CheckRemoveStateAndRetry(const KvDBProperties &property) - { - std::string dataDir = property.GetStringProp(KvDBProperties::DATA_DIR, ""); - std::string identifier = property.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); - std::string identifierName = DBCommon::TransferStringToHex(identifier); - std::string storeDir = dataDir + "/" + identifierName + DBConstant::DELETE_KVSTORE_REMOVING; +int KvDBManager::CheckRemoveStateAndRetry(const KvDBProperties &property) +{ + std::string dataDir = property.GetStringProp(KvDBProperties::DATA_DIR, ""); + std::string identifier = property.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + std::string identifierName = DBCommon::TransferStringToHex(identifier); + std::string storeDir = dataDir + "/" + identifierName + DBConstant::DELETE_KVSTORE_REMOVING; - if (OS::CheckPathExistence(storeDir)) { - KvDBManager::RemoveDatabase(property); - } - // Re-detection deleted had been finish - if (OS::CheckPathExistence(storeDir)) { - LOGD("Deletekvstore unfinished, can not create new same identifier kvstore!"); - return -E_REMOVE_FILE; - } - return E_OK; + if (OS::CheckPathExistence(storeDir)) { + KvDBManager::ExecuteRemoveDatabase(property); + } + // Re-detection deleted had been finish + if (OS::CheckPathExistence(storeDir)) { + LOGD("Deletekvstore unfinished, can not create new same identifier kvstore!"); + return -E_REMOVE_FILE; } + return E_OK; } int KvDBManager::ExecuteRemoveDatabase(const KvDBProperties &properties) { + int errCode = CheckDatabaseFileStatus(properties); + if (errCode != E_OK) { + return errCode; + } IKvDBFactory *factory = IKvDBFactory::GetCurrent(); if (factory == nullptr) { return -E_INVALID_DB; } - int errCode = CreateRemoveStateFlagFile(properties); + errCode = CreateRemoveStateFlagFile(properties); if (errCode != E_OK) { LOGE("create ctrl file failed:%d.", errCode); return errCode; @@ -173,6 +177,75 @@ void KvDBManager::ExitDBOpenCloseProcess(const std::string &identifier) kvDBOpenCondition_.notify_all(); } +// one time 100ms +// In order to prevent long-term blocking of the process, a retry method is used +// The dimensions of the lock by appid-userid-storeid +int KvDBManager::TryLockDB(const KvDBProperties &kvDBProp, int retryTimes) +{ + std::string dataDir = kvDBProp.GetStringProp(KvDBProperties::DATA_DIR, ""); + bool isMemoryDb = kvDBProp.GetBoolProp(KvDBProperties::MEMORY_MODE, false); + std::string id = KvDBManager::GenerateKvDBIdentifier(kvDBProp); + if (dataDir.back() != '/') { + dataDir += "/"; + } + + if (isMemoryDb) { + LOGI("MemoryDb not need lock!"); + return E_OK; + } + + if (locks_.count(id) != 0) { + LOGI("db has been locked!"); + return E_OK; + } + + std::string hexHashId = DBCommon::TransferStringToHex((id)); + OS::FileHandle handle; + int errCode = OS::OpenFile(dataDir + hexHashId + DBConstant::DB_LOCK_POSTFIX, handle); + if (errCode != E_OK) { + LOGE("Open lock file fail errCode = [%d], errno:%d", errCode, errno); + return errCode; + } + + while (retryTimes-- > 0) { + errCode = OS::FileLock(handle, false); // not block process + if (errCode == E_OK) { + LOGI("[%s]locked!", STR_MASK(DBCommon::TransferStringToHex(KvDBManager::GenerateKvDBIdentifier(kvDBProp)))); + locks_[id] = handle; + return errCode; + } else if (errCode == -E_BUSY) { + LOGD("DB already held by process lock!"); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait for 100ms + continue; + } else { + LOGE("Try lock db failed, errCode = [%d] errno:%d", errCode, errno); + OS::CloseFile(handle); + return errCode; + } + } + OS::CloseFile(handle); + return -E_BUSY; +} + +int KvDBManager::UnlockDB(const KvDBProperties &kvDBProp) +{ + bool isMemoryDb = kvDBProp.GetBoolProp(KvDBProperties::MEMORY_MODE, false); + if (isMemoryDb) { + return E_OK; + } + std::string identifierDir = KvDBManager::GenerateKvDBIdentifier(kvDBProp); + if (locks_.count(identifierDir) == 0) { + return E_OK; + } + int errCode = OS::FileUnlock(locks_[identifierDir]); + LOGI("DB unlocked! errCode = [%d]", errCode); + if (errCode != E_OK) { + return errCode; + } + locks_.erase(identifierDir); + return E_OK; +} + // Used to open a kvdb with the given property IKvDBConnection *KvDBManager::GetDatabaseConnection(const KvDBProperties &properties, int &errCode, bool isNeedIfOpened) @@ -184,7 +257,9 @@ IKvDBConnection *KvDBManager::GetDatabaseConnection(const KvDBProperties &proper } IKvDBConnection *connection = nullptr; std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + LOGD("Begin to get [%s] database connection.", STR_MASK(DBCommon::TransferStringToHex(identifier))); manager->EnterDBOpenCloseProcess(identifier); + IKvDB *kvDB = manager->GetDataBase(properties, errCode, isNeedIfOpened); if (kvDB == nullptr) { if (isNeedIfOpened) { @@ -192,19 +267,20 @@ IKvDBConnection *KvDBManager::GetDatabaseConnection(const KvDBProperties &proper } } else { connection = kvDB->GetDBConnection(errCode); - if (connection == nullptr) { + if (connection == nullptr) { // not kill kvdb, Other operations like import may be used concurrently LOGE("Failed to get the db connect for delegate:%d", errCode); } RefObject::DecObjRef(kvDB); // restore the reference increased by the cache. kvDB = nullptr; } + manager->ExitDBOpenCloseProcess(identifier); if (errCode == -E_INVALID_PASSWD_OR_CORRUPTED_DB) { std::string appId = properties.GetStringProp(KvDBProperties::APP_ID, ""); std::string userId = properties.GetStringProp(KvDBProperties::USER_ID, ""); std::string storeId = properties.GetStringProp(KvDBProperties::STORE_ID, ""); manager->DataBaseCorruptNotify(appId, userId, storeId); - LOGE("Database is corrupted:%d", errCode); + LOGE("Database [%s] is corrupted:%d", STR_MASK(DBCommon::TransferStringToHex(identifier)), errCode); } return connection; @@ -228,9 +304,41 @@ int KvDBManager::ReleaseDatabaseConnection(IKvDBConnection *connection) if (errCode != E_OK) { LOGE("[KvDBManager] Release db connection:%d", errCode); } + LOGI("[Connection] db[%s] conn Close", STR_MASK(DBCommon::TransferStringToHex(identifier))); return errCode; } +IKvDB *KvDBManager::CreateDataBase(const KvDBProperties &property, int &errCode) +{ + IKvDB *kvDB = OpenNewDatabase(property, errCode); + if (kvDB == nullptr) { + LOGE("Failed to open the new database."); + if (errCode == -E_INVALID_PASSWD_OR_CORRUPTED_DB && + property.GetBoolProp(KvDBProperties::RM_CORRUPTED_DB, false)) { + LOGI("Remove the corrupted database while open"); + ExecuteRemoveDatabase(property); + kvDB = OpenNewDatabase(property, errCode); + } + return kvDB; + } + + if (property.GetBoolProp(KvDBProperties::CHECK_INTEGRITY, false)) { + int integrityStatus = kvDB->CheckIntegrity(); + if (integrityStatus == -E_INVALID_PASSWD_OR_CORRUPTED_DB) { + RemoveKvDBFromCache(kvDB); + RefObject::KillAndDecObjRef(kvDB); + kvDB = nullptr; + errCode = -E_INVALID_PASSWD_OR_CORRUPTED_DB; + if (property.GetBoolProp(KvDBProperties::RM_CORRUPTED_DB, false)) { + LOGI("Remove the corrupted database for the integrity check"); + ExecuteRemoveDatabase(property); + kvDB = OpenNewDatabase(property, errCode); + } + } + } + return kvDB; +} + IKvDB *KvDBManager::GetDataBase(const KvDBProperties &property, int &errCode, bool isNeedIfOpened) { bool isMemoryDb = property.GetBoolProp(KvDBProperties::MEMORY_MODE, false); @@ -243,16 +351,41 @@ IKvDB *KvDBManager::GetDataBase(const KvDBProperties &property, int &errCode, bo errCode = -E_ALREADY_OPENED; kvDB = nullptr; } - } else { - if (isMemoryDb && !isCreateNecessary) { - LOGI("IsCreateNecessary is false, Not need create database"); - } else if (errCode == -E_NOT_FOUND) { - kvDB = OpenNewDatabase(property, errCode); - if (kvDB == nullptr) { - LOGE("Failed to open the new database."); - } + return kvDB; + } + if (isMemoryDb && !isCreateNecessary) { + LOGI("IsCreateNecessary is false, Not need create database"); + return nullptr; + } + if (errCode != -E_NOT_FOUND) { + return nullptr; + } + + // Taking into account the compatibility of version delivery, + // temporarily use isNeedIntegrityCheck this field to avoid multi-process concurrency + bool isNeedIntegrityCheck = property.GetBoolProp(KvDBProperties::CHECK_INTEGRITY, false); + if (isNeedIntegrityCheck) { + LOGI("db need lock, need check integrity is [%d]", isNeedIntegrityCheck); + errCode = KvDBManager::TryLockDB(property, 10); // default 10 times retry + if (errCode != E_OK) { + return nullptr; } } + + ResFinalizer unlock([&errCode, &property, &kvDB]() { + int err = KvDBManager::UnlockDB(property); + if (err != E_OK) { + LOGE("GetDataBase unlock failed! err [%d] errCode [%d]", err, errCode); + errCode = err; + RefObject::KillAndDecObjRef(kvDB); + kvDB = nullptr; + } + }); + + kvDB = CreateDataBase(property, errCode); + if (errCode != E_OK) { + LOGE("Create data base failed, errCode = [%d]", errCode); + } return kvDB; } @@ -367,7 +500,7 @@ IKvDB *KvDBManager::OpenNewDatabase(const KvDBProperties &property, int &errCode } IKvDB *kvDB = nullptr; - errCode = CreateDataBase(property, kvDB); + errCode = CreateDataBaseInstance(property, kvDB); if (errCode != E_OK) { LOGE("Failed to get IKvDB! err:%d", errCode); return nullptr; @@ -385,6 +518,7 @@ IKvDB *KvDBManager::OpenNewDatabase(const KvDBProperties &property, int &errCode LOGI("Database identifier:%.6s, dir:%.6s", identifier.c_str(), dbDir.c_str()); // Register the callback function when the database is closed, triggered when kvdb free kvDB->OnClose([kvDB, this]() { + LOGI("Remove from the cache"); this->RemoveKvDBFromCache(kvDB); }); @@ -401,15 +535,37 @@ IKvDB *KvDBManager::OpenNewDatabase(const KvDBProperties &property, int &errCode // return BUSY if in use int KvDBManager::RemoveDatabase(const KvDBProperties &properties) { - int errCode = CheckDatabaseFileStatus(properties); - if (errCode != E_OK) { - return errCode; + KvDBManager *manager = GetInstance(); + if (manager == nullptr) { + LOGE("Failed to get kvdb manager while removing the db!"); + return -E_OUT_OF_MEMORY; + } + std::string identifier = GenerateKvDBIdentifier(properties); + manager->EnterDBOpenCloseProcess(identifier); + + LOGI("KvDBManager::RemoveDatabase begin try lock the database!"); + std::string lockFile = properties.GetStringProp(KvDBProperties::DATA_DIR, "") + "/" + + DBCommon::TransferStringToHex(identifier) + DBConstant::DB_LOCK_POSTFIX; + int errCode = E_OK; + if (OS::CheckPathExistence(lockFile)) { + errCode = KvDBManager::TryLockDB(properties, 10); // default 10 times retry + if (errCode != E_OK) { + manager->ExitDBOpenCloseProcess(identifier); + return errCode; + } } errCode = ExecuteRemoveDatabase(properties); if (errCode != E_OK) { LOGE("[KvDBManager] Remove database failed:%d", errCode); } + int err = KvDBManager::UnlockDB(properties); // unlock and delete lock file befor delete dir + if (err != E_OK) { + LOGE("[KvDBManager][RemoveDatabase] UnlockDB failed:%d, errno:%d", err, errno); + errCode = err; + } + + manager->ExitDBOpenCloseProcess(identifier); return errCode; } @@ -420,12 +576,15 @@ std::string KvDBManager::GenerateKvDBIdentifier(const KvDBProperties &property) KvDBManager *KvDBManager::GetInstance() { - std::lock_guard lockGuard(instanceLock_); + // For Double-Checked Locking, we need check instance_ twice if (instance_ == nullptr) { - instance_ = new (std::nothrow) KvDBManager(); + std::lock_guard lockGuard(instanceLock_); if (instance_ == nullptr) { - LOGE("failed to new KvDBManager!"); - return nullptr; + instance_ = new (std::nothrow) KvDBManager(); + if (instance_ == nullptr) { + LOGE("failed to new KvDBManager!"); + return nullptr; + } } } if (IKvDBFactory::GetCurrent() == nullptr) { @@ -551,7 +710,8 @@ IKvDB *KvDBManager::FindKvDBFromCache(const KvDBProperties &properties, const st return kvDB; } else { errCode = -E_INVALID_ARGS; - LOGE("Database type not matched!"); + LOGE("Database [%s] type not matched, type [%d] vs [%d]", + STR_MASK(DBCommon::TransferStringToHex(identifier)), newType, oldType); return nullptr; } } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp index d13027d839d99d9168afa5e7cf1237f070d030fa..ee95611ea9f25a4c0e7e85b499b9a7321a324109 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/kvdb_properties.cpp @@ -34,6 +34,10 @@ const std::string KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY = "createDirByStor const std::string KvDBProperties::SECURITY_LABEL = "securityLabel"; const std::string KvDBProperties::SECURITY_FLAG = "securityFlag"; const std::string KvDBProperties::CONFLICT_RESOLVE_POLICY = "conflictResolvePolicy"; +const std::string KvDBProperties::CHECK_INTEGRITY = "checkIntegrity"; +const std::string KvDBProperties::RM_CORRUPTED_DB = "rmCorruptedDb"; +const std::string KvDBProperties::COMPRESS_ON_SYNC = "needCompressOnSync"; +const std::string KvDBProperties::COMPRESSION_RATE = "compressionRate"; KvDBProperties::KvDBProperties() : cipherType_(CipherType::AES_256_GCM) @@ -55,7 +59,7 @@ std::string KvDBProperties::GetStoreSubDirectory(int type) } } -std::string KvDBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const +std::string DBProperties::GetStringProp(const std::string &name, const std::string &defaultValue) const { auto iter = stringProperties_.find(name); if (iter != stringProperties_.end()) { @@ -65,12 +69,12 @@ std::string KvDBProperties::GetStringProp(const std::string &name, const std::st } } -void KvDBProperties::SetStringProp(const std::string &name, const std::string &value) +void DBProperties::SetStringProp(const std::string &name, const std::string &value) { stringProperties_[name] = value; } -bool KvDBProperties::GetBoolProp(const std::string &name, bool defaultValue) const +bool DBProperties::GetBoolProp(const std::string &name, bool defaultValue) const { auto iter = boolProperties_.find(name); if (iter != boolProperties_.end()) { @@ -80,12 +84,12 @@ bool KvDBProperties::GetBoolProp(const std::string &name, bool defaultValue) con } } -void KvDBProperties::SetBoolProp(const std::string &name, bool value) +void DBProperties::SetBoolProp(const std::string &name, bool value) { boolProperties_[name] = value; } -int KvDBProperties::GetIntProp(const std::string &name, int defaultValue) const +int DBProperties::GetIntProp(const std::string &name, int defaultValue) const { auto iter = intProperties_.find(name); if (iter != intProperties_.end()) { @@ -95,7 +99,7 @@ int KvDBProperties::GetIntProp(const std::string &name, int defaultValue) const } } -void KvDBProperties::SetIntProp(const std::string &name, int value) +void DBProperties::SetIntProp(const std::string &name, int value) { intProperties_[name] = value; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.cpp index fc2b15138b2b33296bc9f725b800bf8ee8dc588e..49e082efb1cb031d72b76f79861e71cdf2b54225 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.cpp @@ -545,6 +545,11 @@ int MultiVerNaturalStore::PutMetaData(const Key &key, const Value &value) return errCode; } +int MultiVerNaturalStore::DeleteMetaData(const std::vector &keys) +{ + return -E_NOT_SUPPORT; +} + // Get all meta data keys. int MultiVerNaturalStore::GetAllMetaKeys(std::vector &keys) const { @@ -1177,6 +1182,16 @@ int MultiVerNaturalStore::GetVersionFilePath(const KvDBProperties &kvDBProp, std return E_OK; } +int MultiVerNaturalStore::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + return -E_NOT_SUPPORT; +} + +void MultiVerNaturalStore::SetDataInterceptor(const PushDataInterceptor &interceptor) +{ + return; +} + DEFINE_OBJECT_TAG_FACILITIES(MultiVerNaturalStore) } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.h index 72594171d826d118ea2e8ac055c144b42ed5ec01..961c159b273f819b3b227abc3faab0eadad0924d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/multiver/multi_ver_natural_store.h @@ -74,6 +74,11 @@ public: // Put meta data as a key-value entry. int PutMetaData(const Key &key, const Value &value) override; + // Delete multiple meta data records in a transaction. + int DeleteMetaData(const std::vector &keys) override; + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + // Get all meta data keys. int GetAllMetaKeys(std::vector &keys) const override; @@ -151,6 +156,8 @@ public: int InitStorages(const KvDBProperties &kvDBProp, bool isChangeTag = false); + void SetDataInterceptor(const PushDataInterceptor &interceptor) override; + private: int CheckSubStorageVersion(const KvDBProperties &kvDBProp, bool &isSubStorageAllExist) const; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/operation/database_oper.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/operation/database_oper.cpp index ffd876fbc7bd59df4e91953dd7ae51f9e4c109e7..4c6a0fc53d2629e73f16053e648e4fc064616d34 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/operation/database_oper.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/operation/database_oper.cpp @@ -34,7 +34,7 @@ int DatabaseOper::ExecuteRekey(const CipherPassword &passwd, const KvDBPropertie { int errCode = E_OK; if (!RekeyPreHandle(passwd, errCode)) { - LOGE("Rekey fail when RekeyPre Handle, errCode = [%d]", errCode); + LOGI("Finish rekey when RekeyPre Handle, errCode = [%d]", errCode); return errCode; } @@ -366,7 +366,7 @@ int DatabaseOper::RemoveFile(const std::string &fileName) return E_OK; } - if (remove(fileName.c_str()) != 0) { + if (OS::RemoveFile(fileName.c_str()) != E_OK) { LOGE("Remove file failed:%d", errno); return -E_REMOVE_FILE; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/operation/single_ver_database_oper.h b/services/distributeddataservice/libs/distributeddb/storage/src/operation/single_ver_database_oper.h index 946409d7aef04c2d3d020c384df08a3b95a62236..b0ea78704d1198f4ff9853c1d6212f80c1856c5f 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/operation/single_ver_database_oper.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/operation/single_ver_database_oper.h @@ -68,7 +68,7 @@ private: int ImportUnpackedMetaDatabase(const ImportFileInfo &info) const; - int SetSecOpt(const std::string &dir, bool isDir = true) const; + int SetSecOpt(const std::string &path, bool isDir = true) const; int BackupDatabase(const ImportFileInfo &info) const; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/package_file.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/package_file.cpp index cc1625c6b5cbd2c3eeaba6ca7c16f1fcd5059d29..85eee2744f5fecc4ac2df84866dc1b962bf7ed9d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/package_file.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/package_file.cpp @@ -60,7 +60,7 @@ static void Clear(ofstream &target, string targetFile) if (target.is_open()) { target.close(); } - if (remove(targetFile.c_str()) != EOK) { + if (OS::RemoveFile(targetFile.c_str()) != E_OK) { LOGE("Remove file failed."); } return; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/sync_types.h b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp old mode 100755 new mode 100644 similarity index 64% rename from services/distributeddataservice/libs/distributeddb/syncer/include/sync_types.h rename to services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp index 94ada2dea2d58d0d16506d17915dcbe6bb1ee1c6..4fc241f370cb7f2491f2662f268878c0b745df8b --- a/services/distributeddataservice/libs/distributeddb/syncer/include/sync_types.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_connection.cpp @@ -12,23 +12,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#ifndef SYNC_TYPES_H -#define SYNC_TYPES_H +#ifdef RELATIONAL_STORE +#include "relational_store_connection.h" +#include "db_errno.h" +#include "sqlite_single_ver_relational_storage_executor.h" namespace DistributedDB { -enum MessageId { - TIME_SYNC_MESSAGE = 1, - DATA_SYNC_MESSAGE, - COMMIT_HISTORY_SYNC_MESSAGE, - MULTI_VER_DATA_SYNC_MESSAGE, - VALUE_SLICE_SYNC_MESSAGE, - LOCAL_DATA_CHANGED, - ABILITY_SYNC_MESSAGE, -}; - -const static uint32_t SEND_TIME_OUT = 3000; // 3s -const int NOT_SURPPORT_SEC_CLASSIFICATION = 0xff; +int RelationalStoreConnection::Pragma(int cmd, void *parameter) +{ + return E_OK; +} } - #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e48bc8230e24d145e283748d8e9e40b23f8c1f9c --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_store_instance.cpp @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "relational_store_instance.h" + +#include +#include + +#include "sqlite_relational_store.h" +#include "db_errno.h" +#include "log_print.h" + +namespace DistributedDB { +RelationalStoreInstance *RelationalStoreInstance::instance_ = nullptr; +std::mutex RelationalStoreInstance::instanceLock_; + +static std::mutex storeLock_; +static std::map dbs_; + +RelationalStoreInstance::RelationalStoreInstance() +{} + +RelationalStoreInstance *RelationalStoreInstance::GetInstance() +{ + std::lock_guard lockGuard(instanceLock_); + if (instance_ == nullptr) { + instance_ = new (std::nothrow) RelationalStoreInstance(); + if (instance_ == nullptr) { + LOGE("failed to new RelationalStoreManager!"); + return nullptr; + } + } + return instance_; +} + +int RelationalStoreInstance::CheckDatabaseFileStatus(const std::string &id) +{ + std::lock_guard lockGuard(storeLock_); + if (dbs_.count(id) != 0 && dbs_[id] != nullptr) { + return -E_BUSY; + } + return E_OK; +} + +static IRelationalStore *GetFromCache(const DBProperties &properties, int &errCode) +{ + errCode = E_OK; + std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + auto iter = dbs_.find(identifier); + if (iter == dbs_.end()) { + errCode = -E_NOT_FOUND; + return nullptr; + } + + auto *db = iter->second; + + if (db == nullptr) { + LOGE("Store cache is nullptr, there may be a logic error"); + errCode = -E_INTERNAL_ERROR; + return nullptr; + } + db->IncObjRef(db); + return db; +} + +// Save to IKvDB to the global map +void RelationalStoreInstance::RemoveKvDBFromCache(const DBProperties &properties) +{ + std::lock_guard lockGuard(storeLock_); + std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + dbs_.erase(identifier); +} + +void RelationalStoreInstance::SaveKvDBToCache(IRelationalStore *store, const DBProperties &properties) +{ + if (store == nullptr) { + return; + } + + { + std::string identifier = properties.GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); + store->WakeUpSyncer(); + if (dbs_.count(identifier) == 0) { + dbs_.insert(std::pair(identifier, store)); + } + } +} + +IRelationalStore *RelationalStoreInstance::OpenDatabase(const DBProperties &properties, int &errCode) +{ + auto db = new (std::nothrow) SQLiteRelationalStore(); + if (db == nullptr) { + LOGE("Failed to get IKvDB! err:%d", errCode); + return nullptr; + } + + db->OnClose([this, properties]() { + LOGI("Remove from the cache"); + this->RemoveKvDBFromCache(properties); + }); + + errCode = db->Open(properties); + if (errCode != E_OK) { + LOGE("Failed to open db! err:%d", errCode); + RefObject::KillAndDecObjRef(db); + return nullptr; + } + + SaveKvDBToCache(db, properties); + return db; +} + +IRelationalStore *RelationalStoreInstance::GetDataBase(const DBProperties &properties, int &errCode) +{ + std::lock_guard lockGuard(storeLock_); + auto *db = GetFromCache(properties, errCode); + if (db != nullptr) { + LOGD("Get db from cache."); + return db; + } + + // file lock + RelationalStoreInstance *manager = RelationalStoreInstance::GetInstance(); + if (manager == nullptr) { + errCode = -E_OUT_OF_MEMORY; + return nullptr; + } + + db = manager->OpenDatabase(properties, errCode); + if (errCode != E_OK) { + LOGE("Create data base failed, errCode = [%d]", errCode); + } + return db; +} + +RelationalStoreConnection *RelationalStoreInstance::GetDatabaseConnection(const DBProperties &properties, int &errCode) +{ + IRelationalStore *db = GetDataBase(properties, errCode); + if (db == nullptr) { + LOGE("Failed to open the db:%d", errCode); + return nullptr; + } + + auto connection = db->GetDBConnection(errCode); + if (connection == nullptr) { // not kill db, Other operations like import may be used concurrently + LOGE("Failed to get the db connect for delegate:%d", errCode); + } + RefObject::DecObjRef(db); // restore the reference increased by the cache. + // kvDB = nullptr; + + return connection; +} +} // namespace DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c8082bad6ac946c1421d1f5bd24f224a3bcc3ec2 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/relational_sync_able_storage.cpp @@ -0,0 +1,503 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "relational_sync_able_storage.h" +#include "platform_specific.h" +#include "generic_single_ver_kv_entry.h" + +namespace DistributedDB { +#define CHECK_STORAGE_ENGINE do { \ + if (storageEngine_ == nullptr) { \ + return -E_INVALID_DB; \ + } \ +} while (0) + +RelationalSyncAbleStorage::RelationalSyncAbleStorage(StorageEngine *engine) + : storageEngine_(static_cast(engine)) +{} + +RelationalSyncAbleStorage::~RelationalSyncAbleStorage() +{} + +// Get interface type of this kvdb. +int RelationalSyncAbleStorage::GetInterfaceType() const +{ + return SYNC_RELATION; +} + +// Get the interface ref-count, in order to access asynchronously. +void RelationalSyncAbleStorage::IncRefCount() +{ + LOGD("RelationalSyncAbleStorage ref +1"); + IncObjRef(this); +} + +// Drop the interface ref-count. +void RelationalSyncAbleStorage::DecRefCount() +{ + LOGD("RelationalSyncAbleStorage ref -1"); + DecObjRef(this); +} + +// Get the identifier of this kvdb. +std::vector RelationalSyncAbleStorage::GetIdentifier() const +{ + return {}; +} + +// Get the max timestamp of all entries in database. +void RelationalSyncAbleStorage::GetMaxTimeStamp(TimeStamp &stamp) const +{ + std::lock_guard lock(maxTimeStampMutex_); + stamp = currentMaxTimeStamp_; +} + +int RelationalSyncAbleStorage::SetMaxTimeStamp(TimeStamp timestamp) +{ + std::lock_guard lock(maxTimeStampMutex_); + if (timestamp > currentMaxTimeStamp_) { + currentMaxTimeStamp_ = timestamp; + } + return E_OK; +} + +SQLiteSingleVerRelationalStorageExecutor *RelationalSyncAbleStorage::GetHandle(bool isWrite, int &errCode, + OperatePerm perm) const +{ + if (storageEngine_ == nullptr) { + errCode = -E_INVALID_DB; + return nullptr; + } + return static_cast(storageEngine_->FindExecutor(isWrite, perm, + errCode)); +} + +void RelationalSyncAbleStorage::ReleaseHandle(SQLiteSingleVerRelationalStorageExecutor *&handle) const +{ + if (storageEngine_ == nullptr) { + return; + } + StorageExecutor *databaseHandle = handle; + storageEngine_->Recycle(databaseHandle); +} + +// Get meta data associated with the given key. +int RelationalSyncAbleStorage::GetMetaData(const Key &key, Value &value) const +{ + CHECK_STORAGE_ENGINE; + if (key.size() > DBConstant::MAX_KEY_SIZE) { + return -E_INVALID_ARGS; + } + + int errCode = E_OK; + auto handle = GetHandle(true, errCode, OperatePerm::NORMAL_PERM); + if (handle == nullptr) { + return errCode; + } + errCode = handle->GetKvData(key, value); + ReleaseHandle(handle); + return errCode; +} + +// Put meta data as a key-value entry. +int RelationalSyncAbleStorage::PutMetaData(const Key &key, const Value &value) +{ + CHECK_STORAGE_ENGINE; + int errCode = E_OK; + auto *handle = GetHandle(true, errCode, OperatePerm::NORMAL_PERM); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->PutKvData(key, value); // meta doesn't need time. + if (errCode != E_OK) { + LOGE("Put kv data err:%d", errCode); + } + ReleaseHandle(handle); + return errCode; +} + +// Delete multiple meta data records in a transaction. +int RelationalSyncAbleStorage::DeleteMetaData(const std::vector &keys) +{ + for (const auto &key : keys) { + if (key.empty() || key.size() > DBConstant::MAX_KEY_SIZE) { + return -E_INVALID_ARGS; + } + } + int errCode = E_OK; + auto handle = GetHandle(true, errCode, OperatePerm::NORMAL_PERM); + if (handle == nullptr) { + return errCode; + } + + handle->StartTransaction(TransactType::IMMEDIATE); + errCode = handle->DeleteMetaData(keys); + if (errCode != E_OK) { + handle->Rollback(); + LOGE("[SinStore] DeleteMetaData failed, errCode = %d", errCode); + } else { + handle->Commit(); + } + ReleaseHandle(handle); + return errCode; +} + +// Delete multiple meta data records with key prefix in a transaction. +int RelationalSyncAbleStorage::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + if (keyPrefix.empty() || keyPrefix.size() > DBConstant::MAX_KEY_SIZE) { + return -E_INVALID_ARGS; + } + + int errCode = E_OK; + auto handle = GetHandle(true, errCode, OperatePerm::NORMAL_PERM); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->DeleteMetaDataByPrefixKey(keyPrefix); + if (errCode != E_OK) { + LOGE("[SinStore] DeleteMetaData by prefix key failed, errCode = %d", errCode); + } + ReleaseHandle(handle); + return errCode; +} + +// Get all meta data keys. +int RelationalSyncAbleStorage::GetAllMetaKeys(std::vector &keys) const +{ + CHECK_STORAGE_ENGINE; + int errCode = E_OK; + auto *handle = GetHandle(true, errCode, OperatePerm::NORMAL_PERM); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->GetAllMetaKeys(keys); + ReleaseHandle(handle); + return errCode; +} + +const KvDBProperties &RelationalSyncAbleStorage::GetDbProperties() const +{ + return properties; +} + +static void ReleaseKvEntries(std::vector &entries) +{ + for (auto &itemEntry : entries) { + delete itemEntry; + itemEntry = nullptr; + } + entries.clear(); +} + +static int GetKvEntriesByDataItems(std::vector &entries, std::vector &dataItems) +{ + int errCode = E_OK; + for (auto &item : dataItems) { + auto entry = new (std::nothrow) GenericSingleVerKvEntry(); + if (entry == nullptr) { + errCode = -E_OUT_OF_MEMORY; + LOGE("GetKvEntries failed, errCode:%d", errCode); + ReleaseKvEntries(entries); + break; + } + entry->SetEntryData(std::move(item)); + entries.push_back(entry); + } + return errCode; +} + +static size_t GetDataItemSerialSize(const DataItem &item, size_t appendLen) +{ + // timestamp and local flag: 3 * uint64_t, version(uint32_t), key, value, origin dev and the padding size. + // the size would not be very large. + static const size_t maxOrigDevLength = 40; + size_t devLength = std::max(maxOrigDevLength, item.origDev.size()); + size_t dataSize = (Parcel::GetUInt64Len() * 3 + Parcel::GetUInt32Len() + Parcel::GetVectorCharLen(item.key) + + Parcel::GetVectorCharLen(item.value) + devLength + appendLen); + return dataSize; +} + +static constexpr float QUERY_SYNC_THRESHOLD = 0.50; +static bool CanHoldDeletedData(const std::vector &dataItems, const DataSizeSpecInfo &dataSizeInfo, + size_t appendLen) +{ + bool reachThreshold = false; + for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { + blockSize += GetDataItemSerialSize(dataItems[i], appendLen); + reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); + } + return !reachThreshold; +} + +static void ProcessContinueTokenForQuerySync(const std::vector &dataItems, int &errCode, + SQLiteSingleVerContinueToken *&token) +{ + if (errCode != -E_UNFINISHED) { // Error happened or get data finished. Token should be cleared. + delete token; + token = nullptr; + return; + } + + if (dataItems.empty()) { + errCode = -E_INTERNAL_ERROR; + LOGE("Get data unfinished but data items is empty."); + delete token; + token = nullptr; + return; + } + + TimeStamp nextBeginTime = dataItems.back().timeStamp + 1; + if (nextBeginTime > INT64_MAX) { + nextBeginTime = INT64_MAX; + } + bool getDeleteData = ((dataItems.back().flag & DataItem::DELETE_FLAG) != 0); + if (getDeleteData) { + token->FinishGetQueryData(); + token->SetDeletedNextBeginTime("", nextBeginTime); + } else { + token->SetNextBeginTime("", nextBeginTime); + } +} + +/** + * Caller must ensure that parameter continueStmtToken is valid. + * If error happened, token will be deleted here. + */ +int RelationalSyncAbleStorage::GetSyncDataForQuerySync(std::vector &dataItems, + SQLiteSingleVerContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const +{ + if (storageEngine_ == nullptr) { + return -E_INVALID_DB; + } + + int errCode = E_OK; + auto handle = static_cast(storageEngine_->FindExecutor(false, + OperatePerm::NORMAL_PERM, errCode)); + if (handle == nullptr) { + goto ERROR; + } + + errCode = handle->SetTableInfo(continueStmtToken->GetQuery()); + if (errCode != E_OK) { + goto ERROR; + } + + // Get query data. + if (!continueStmtToken->IsGetQueryDataFinished()) { + LOGD("[SingleVerNStore] Get query data between %llu and %llu.", continueStmtToken->GetQueryBeginTime(), + continueStmtToken->GetQueryEndTime()); + errCode = handle->GetSyncDataByQuery( + dataItems, Parcel::GetAppendedLen(), continueStmtToken->GetQuery(), dataSizeInfo, + // need modify + std::make_pair(continueStmtToken->GetQueryBeginTime(), continueStmtToken->GetQueryEndTime())); + } + + // Get query data finished. + if (errCode == E_OK || errCode == -E_FINISHED) { + // Clear query timeRange. + continueStmtToken->FinishGetQueryData(); + + // Get delete time next. + if (!continueStmtToken->IsGetDeletedDataFinished() && + CanHoldDeletedData(dataItems, dataSizeInfo, Parcel::GetAppendedLen())) { + LOGD("[SingleVerNStore] Get deleted data between %llu and %llu.", continueStmtToken->GetDeletedBeginTime(), + continueStmtToken->GetDeletedEndTime()); + errCode = handle->GetDeletedSyncDataByTimestamp( + dataItems, Parcel::GetAppendedLen(), + // need modify + continueStmtToken->GetDeletedBeginTime(), continueStmtToken->GetDeletedEndTime(), dataSizeInfo); + } + } + + if (errCode == -E_FINISHED) { + errCode = E_OK; + } + +ERROR: + if (errCode != -E_UNFINISHED && errCode != E_OK) { // Error happened. + dataItems.clear(); + } + ProcessContinueTokenForQuerySync(dataItems, errCode, continueStmtToken); + ReleaseHandle(handle); + return errCode; +} + +// use kv struct data to sync +// Get the data which would be synced with query condition +int RelationalSyncAbleStorage::GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const +{ + if (!timeRange.IsValid()) { + return -E_INVALID_ARGS; + } + + auto token = new (std::nothrow) SQLiteSingleVerContinueToken(timeRange, query); // release in sync module + if (token == nullptr) { + LOGE("[SingleVerNStore] Allocate continue token failed."); + return -E_OUT_OF_MEMORY; + } + + int innerCode; + std::vector dataItems; + int errCode = GetSyncDataForQuerySync(dataItems, token, dataSizeInfo); + if (errCode != E_OK && errCode != -E_UNFINISHED) { // The code need be sent to outside except new error happened. + continueStmtToken = static_cast(token); + return errCode; + } + + innerCode = GetKvEntriesByDataItems(entries, dataItems); + if (innerCode != E_OK) { + errCode = innerCode; + delete token; + token = nullptr; + } + continueStmtToken = static_cast(token); + return errCode; +} + +int RelationalSyncAbleStorage::GetSyncDataNext(std::vector &entries, + ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const +{ + return E_OK; +} + +// Release the continue token of getting data. +void RelationalSyncAbleStorage::ReleaseContinueToken(ContinueToken &continueStmtToken) const +{ + return; +} + +int RelationalSyncAbleStorage::PutSyncDataWithQuery(const QueryObject &object, + const std::vector &entries, const DeviceID &deviceName) +{ + std::vector dataItems; + for (auto itemEntry : entries) { + GenericSingleVerKvEntry *entry = static_cast(itemEntry); + if (entry != nullptr) { + DataItem item; + item.origDev = entry->GetOrigDevice(); + item.flag = entry->GetFlag(); + item.timeStamp = entry->GetTimestamp(); + item.writeTimeStamp = entry->GetWriteTimestamp(); + entry->GetKey(item.key); + entry->GetValue(item.value); + entry->GetHashKey(item.hashKey); + dataItems.push_back(item); + } + } + + return PutSyncData(object, dataItems, deviceName); +} + +int RelationalSyncAbleStorage::SaveSyncDataItems(const QueryObject &object, std::vector &dataItems, + const std::string &deviceName) +{ + int errCode = E_OK; + LOGD("[SQLiteSingleVerNaturalStore::SaveSyncData] Get write handle."); + auto *handle = GetHandle(true, errCode, OperatePerm::NORMAL_PERM); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->SetTableInfo(object); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + + TimeStamp maxTimestamp = 0; + errCode = handle->SaveSyncItems(object, dataItems, deviceName, maxTimestamp); + if (errCode == E_OK) { + (void)SetMaxTimeStamp(maxTimestamp); + } + + ReleaseHandle(handle); + return errCode; +} + +int RelationalSyncAbleStorage::PutSyncData(const QueryObject &query, std::vector &dataItems, + const std::string &deviceName) +{ + if (deviceName.length() > DBConstant::MAX_DEV_LENGTH) { + LOGW("Device length is invalid for sync put"); + return -E_INVALID_ARGS; + } + + int errCode = SaveSyncDataItems(query, dataItems, deviceName); // Currently true to check value content + if (errCode != E_OK) { + LOGE("[Relational] PutSyncData errCode:%d", errCode); + } + return errCode; +} + +int RelationalSyncAbleStorage::RemoveDeviceData(const std::string &deviceName, bool isNeedNotify) +{ + return -E_NOT_SUPPORT; +} + +RelationalSchemaObject RelationalSyncAbleStorage::GetSchemaInfo() const +{ + return RelationalSchemaObject(); +} + +int RelationalSyncAbleStorage::GetSecurityOption(SecurityOption &option) const +{ + return -E_NOT_SUPPORT; +} + +void RelationalSyncAbleStorage::NotifyRemotePushFinished(const std::string &deviceId) const +{ + return; +} + +// Get the timestamp when database created or imported +int RelationalSyncAbleStorage::GetDatabaseCreateTimeStamp(TimeStamp &outTime) const +{ + return OS::GetCurrentSysTimeInMicrosecond(outTime); +} + +// Get batch meta data associated with the given key. +int RelationalSyncAbleStorage::GetBatchMetaData(const std::vector &keys, std::vector &entries) const +{ + return -E_NOT_SUPPORT; +} + +// Put batch meta data as a key-value entry vector +int RelationalSyncAbleStorage::PutBatchMetaData(std::vector &entries) +{ + return -E_NOT_SUPPORT; +} + +std::vector RelationalSyncAbleStorage::GetTablesQuery() +{ + return {}; +} + +int RelationalSyncAbleStorage::LocalDataChanged(int notifyEvent, std::vector &queryObj) +{ + return -E_NOT_SUPPORT; +} + +int RelationalSyncAbleStorage::SchemaChanged(int notifyEvent) +{ + return -E_NOT_SUPPORT; +} +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_natural_store_commit_notify_data.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_natural_store_commit_notify_data.cpp index 2aec796fe2c5aec0db5374faca6509cb1f34015a..fd3db2a6c05f29db6dbce7346a20d9162faf36ac 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_natural_store_commit_notify_data.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/single_ver_natural_store_commit_notify_data.cpp @@ -197,7 +197,7 @@ int SingleVerNaturalStoreCommitNotifyData::GetConflictedNotifiedFlag() const } bool SingleVerNaturalStoreCommitNotifyData::IsConflictedNotifyMatched(const DataItem &itemPut, - const DataItem &itemOri) const + const DataItem &itemGet) const { int dataConflictedType = 0; // Local put @@ -205,7 +205,7 @@ bool SingleVerNaturalStoreCommitNotifyData::IsConflictedNotifyMatched(const Data dataConflictedType = SINGLE_VER_CONFLICT_NATIVE_ALL; } else { // Compare the origin device of the get and put item. - if (itemPut.origDev != itemOri.origDev) { + if (itemPut.origDev != itemGet.origDev) { dataConflictedType = SINGLE_VER_CONFLICT_FOREIGN_KEY_ORIG; } else { dataConflictedType = SINGLE_VER_CONFLICT_FOREIGN_KEY_ONLY; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp index bb7aeb6c45a23669bf9ef004c4c80b55407784fc..8c465a1c83b019edcbe106885cde5ad06d10737c 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.cpp @@ -15,204 +15,119 @@ #include "query_object.h" #include "db_errno.h" -#include "schema_utils.h" #include "get_query_info.h" #include "log_print.h" -#include "sqlite_utils.h" -#include "db_constant.h" -#include "macro_utils.h" namespace DistributedDB { namespace { - const std::string PRE_QUERY_KV_SQL = "SELECT key, value FROM sync_data "; - const std::string PRE_QUERY_ROWID_SQL = "SELECT rowid FROM sync_data "; - const std::string PRE_GET_COUNT_SQL = "SELECT count(*) FROM sync_data "; - const std::string FILTER_NATIVE_DATA_SQL = "WHERE (flag&0x01=0) "; - const std::string USING_INDEX = "INDEXED BY "; - const int MAX_SQL_LEN = 1024 * 1024; // 1M bytes - const int LIMIT_FIELD_VALUE_SIZE = 2; - const int SINGLE_FIELD_VALUE_SIZE = 1; - const int INVALID_LIMIT = INT_MAX; - const int MAX_CONDITIONS_SIZE = 128; - const int MAX_SQLITE_BIND_SIZE = 50000; - - enum SymbolType { - INVALID_SYMBOL = -1, - COMPARE_SYMBOL, // relation symbol use to compare - RELATIONAL_SYMBOL, - RANGE_SYMBOL, - LOGIC_SYMBOL, - LINK_SYMBOL, // use to link relatonal symbol - SPECIAL_SYMBOL, // need special precess and need at the last - PREFIXKEY_SYMBOL, - SUGGEST_INDEX_SYMBOL, - }; - - bool IsNeedCheckEqualFormat(SymbolType type) - { - return type == COMPARE_SYMBOL || type == RELATIONAL_SYMBOL || type == RANGE_SYMBOL; - } +const int INVALID_LIMIT = INT_MAX; +const int LIMIT_FIELD_VALUE_SIZE = 2; +bool IsNeedCheckEqualFormat(SymbolType type) +{ + return type == COMPARE_SYMBOL || type == RELATIONAL_SYMBOL || type == RANGE_SYMBOL; +} +} - // Considering our company's style, give up hexadecimal expressions state machines to avoid unreadable - const std::map SYMBOL_TYPE_DIC { - {QueryObjType::EQUALTO, COMPARE_SYMBOL}, - {QueryObjType::NOT_EQUALTO, COMPARE_SYMBOL}, - {QueryObjType::GREATER_THAN, COMPARE_SYMBOL}, - {QueryObjType::LESS_THAN, COMPARE_SYMBOL}, - {QueryObjType::GREATER_THAN_OR_EQUALTO, COMPARE_SYMBOL}, - {QueryObjType::LESS_THAN_OR_EQUALTO, COMPARE_SYMBOL}, - {QueryObjType::LIKE, RELATIONAL_SYMBOL}, - {QueryObjType::NOT_LIKE, RELATIONAL_SYMBOL}, - {QueryObjType::IS_NULL, RELATIONAL_SYMBOL}, - {QueryObjType::IS_NOT_NULL, RELATIONAL_SYMBOL}, - {QueryObjType::IN, RANGE_SYMBOL}, - {QueryObjType::NOT_IN, RANGE_SYMBOL}, - {QueryObjType::BEGIN_GROUP, LOGIC_SYMBOL}, - {QueryObjType::END_GROUP, LOGIC_SYMBOL}, - {QueryObjType::AND, LINK_SYMBOL}, - {QueryObjType::OR, LINK_SYMBOL}, - {QueryObjType::LIMIT, SPECIAL_SYMBOL}, - {QueryObjType::ORDERBY, SPECIAL_SYMBOL}, - {QueryObjType::QUERY_BY_KEY_PREFIX, PREFIXKEY_SYMBOL}, - {QueryObjType::SUGGEST_INDEX, SUGGEST_INDEX_SYMBOL} - }; - - SymbolType GetSymbolType(const QueryObjType &queryObjType) - { - if (SYMBOL_TYPE_DIC.find(queryObjType) == SYMBOL_TYPE_DIC.end()) { - return INVALID_SYMBOL; - } - return SYMBOL_TYPE_DIC.at(queryObjType); - } +QueryObject::QueryObject() + : isValid_(true), + initialized_(false), + limit_(INVALID_LIMIT), + offset_(0), + hasOrderBy_(false), + hasLimit_(false), + hasPrefixKey_(false), + orderByCounts_(0) +{ +} - const std::map RELATIONAL_SYMBOL_TO_SQL { - {QueryObjType::EQUALTO, "= "}, - {QueryObjType::NOT_EQUALTO, "!= "}, - {QueryObjType::GREATER_THAN, "> "}, - {QueryObjType::LESS_THAN, "< "}, - {QueryObjType::GREATER_THAN_OR_EQUALTO, ">= "}, - {QueryObjType::LESS_THAN_OR_EQUALTO, "<= "}, - {QueryObjType::LIKE, " LIKE "}, - {QueryObjType::NOT_LIKE, " NOT LIKE "}, - {QueryObjType::IS_NULL, " IS NULL "}, - {QueryObjType::IS_NOT_NULL, " IS NOT NULL "}, - {QueryObjType::IN, " IN ("}, - {QueryObjType::NOT_IN, " NOT IN ("}, - }; - - const std::map LOGIC_SYMBOL_TO_SQL { - {QueryObjType::AND, " AND "}, - {QueryObjType::OR, " OR "}, - {QueryObjType::BEGIN_GROUP, "("}, - {QueryObjType::END_GROUP, ")"}, - }; - - int CheckLinkerBefor(std::list::iterator &iter) - { - auto preIter = std::prev(iter, 1); - SymbolType symbolType = GetSymbolType(preIter->operFlag); - if (symbolType != COMPARE_SYMBOL && symbolType != RELATIONAL_SYMBOL && symbolType != LOGIC_SYMBOL && - symbolType != RANGE_SYMBOL && symbolType != PREFIXKEY_SYMBOL) { - LOGE("Must be a comparison operation before the connective! operFage = %s", VNAME(preIter->operFlag)); - return -E_INVALID_QUERY_FORMAT; +void QueryObject::GetAttrFromQueryObjNodes() +{ + for (const auto &iter : queryObjNodes_) { + SymbolType symbolType = SqliteQueryHelper::GetSymbolType(iter.operFlag); + if (iter.operFlag == QueryObjType::LIMIT) { + hasLimit_ = true; + if (iter.fieldValue.size() == LIMIT_FIELD_VALUE_SIZE) { + limit_ = iter.fieldValue[0].integerValue; + offset_ = iter.fieldValue[1].integerValue; + } + } else if (iter.operFlag == QueryObjType::ORDERBY) { + hasOrderBy_ = true; + } else if (symbolType == PREFIXKEY_SYMBOL) { + hasPrefixKey_ = true; } - return E_OK; } } -QueryObject::QueryObject() - : limit_(INVALID_LIMIT), +QueryObject::QueryObject(const Query &query) + : initialized_(false), + limit_(INVALID_LIMIT), offset_(0), - orderByCounts_(0), - isValid_(true), - transformed_(false), hasOrderBy_(false), hasLimit_(false), - isOrderByAppeared_(false), hasPrefixKey_(false), - isNeedOrderbyKey_(true) -{} - -QueryObject::QueryObject(const Query &query) - : limit_(INVALID_LIMIT), offset_(0), orderByCounts_(0), transformed_(false), hasOrderBy_(false), - hasLimit_(false), isOrderByAppeared_(false), hasPrefixKey_(false), isNeedOrderbyKey_(true) + orderByCounts_(0) { QueryExpression queryExpressions = GetQueryInfo::GetQueryExpression(query); queryObjNodes_ = queryExpressions.GetQueryExpression(); + GetAttrFromQueryObjNodes(); isValid_ = queryExpressions.GetErrFlag(); prefixKey_ = queryExpressions.GetPreFixKey(); suggestIndex_ = queryExpressions.GetSuggestIndex(); + tableName_ = queryExpressions.GetTableName(); + isTableNameSpecified_ = queryExpressions.IsTableNameSpacified(); } -int QueryObject::GetQuerySql(std::string &sql, bool onlyRowid) +QueryObject::QueryObject(const std::list &queryObjNodes, const std::vector &prefixKey) + : queryObjNodes_(queryObjNodes), + prefixKey_(prefixKey), + isValid_(true), + initialized_(false), + limit_(INVALID_LIMIT), + offset_(0), + hasOrderBy_(false), + hasLimit_(false), + hasPrefixKey_(false), + orderByCounts_(0) { - int errCode = E_OK; - if (!IsValid(errCode)) { - return errCode; - } - - const std::string &querySqlForUse = (onlyRowid ? PRE_QUERY_ROWID_SQL : PRE_QUERY_KV_SQL); - sql = AssembleSqlForSuggestIndex(querySqlForUse); - sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); - if (transformed_) { - LOGD("This query object has been parsed."); - sql += querySql_; - return E_OK; - } - errCode = ToQuerySql(); - if (errCode != E_OK) { - LOGE("To query sql fail! errCode[%d]", errCode); - return errCode; - } - transformed_ = true; - sql += querySql_; - return errCode; + GetAttrFromQueryObjNodes(); } -int QueryObject::GetCountQuerySql(std::string &sql) +QueryObject::~QueryObject() +{} + +int QueryObject::Init() { - int errCode = E_OK; - if (!IsValid(errCode)) { - return errCode; + if (initialized_) { + return E_OK; } - errCode = ToGetCountSql(); + int errCode = Parse(); if (errCode != E_OK) { + LOGE("Parse query object err[%d]!", errCode); return errCode; } - sql = AssembleSqlForSuggestIndex(PRE_GET_COUNT_SQL); - sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); - sql += countSql_; - return E_OK; -} -void QueryObject::SetSchema(const SchemaObject &schema) -{ - schema_ = schema; + initialized_ = true; + return errCode; } -bool QueryObject::IsValid(int &errCode) +SqliteQueryHelper QueryObject::GetQueryHelper(int &errCode) { - if (!isValid_) { - errCode = -E_INVALID_QUERY_FORMAT; - LOGE("Invalid query object!"); - return isValid_; - } - errCode = CheckQueryLegality(); + errCode = Init(); if (errCode != E_OK) { - LOGE("Check query object illegal!"); - isValid_ = false; + return SqliteQueryHelper(QueryObjInfo{}); } - return isValid_; + QueryObjInfo info {schema_, queryObjNodes_, prefixKey_, suggestIndex_, + orderByCounts_, isValid_, hasOrderBy_, hasLimit_, hasPrefixKey_, tableName_}; + return SqliteQueryHelper {info}; // compiler RVO by default, and RVO is generally required after C++17 } -bool QueryObject::IsCountValid() const +bool QueryObject::IsValid() { - if (hasLimit_ || hasOrderBy_) { - LOGI("It is invalid for limit and orderby!"); - return false; + if (!initialized_) { + (void)Init(); } - return true; + return isValid_; } bool QueryObject::HasLimit() const @@ -226,171 +141,114 @@ void QueryObject::GetLimitVal(int &limit, int &offset) const offset = offset_; } -void QueryObject::FilterSymbolToAddBracketLink(bool &isNeedEndBracket, std::string &querySql) const +void QueryObject::SetSchema(const SchemaObject &schema) { - for (const auto &iter : queryObjNodes_) { - SymbolType symbolType = GetSymbolType(iter.operFlag); - if (symbolType == COMPARE_SYMBOL || symbolType == RELATIONAL_SYMBOL || symbolType == RANGE_SYMBOL) { - querySql += " AND ("; - isNeedEndBracket = true; - break; - } else if (symbolType == LOGIC_SYMBOL || symbolType == PREFIXKEY_SYMBOL) { - continue; - } else { - break; - } - } + schema_ = schema; } -int QueryObject::ToQuerySql() +bool QueryObject::IsCountValid() const { - if (queryObjNodes_.empty()) { - querySql_ += ";"; - return E_OK; + if (hasLimit_ || hasOrderBy_) { + LOGI("It is invalid for limit and orderby!"); + return false; } + return true; +} - bool isNeedEndBracket = false; - QueryObject::FilterSymbolToAddBracketLink(isNeedEndBracket, querySql_); +const std::vector &QueryObject::GetPrefixKey() const +{ + return prefixKey_; +} - int errCode = E_OK; - for (const QueryObjNode &objNode : queryObjNodes_) { - SymbolType symbolType = GetSymbolType(objNode.operFlag); - if (symbolType == SPECIAL_SYMBOL && isNeedEndBracket) { - querySql_ += ") "; - isNeedEndBracket = false; - } - errCode = ParseQueryExpression(objNode, querySql_); - if (errCode != E_OK) { - querySql_.clear(); - return errCode; - } - } +void QueryObject::ClearNodesFlag() +{ + limit_ = INVALID_LIMIT; + offset_ = 0; + isValid_ = true; + hasOrderBy_ = false; + hasLimit_ = false; + hasPrefixKey_ = false; + orderByCounts_ = 0; +} - if (isNeedEndBracket) { - querySql_ += ") "; +int QueryObject::Parse() +{ + if (!isValid_) { + LOGE("Invalid query object!"); + return -E_INVALID_QUERY_FORMAT; } - - // Limit needs to be placed after orderby and processed separately in the limit branch - if (hasPrefixKey_ && !hasOrderBy_ && !hasLimit_ && isNeedOrderbyKey_) { - LOGD("Need add order by key at last when has prefixkey no need order by value and limit!"); - querySql_ += "ORDER BY key ASC"; + int errCode = ParseQueryObjNodes(); + if (errCode != E_OK) { + LOGE("Check query object illegal!"); + isValid_ = false; } - - querySql_ += ";"; return errCode; } -int QueryObject::ToGetCountSql() +int QueryObject::ParseQueryObjNodes() { - countSql_.clear(); - if (queryObjNodes_.empty()) { - countSql_ += ";"; - return E_OK; - } - bool isNeedEndBracket = false; - QueryObject::FilterSymbolToAddBracketLink(isNeedEndBracket, countSql_); + ClearNodesFlag(); + auto iter = queryObjNodes_.begin(); int errCode = E_OK; - for (const QueryObjNode &objNode : queryObjNodes_) { - SymbolType symbolType = GetSymbolType(objNode.operFlag); - if (symbolType == SPECIAL_SYMBOL && isNeedEndBracket) { - countSql_ += ") "; - isNeedEndBracket = false; - } - - if (objNode.operFlag == QueryObjType::LIMIT) { - hasLimit_ = true; - if (objNode.fieldValue.size() == LIMIT_FIELD_VALUE_SIZE) { - limit_ = objNode.fieldValue[0].integerValue; - offset_ = objNode.fieldValue[1].integerValue; - } - continue; - } - if (objNode.operFlag == QueryObjType::ORDERBY) { - hasOrderBy_ = true; - continue; - } - errCode = ParseQueryExpression(objNode, countSql_); + while (iter != queryObjNodes_.end()) { + errCode = ParseNode(iter); if (errCode != E_OK) { - countSql_.clear(); return errCode; } + iter++; } - - if (isNeedEndBracket) { - countSql_ += ") "; - } - - // Limit needs to be placed after orderby and processed separately in the limit branch - if (hasPrefixKey_ && !hasOrderBy_ && !hasLimit_ && isNeedOrderbyKey_) { - LOGD("Need add order by key at last when has prefixkey no need order by value and limit!"); - countSql_ += "ORDER BY key ASC"; - } - countSql_ += ";"; return errCode; } -int QueryObject::GetQuerySqlStatement(sqlite3 *dbHandle, const std::string &sql, sqlite3_stmt *&statement) +int QueryObject::ParseNode(const std::list::iterator &iter) { - int errCode = SQLiteUtils::GetStatement(dbHandle, sql, statement); - if (errCode != E_OK) { - LOGE("[Query] Get statement fail!"); + SymbolType symbolType = SqliteQueryHelper::GetSymbolType(iter->operFlag); + // The object is newly instantiated in the connection, and there is no reentrancy problem. + if (symbolType == PREFIXKEY_SYMBOL && hasPrefixKey_) { + LOGE("Only filter by prefix key once!!"); return -E_INVALID_QUERY_FORMAT; } - int index = 1; - if (hasPrefixKey_) { - // bind the prefix key for the first and second args. - errCode = SQLiteUtils::BindPrefixKey(statement, 1, prefixKey_); - if (errCode != E_OK) { - LOGE("[Query] Get statement when bind prefix key, errCode = %d", errCode); - return errCode; - } - index = 3; // begin from 3rd args - } - for (const QueryObjNode &objNode : queryObjNodes_) { - errCode = BindFieldValue(statement, objNode, index); - if (errCode != E_OK) { - LOGE("[Query] Get statement fail when bind field value, errCode = %d", errCode); - return errCode; + if (iter->operFlag == QueryObjType::OPER_ILLEGAL || iter->type == QueryValueType::VALUE_TYPE_INVALID) { + return -E_INVALID_QUERY_FORMAT; + } else if (IsNeedCheckEqualFormat(symbolType)) { + return CheckEqualFormat(iter); + } else if (symbolType == LINK_SYMBOL) { + return CheckLinkerFormat(iter); + } else if (iter->operFlag == QueryObjType::LIMIT) { + hasLimit_ = true; + if (iter->fieldValue.size() == LIMIT_FIELD_VALUE_SIZE) { + limit_ = iter->fieldValue[0].integerValue; + offset_ = iter->fieldValue[1].integerValue; + } + return CheckLimitFormat(iter); + } else if (iter->operFlag == QueryObjType::ORDERBY) { + return CheckOrderByFormat(iter); + } else if (symbolType == PREFIXKEY_SYMBOL) { + hasPrefixKey_ = true; + if (prefixKey_.size() > DBConstant::MAX_KEY_SIZE) { + return -E_INVALID_ARGS; } + } else if (symbolType == SUGGEST_INDEX_SYMBOL) { + return CheckSuggestIndexFormat(iter); } - return errCode; + return E_OK; } -int QueryObject::GetCountSqlStatement(sqlite3 *dbHandle, const std::string &countSql, sqlite3_stmt *&countStmt) +int QueryObject::CheckLinkerBefore(const std::list::iterator &iter) const { - // bind statement for count - int errCode = SQLiteUtils::GetStatement(dbHandle, countSql, countStmt); - if (errCode != E_OK) { - LOGE("Get count statement error:%d", errCode); + auto preIter = std::prev(iter, 1); + SymbolType symbolType = SqliteQueryHelper::GetSymbolType(preIter->operFlag); + if (symbolType != COMPARE_SYMBOL && symbolType != RELATIONAL_SYMBOL && symbolType != LOGIC_SYMBOL && + symbolType != RANGE_SYMBOL && symbolType != PREFIXKEY_SYMBOL) { + LOGE("Must be a comparison operation before the connective! operFlag = %s", VNAME(preIter->operFlag)); return -E_INVALID_QUERY_FORMAT; } - int index = 1; - if (hasPrefixKey_) { - // bind the prefix key for the first and second args. - errCode = SQLiteUtils::BindPrefixKey(countStmt, 1, prefixKey_); - if (errCode != E_OK) { - LOGE("[Query] Get count statement fail when bind prefix key, errCode = %d", errCode); - return errCode; - } - index = 3; // begin from 3rd args - } - - for (const QueryObjNode &objNode : queryObjNodes_) { - if (GetSymbolType(objNode.operFlag) == SPECIAL_SYMBOL) { - continue; - } - errCode = BindFieldValue(countStmt, objNode, index); - if (errCode != E_OK) { - LOGE("[Query] Get count statement fail when bind field value, errCode = %d", errCode); - return errCode; - } - } - return errCode; + return E_OK; } -int QueryObject::CheckEqualFormat(std::list::iterator &iter) const +int QueryObject::CheckEqualFormat(const std::list::iterator &iter) const { if (!schema_.IsSchemaValid()) { LOGE("Schema is invalid!"); @@ -410,14 +268,15 @@ int QueryObject::CheckEqualFormat(std::list::iterator &iter) const return -E_INVALID_QUERY_FIELD; } - if (schemaFieldType == FieldType::LEAF_FIELD_BOOL && GetSymbolType(iter->operFlag) == COMPARE_SYMBOL && + if (schemaFieldType == FieldType::LEAF_FIELD_BOOL && + SqliteQueryHelper::GetSymbolType(iter->operFlag) == COMPARE_SYMBOL && iter->operFlag != QueryObjType::EQUALTO && iter->operFlag != QueryObjType::NOT_EQUALTO) { // bool can == or != LOGE("Bool forbid compare!!!"); return -E_INVALID_QUERY_FORMAT; } auto nextIter = std::next(iter, 1); if (nextIter != queryObjNodes_.end()) { - SymbolType symbolType = GetSymbolType(nextIter->operFlag); + SymbolType symbolType = SqliteQueryHelper::GetSymbolType(nextIter->operFlag); if (symbolType == RELATIONAL_SYMBOL || symbolType == COMPARE_SYMBOL || symbolType == RANGE_SYMBOL) { LOGE("After Compare you need, You need the conjunction like and or for connecting!"); return -E_INVALID_QUERY_FORMAT; @@ -426,7 +285,7 @@ int QueryObject::CheckEqualFormat(std::list::iterator &iter) const return E_OK; } -int QueryObject::CheckLinkerFormat(std::list::iterator &iter) const +int QueryObject::CheckLinkerFormat(const std::list::iterator &iter) const { if (iter == queryObjNodes_.begin()) { LOGE("Connectives are not allowed in the first place!"); @@ -437,16 +296,25 @@ int QueryObject::CheckLinkerFormat(std::list::iterator &iter) cons LOGE("Connectives are not allowed in the last place!"); return -E_INVALID_QUERY_FORMAT; } - SymbolType symbolType = GetSymbolType(nextIter->operFlag); + SymbolType symbolType = SqliteQueryHelper::GetSymbolType(nextIter->operFlag); if (symbolType == INVALID_SYMBOL || symbolType == LINK_SYMBOL || symbolType == SPECIAL_SYMBOL) { LOGE("Must be followed by comparison operation! operflag[%d], symbolType[%d]", nextIter->operFlag, symbolType); return -E_INVALID_QUERY_FORMAT; } + return CheckLinkerBefore(iter); +} - return CheckLinkerBefor(iter); +int QueryObject::CheckSuggestIndexFormat(const std::list::iterator &iter) const +{ + auto next = std::next(iter, 1); + if (next != queryObjNodes_.end()) { + LOGE("SuggestIndex only allowed once, and must appear at the end!"); + return -E_INVALID_QUERY_FORMAT; + } + return E_OK; } -int QueryObject::CheckOrderByFormat(std::list::iterator &iter) +int QueryObject::CheckOrderByFormat(const std::list::iterator &iter) { if (!schema_.IsSchemaValid()) { return -E_NOT_SUPPORT; @@ -466,271 +334,36 @@ int QueryObject::CheckOrderByFormat(std::list::iterator &iter) return -E_INVALID_QUERY_FORMAT; } hasOrderBy_ = true; - orderByCounts_++; + ++orderByCounts_; LOGD("Need order by %d filed value!", orderByCounts_); return E_OK; } -int QueryObject::CheckLimitFormat(std::list::iterator &iter) const +int QueryObject::CheckLimitFormat(const std::list::iterator &iter) const { - std::list::iterator next = std::next(iter, 1); - if (next != queryObjNodes_.end() && GetSymbolType(next->operFlag) != SUGGEST_INDEX_SYMBOL) { - LOGE("Limit should be last node or just before suggest-index nod!"); + auto next = std::next(iter, 1); + if (next != queryObjNodes_.end() && SqliteQueryHelper::GetSymbolType(next->operFlag) != SUGGEST_INDEX_SYMBOL) { + LOGE("Limit should be last node or just before suggest-index node!"); return -E_INVALID_QUERY_FORMAT; } return E_OK; } -int QueryObject::CheckSuggestIndexFormat(std::list::iterator &iter) const +bool QueryObject::IsQueryOnlyByKey() const { - std::list::iterator next = std::next(iter, 1); - if (next != queryObjNodes_.end()) { - LOGE("SuggestIndex only allowed once, and must appear at the end!"); - return -E_INVALID_QUERY_FORMAT; - } - return E_OK; + return std::none_of(queryObjNodes_.begin(), queryObjNodes_.end(), [&](const QueryObjNode &node) { + return node.operFlag != QueryObjType::LIMIT && node.operFlag != QueryObjType::QUERY_BY_KEY_PREFIX; + }); } -int QueryObject::CheckExpressionFormat(std::list::iterator &iter) +bool QueryObject::HasOrderBy() const { - SymbolType symbolType = GetSymbolType(iter->operFlag); - // The object is newly instantiated in the connection, and there is no reentrancy problem. - if (symbolType == PREFIXKEY_SYMBOL && hasPrefixKey_) { - LOGE("Only filt by prefix key once!!"); - return -E_INVALID_QUERY_FORMAT; - } - - if (iter->operFlag == QueryObjType::OPER_ILLEGAL || iter->type == QueryValueType::VALUE_TYPE_INVALID) { - return -E_INVALID_QUERY_FORMAT; - } else if (IsNeedCheckEqualFormat(symbolType)) { - return CheckEqualFormat(iter); - } else if (symbolType == LINK_SYMBOL) { - return CheckLinkerFormat(iter); - } else if (iter->operFlag == QueryObjType::LIMIT) { - hasLimit_ = true; - return CheckLimitFormat(iter); - } else if (iter->operFlag == QueryObjType::ORDERBY) { - return CheckOrderByFormat(iter); - } else if (symbolType == PREFIXKEY_SYMBOL) { - hasPrefixKey_ = true; - if (prefixKey_.size() > DBConstant::MAX_KEY_SIZE) { - return -E_INVALID_ARGS; - } - } else if (symbolType == SUGGEST_INDEX_SYMBOL) { - return CheckSuggestIndexFormat(iter); - } - return E_OK; + return hasOrderBy_; } -int QueryObject::CheckQueryLegality() +bool QueryObject::Empty() const { - hasPrefixKey_ = false; // For Check preFixkey once - orderByCounts_ = 0; - - auto iter = queryObjNodes_.begin(); - int errCode = E_OK; - while (iter != queryObjNodes_.end()) { - errCode = CheckExpressionFormat(iter); - if (errCode != E_OK) { - return errCode; - } - iter++; - } - return errCode; -} - -std::string QueryObject::MapRelationalSymbolToSql(const QueryObjNode &queryNode) -{ - if (RELATIONAL_SYMBOL_TO_SQL.find(queryNode.operFlag) == RELATIONAL_SYMBOL_TO_SQL.end()) { - return ""; - }; - std::string sql = RELATIONAL_SYMBOL_TO_SQL.at(queryNode.operFlag) + MapValueToSql(queryNode); - if (GetSymbolType(queryNode.operFlag) == RANGE_SYMBOL) { - sql += ")"; - } - return sql; -} - -std::string QueryObject::MapLogicSymbolToSql(const QueryObjNode &queryNode) const -{ - if (LOGIC_SYMBOL_TO_SQL.find(queryNode.operFlag) == LOGIC_SYMBOL_TO_SQL.end()) { - return ""; - } - return LOGIC_SYMBOL_TO_SQL.at(queryNode.operFlag); -} - -std::string QueryObject::MapKeywordSymbolToSql(const QueryObjNode &queryNode) -{ - std::string sql; - switch (queryNode.operFlag) { - case QueryObjType::ORDERBY: - if (queryNode.fieldValue.size() == SINGLE_FIELD_VALUE_SIZE) { - if (!isOrderByAppeared_) { - sql += "ORDER BY "; - } - - sql += QueryObject::MapCastFuncSql(queryNode); - sql += queryNode.fieldValue[0].boolValue ? "ASC," : "DESC,"; - orderByCounts_--; - if (orderByCounts_ == 0) { - sql.pop_back(); - } - isOrderByAppeared_ = true; - } - return sql; - case QueryObjType::LIMIT: - if (queryNode.fieldValue.size() == LIMIT_FIELD_VALUE_SIZE) { - if (hasPrefixKey_ && !hasOrderBy_ && isNeedOrderbyKey_) { - sql += "ORDER BY key ASC "; - } - sql += " LIMIT "; - sql += std::to_string(queryNode.fieldValue[0].integerValue); - sql += " OFFSET "; - sql += std::to_string(queryNode.fieldValue[1].integerValue); - } - return sql; - default: - return ""; - } - return sql; -} - -std::string QueryObject::MapValueToSql(const QueryObjNode &queryNode) -{ - std::string resultSql; - for (size_t i = 0; i < queryNode.fieldValue.size(); i++) { - if (i == 0) { - resultSql += " ? "; - } else { - resultSql += ", ? "; - } - } - return resultSql; -} - -static bool IsNeedCastWitEmptyValue(const QueryObjNode &queryNode) -{ - return (queryNode.operFlag == QueryObjType::IS_NULL || queryNode.operFlag == QueryObjType::IS_NOT_NULL || - queryNode.operFlag == QueryObjType::IN || queryNode.operFlag == QueryObjType::NOT_IN); -} - -std::string QueryObject::MapCastFuncSql(const QueryObjNode &queryNode) -{ - std::string resultSql; - if (queryNode.fieldValue.empty() && !IsNeedCastWitEmptyValue(queryNode)) { - return resultSql; - } - // fieldPath and isQueryable had been checked ok in the previous code, So here parse path and get type won't fail. - FieldPath fieldPath; - SchemaUtils::ParseAndCheckFieldPath(queryNode.fieldName, fieldPath); - FieldType fieldType = FieldType::LEAF_FIELD_INTEGER; - schema_.CheckQueryableAndGetFieldType(fieldPath, fieldType); - resultSql += SchemaObject::GenerateExtractSQL(schema_.GetSchemaType(), fieldPath, fieldType, schema_.GetSkipSize()); - isNeedOrderbyKey_ = false; // When index by value, No need order by key - return resultSql; -} - -int QueryObject::BindFieldValue(sqlite3_stmt *statement, const QueryObjNode &queryNode, int &index) const -{ - int errCode = E_OK; - SymbolType symbolType = GetSymbolType(queryNode.operFlag); - if (symbolType != COMPARE_SYMBOL && symbolType != RELATIONAL_SYMBOL && symbolType != RANGE_SYMBOL) { - return errCode; - } - - for (size_t i = 0; i < queryNode.fieldValue.size(); i++) { - if (queryNode.type == QueryValueType::VALUE_TYPE_BOOL) { - errCode = sqlite3_bind_int(statement, index, queryNode.fieldValue[i].boolValue); - } else if (queryNode.type == QueryValueType::VALUE_TYPE_INTEGER) { - errCode = sqlite3_bind_int(statement, index, queryNode.fieldValue[i].integerValue); - } else if (queryNode.type == QueryValueType::VALUE_TYPE_LONG) { - errCode = sqlite3_bind_int64(statement, index, queryNode.fieldValue[i].longValue); - } else if (queryNode.type == QueryValueType::VALUE_TYPE_DOUBLE) { - errCode = sqlite3_bind_double(statement, index, queryNode.fieldValue[i].doubleValue); - } else { - if (queryNode.fieldValue[i].stringValue.size() > MAX_SQLITE_BIND_SIZE) { - return -E_MAX_LIMITS; - } - errCode = sqlite3_bind_text(statement, index, queryNode.fieldValue[i].stringValue.c_str(), - queryNode.fieldValue[i].stringValue.size(), SQLITE_TRANSIENT); - } - if (errCode != SQLITE_OK) { - break; - } - index++; - } - return SQLiteUtils::MapSQLiteErrno(errCode); + return queryObjNodes_.empty(); } - -std::string QueryObject::MapCastTypeSql(const FieldType &type) -{ - switch (type) { - case FieldType::LEAF_FIELD_BOOL: - case FieldType::LEAF_FIELD_INTEGER: - case FieldType::LEAF_FIELD_LONG: - return "INT"; - case FieldType::LEAF_FIELD_DOUBLE: - return "REAL"; - case FieldType::LEAF_FIELD_STRING: - return "TEXT"; - case FieldType::LEAF_FIELD_NULL: - return "NULL"; - default: - return ""; - } } -int QueryObject::ParseQueryExpression(const QueryObjNode &queryNode, std::string &querySql) -{ - SymbolType symbolType = GetSymbolType(queryNode.operFlag); - if (symbolType == RANGE_SYMBOL && queryNode.fieldValue.size() > MAX_CONDITIONS_SIZE) { - LOGE("[Query][Parse][Expression] conditions is too many!"); - return -E_MAX_LIMITS; - } - - if (symbolType == COMPARE_SYMBOL || symbolType == RELATIONAL_SYMBOL || symbolType == RANGE_SYMBOL) { - querySql += MapCastFuncSql(queryNode); - querySql += MapRelationalSymbolToSql(queryNode); - } else if (symbolType == LOGIC_SYMBOL || symbolType == LINK_SYMBOL) { - querySql += MapLogicSymbolToSql(queryNode); - } else { - querySql += MapKeywordSymbolToSql(queryNode); - } - - if (querySql.size() > MAX_SQL_LEN) { - LOGE("[Query][Parse][Expression] Sql is too long!"); - return -E_MAX_LIMITS; - } - return E_OK; -} - -std::string QueryObject::AssembleSqlForSuggestIndex(const std::string &baseSql) const -{ - std::string formatIndex = CheckAndFormatSuggestIndex(); - if (formatIndex.empty()) { - return baseSql + FILTER_NATIVE_DATA_SQL; - } - - return baseSql + USING_INDEX + "'" + formatIndex + "' " + FILTER_NATIVE_DATA_SQL; -} - -std::string QueryObject::CheckAndFormatSuggestIndex() const -{ - if (suggestIndex_.empty()) { - return ""; - } - IndexName indexName; - int errCode = SchemaUtils::ParseAndCheckFieldPath(suggestIndex_, indexName); - if (errCode != E_OK) { - LOGW("Check and format suggest index failed! %d", errCode); - return ""; - } - - if (!schema_.IsIndexExist(indexName)) { - LOGW("The suggest index not exist!"); - return ""; - } - - return SchemaUtils::FieldPathString(indexName); -} -} diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h index 4884867e9420ce74ba051d6e05cbbf805f0b179d..b65ef70258fe0191fc7273b4ed5397657ed138e6 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_object.h @@ -18,61 +18,77 @@ #include #include "schema_object.h" #include "query.h" -#include "sqlite_import.h" +#include "sqlite_query_helper.h" namespace DistributedDB { class QueryObject { public: QueryObject(); explicit QueryObject(const Query &query); - ~QueryObject() = default; - int GetQuerySql(std::string &sql, bool onlyRowid = false); - int GetCountQuerySql(std::string &sql); - bool IsValid(int &errCode); - bool IsCountValid() const; + // for query sync + QueryObject(const std::list &queryObjNodes, const std::vector &prefixKey); + virtual ~QueryObject(); + int Init(); + SqliteQueryHelper GetQueryHelper(int &errCode); + + // suggest: get those attributes after init or GetQueryHelper to parsed query + bool IsValid(); bool HasLimit() const; void GetLimitVal(int &limit, int &offset) const; + bool IsCountValid() const; + + const std::vector &GetPrefixKey() const; void SetSchema(const SchemaObject &schema); - int GetQuerySqlStatement(sqlite3 *dbHandle, const std::string &sql, sqlite3_stmt *&statement); - int GetCountSqlStatement(sqlite3 *dbHandle, const std::string &countSql, sqlite3_stmt *&countStmt); -private: - int ToQuerySql(); - int ToGetCountSql(); - int CheckEqualFormat(std::list::iterator &iter) const; - int CheckLinkerFormat(std::list::iterator &iter) const; - int CheckOrderByFormat(std::list::iterator &iter); - int CheckExpressionFormat(std::list::iterator &iter); - int CheckLimitFormat(std::list::iterator &iter) const; - int CheckSuggestIndexFormat(std::list::iterator &iter) const; - int CheckQueryLegality(); - int ParseQueryExpression(const QueryObjNode &queryNode, std::string &querySql); - static std::string MapRelationalSymbolToSql(const QueryObjNode &queryNode); - std::string MapKeywordSymbolToSql(const QueryObjNode &queryNode); - std::string MapLogicSymbolToSql(const QueryObjNode &queryNode) const; - static std::string MapValueToSql(const QueryObjNode &queryNode); - std::string MapCastFuncSql(const QueryObjNode &queryNode); - static std::string MapCastTypeSql(const FieldType &type); - int BindFieldValue(sqlite3_stmt *statement, const QueryObjNode &queryNode, int &index) const; - void FilterSymbolToAddBracketLink(bool &isNeedEndBracket, std::string &querySql) const; - std::string AssembleSqlForSuggestIndex(const std::string &baseSql) const; - std::string CheckAndFormatSuggestIndex() const; - SchemaObject schema_; + + bool IsQueryOnlyByKey() const; + + void SetTableName(const std::string &tableName) + { + tableName_ = tableName; + isTableNameSpecified_ = true; + } + + const std::string &GetTableName() const + { + return tableName_; + } + + bool HasOrderBy() const; + + int ParseQueryObjNodes(); + + bool Empty() const; + +protected: std::list queryObjNodes_; std::vector prefixKey_; - std::string querySql_; - std::string countSql_; + std::string tableName_ = "sync_data"; std::string suggestIndex_; + bool isValid_ = true; + + bool initialized_ = false; // use function need after init + bool isTableNameSpecified_ = false; + +private: + int Parse(); + int ParseNode(const std::list::iterator &iter); + int CheckEqualFormat(const std::list::iterator &iter) const; + int CheckLinkerFormat(const std::list::iterator &iter) const; + int CheckSuggestIndexFormat(const std::list::iterator &iter) const; + int CheckOrderByFormat(const std::list::iterator &iter); + int CheckLimitFormat(const std::list::iterator &iter) const; + int CheckLinkerBefore(const std::list::iterator &iter) const; + void ClearNodesFlag(); + void GetAttrFromQueryObjNodes(); + + SchemaObject schema_; // used to check and parse schema filed int limit_; int offset_; - int orderByCounts_; // Record processing to which orderBy node - bool isValid_; - bool transformed_; bool hasOrderBy_; bool hasLimit_; - bool isOrderByAppeared_; bool hasPrefixKey_; - bool isNeedOrderbyKey_; // The tag field is used for prefix query filtering key sorting + int orderByCounts_; }; } #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp new file mode 100644 index 0000000000000000000000000000000000000000..31b551677cf25ec1d314518bfc4d5c429b576db1 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.cpp @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "query_sync_object.h" + +#include "db_errno.h" +#include "log_print.h" +#include "db_common.h" +#include "version.h" + +namespace DistributedDB { +namespace { +const std::string MAGIC = "remote query"; + +int SerializeDataObjNode(Parcel &parcel, const QueryObjNode &objNode) +{ + if (objNode.operFlag == QueryObjType::OPER_ILLEGAL) { + return -E_INVALID_QUERY_FORMAT; + } + (void)parcel.WriteUInt32(static_cast(objNode.operFlag)); + parcel.EightByteAlign(); + (void)parcel.WriteString(objNode.fieldName); + (void)parcel.WriteInt(static_cast(objNode.type)); + (void)parcel.WriteUInt32(objNode.fieldValue.size()); + + for (const FieldValue &value : objNode.fieldValue) { + (void)parcel.WriteString(value.stringValue); + + // string may not closely arranged continuously + // longValue is maximum length in union + (void)parcel.WriteInt64(value.longValue); + } + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + return E_OK; +} + +int DeSerializeDataObjNode(Parcel &parcel, QueryObjNode &objNode) +{ + uint32_t readOperFlag = 0; + (void)parcel.ReadUInt32(readOperFlag); + objNode.operFlag = static_cast(readOperFlag); + parcel.EightByteAlign(); + + (void)parcel.ReadString(objNode.fieldName); + + int readInt = -1; + (void)parcel.ReadInt(readInt); + objNode.type = static_cast(readInt); + + uint32_t valueSize = 0; + (void)parcel.ReadUInt32(valueSize); + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + + for (size_t i = 0; i < valueSize; i++) { + FieldValue value; + (void)parcel.ReadString(value.stringValue); + + (void)parcel.ReadInt64(value.longValue); + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + objNode.fieldValue.push_back(value); + } + return E_OK; +} +} + +QuerySyncObject::QuerySyncObject() +{} + +QuerySyncObject::QuerySyncObject(const std::list &queryObjNodes, const std::vector &prefixKey) + : QueryObject(queryObjNodes, prefixKey) +{} + +QuerySyncObject::QuerySyncObject(const Query &query) + : QueryObject(query) +{} + +QuerySyncObject::~QuerySyncObject() +{} + +int QuerySyncObject::GetObjContext(ObjContext &objContext) const +{ + if (!isValid_) { + return -E_INVALID_QUERY_FORMAT; + } + objContext.version = isTableNameSpecified_ ? QUERY_SYNC_OBJECT_VERSION_1 : QUERY_SYNC_OBJECT_VERSION_0; + objContext.prefixKey.assign(prefixKey_.begin(), prefixKey_.end()); + objContext.suggestIndex = suggestIndex_; + objContext.queryObjNodes = queryObjNodes_; + return E_OK; +} + +std::string QuerySyncObject::GetIdentify() const +{ + if (!isValid_) { + return std::string(); + } + // suggestionIndex is local attribute, do not need to be propagated to remote + uint64_t len = Parcel::GetVectorCharLen(prefixKey_); + if (isTableNameSpecified_) { + len += Parcel::GetStringLen(tableName_); + } + for (const QueryObjNode &node : queryObjNodes_) { + if (node.operFlag == QueryObjType::LIMIT || node.operFlag == QueryObjType::ORDERBY || + node.operFlag == QueryObjType::SUGGEST_INDEX) { + continue; + } + // operFlag and valueType is int + len += Parcel::GetUInt32Len() + Parcel::GetIntLen() + Parcel::GetStringLen(node.fieldName); + for (const FieldValue &value : node.fieldValue) { + len += Parcel::GetStringLen(value.stringValue) + Parcel::GetInt64Len(); + } + } + + std::vector buff(len, 0); // It will affect the hash result, the default value cannot be modified + Parcel parcel(buff.data(), len); + + // The order needs to be consistent, otherwise it will affect the hash result + (void)parcel.WriteVectorChar(prefixKey_); + if (isTableNameSpecified_) { + (void)parcel.WriteString(tableName_); + } + for (const QueryObjNode &node : queryObjNodes_) { + if (node.operFlag == QueryObjType::LIMIT || node.operFlag == QueryObjType::ORDERBY || + node.operFlag == QueryObjType::SUGGEST_INDEX) { + continue; + } + (void)parcel.WriteUInt32(static_cast(node.operFlag)); + (void)parcel.WriteInt(static_cast(node.type)); + (void)parcel.WriteString(node.fieldName); + for (const FieldValue &value : node.fieldValue) { + (void)parcel.WriteInt64(value.longValue); + (void)parcel.WriteString(value.stringValue); + } + } + + if (parcel.IsError()) { + return std::string(); + } + + std::vector hashBuff; + int errCode = DBCommon::CalcValueHash(buff, hashBuff); + if (errCode != E_OK) { + return std::string(); + } + return DBCommon::VectorToHexString(hashBuff); +} + +uint32_t QuerySyncObject::CalculateParcelLen(uint32_t softWareVersion) const +{ + if (softWareVersion == SOFTWARE_VERSION_CURRENT) { + return CalculateLen(); + } + LOGE("current not support!"); + return 0; +} + +int QuerySyncObject::SerializeData(Parcel &parcel, uint32_t softWareVersion) +{ + ObjContext context; + int errCode = GetObjContext(context); + if (errCode != E_OK) { + return errCode; + } + + (void)parcel.WriteString(MAGIC); + (void)parcel.WriteUInt32(context.version); + (void)parcel.WriteVectorChar(context.prefixKey); + (void)parcel.WriteString(context.suggestIndex); + if (isTableNameSpecified_) { + (void)parcel.WriteString(tableName_); + } + (void)parcel.WriteUInt32(context.queryObjNodes.size()); + + parcel.EightByteAlign(); + + for (const QueryObjNode &node : context.queryObjNodes) { + errCode = SerializeDataObjNode(parcel, node); + if (errCode != E_OK) { + return errCode; + } + } + if (parcel.IsError()) { // parcel almost success + return -E_INVALID_ARGS; + } + parcel.EightByteAlign(); + return E_OK; +} + +int QuerySyncObject::DeSerializeData(Parcel &parcel, QuerySyncObject &queryObj) +{ + std::string magic; + (void)parcel.ReadString(magic); + if (magic != MAGIC) { + return -E_INVALID_ARGS; + } + + ObjContext context; + (void)parcel.ReadUInt32(context.version); + if (context.version > QUERY_SYNC_OBJECT_VERSION_1) { + LOGE("Parcel version and deserialize version not matched! ver=%u", context.version); + return -E_VERSION_NOT_SUPPORT; + } + + (void)parcel.ReadVectorChar(context.prefixKey); + (void)parcel.ReadString(context.suggestIndex); + std::string tableName; + if (context.version == QUERY_SYNC_OBJECT_VERSION_1) { + parcel.ReadString(tableName); + } + + uint32_t nodesSize = 0; + (void)parcel.ReadUInt32(nodesSize); + if (parcel.IsError()) { // almost success + return -E_INVALID_ARGS; + } + parcel.EightByteAlign(); + for (size_t i = 0; i < nodesSize; i++) { + QueryObjNode node; + int errCode = DeSerializeDataObjNode(parcel, node); + if (errCode != E_OK) { + return errCode; + } + context.queryObjNodes.emplace_back(node); + } + + if (parcel.IsError()) { // almost success + return -E_INVALID_ARGS; + } + queryObj = QuerySyncObject(context.queryObjNodes, context.prefixKey); + if (context.version == QUERY_SYNC_OBJECT_VERSION_1) { + queryObj.SetTableName(tableName); + } + return E_OK; +} + +uint32_t QuerySyncObject::CalculateLen() const +{ + uint64_t len = Parcel::GetStringLen(MAGIC); + len += Parcel::GetUInt32Len(); // version + len += Parcel::GetVectorCharLen(prefixKey_); + len += Parcel::GetStringLen(suggestIndex_); + if (isTableNameSpecified_) { + len += Parcel::GetStringLen(tableName_); + } + len += Parcel::GetUInt32Len(); // nodes size + len = Parcel::GetEightByteAlign(len); + for (const QueryObjNode &node : queryObjNodes_) { + if (node.operFlag == QueryObjType::OPER_ILLEGAL) { + LOGE("contain illegal operator for query sync!"); + return 0; + } + // operflag, fieldName, query value type, value size, union max size, string value + len += Parcel::GetUInt32Len(); + len = Parcel::GetEightByteAlign(len); + len += Parcel::GetStringLen(node.fieldName) + + Parcel::GetIntLen() + Parcel::GetUInt32Len(); + for (size_t i = 0; i < node.fieldValue.size(); i++) { + len += Parcel::GetInt64Len() + Parcel::GetStringLen(node.fieldValue[i].stringValue); + } + } + len = Parcel::GetEightByteAlign(len); + if (len > INT32_MAX) { + return 0; + } + return static_cast(len); +} + +std::string QuerySyncObject::GetRelationTableName() const +{ + if (!isTableNameSpecified_) { + return {}; + } + return tableName_; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h new file mode 100644 index 0000000000000000000000000000000000000000..052b9ea29b5d6f4a51cd09f4920b77054170d2ab --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/query_sync_object.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef QUERY_SYNC_OBJECT_H +#define QUERY_SYNC_OBJECT_H + +#include + +#include "query_object.h" +#include "parcel.h" + +namespace DistributedDB { +const uint32_t QUERY_SYNC_OBJECT_VERSION_0 = 0; // for kvDB +const uint32_t QUERY_SYNC_OBJECT_VERSION_1 = 1; // for relational DB, which need specify tableName + +struct ObjContext { + uint32_t version = QUERY_SYNC_OBJECT_VERSION_0; // serialized struct version + std::vector prefixKey{}; + std::string suggestIndex{}; + std::list queryObjNodes{}; +}; + +class QuerySyncObject : public QueryObject { +public: + QuerySyncObject(); + QuerySyncObject(const std::list &queryObjNodes, const std::vector &prefixKey); + explicit QuerySyncObject(const Query &query); + ~QuerySyncObject() override; + + std::string GetIdentify() const; + + int SerializeData(Parcel &parcel, uint32_t softWareVersion); + // should call Parcel.IsError() to Get result. + static int DeSerializeData(Parcel &parcel, QuerySyncObject &queryObj); + uint32_t CalculateParcelLen(uint32_t softWareVersion) const; + + std::string GetRelationTableName() const; + +private: + uint32_t CalculateLen() const; + int GetObjContext(ObjContext &objContext) const; +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ed7bdb058df387637c4cb5f97ec47881e8c10d46 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.cpp @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "sqlite_relational_store.h" + +#include "db_errno.h" +#include "log_print.h" +#include "db_types.h" +#include "sqlite_relational_store_connection.h" + +namespace DistributedDB { +SQLiteRelationalStore::~SQLiteRelationalStore() +{ + delete sqliteStorageEngine_; +} + + +// Called when a new connection created. +void SQLiteRelationalStore::IncreaseConnectionCounter() +{ + connectionCount_.fetch_add(1, std::memory_order_seq_cst); + if (connectionCount_.load() > 0) { + sqliteStorageEngine_->SetConnectionFlag(true); + } +} + +RelationalStoreConnection *SQLiteRelationalStore::GetDBConnection(int &errCode) +{ + std::lock_guard lock(connectMutex_); + RelationalStoreConnection* connection = new (std::nothrow) SQLiteRelationalStoreConnection(this); + + if (connection == nullptr) { + errCode = -E_OUT_OF_MEMORY; + return nullptr; + } + IncObjRef(this); + IncreaseConnectionCounter(); + return connection; +} + +static void InitDataBaseOption(const DBProperties &kvDBProp, OpenDbProperties &option) +{ + option.uri = kvDBProp.GetStringProp(KvDBProperties::DATA_DIR, ""); + option.createIfNecessary = kvDBProp.GetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, false); +} + +int SQLiteRelationalStore::InitStorageEngine(const DBProperties &kvDBProp) +{ + OpenDbProperties option; + InitDataBaseOption(kvDBProp, option); + + StorageEngineAttr poolSize = {1, 1, 0, 16}; // at most 1 write 16 read. + int errCode = sqliteStorageEngine_->InitSQLiteStorageEngine(poolSize, option); + if (errCode != E_OK) { + LOGE("Init the sqlite storage engine failed:%d", errCode); + } + return errCode; +} + +int SQLiteRelationalStore::Open(const DBProperties &properties) +{ + sqliteStorageEngine_ = new (std::nothrow) SQLiteSingleRelationalStorageEngine(); + if (sqliteStorageEngine_ == nullptr) { + LOGE("[RelationalStore] Create storage engine failed"); + return -E_OUT_OF_MEMORY; + } + + int errCode = InitStorageEngine(properties); + if (errCode != E_OK) { + LOGE("[RelationalStore][Open] Init database context fail! errCode = [%d]", errCode); + return errCode; + } + storageEngine_ = new(std::nothrow) RelationalSyncAbleStorage(sqliteStorageEngine_); + syncEngine_ = std::make_shared(storageEngine_); + return E_OK; +} + +void SQLiteRelationalStore::OnClose(const std::function ¬ifier) +{ + AutoLock lockGuard(this); + if (notifier) { + closeNotifiers_.push_back(notifier); + } else { + LOGW("Register 'Close()' notifier failed, notifier is null."); + } +} + +SQLiteSingleVerRelationalStorageExecutor *SQLiteRelationalStore::GetHandle(bool isWrite, int &errCode) const +{ + if (sqliteStorageEngine_ == nullptr) { + errCode = -E_INVALID_DB; + return nullptr; + } + + return static_cast(sqliteStorageEngine_->FindExecutor(isWrite, + OperatePerm::NORMAL_PERM, errCode)); +} +void SQLiteRelationalStore::ReleaseHandle(SQLiteSingleVerRelationalStorageExecutor *&handle) const +{ + if (handle == nullptr) { + return; + } + + if (sqliteStorageEngine_ != nullptr) { + StorageExecutor *databaseHandle = handle; + sqliteStorageEngine_->Recycle(databaseHandle); + handle = nullptr; + } +} + +int SQLiteRelationalStore::Sync(const ISyncer::SyncParma &syncParam) +{ + return syncEngine_->Sync(syncParam); +} + +// Called when a connection released. +void SQLiteRelationalStore::DecreaseConnectionCounter() +{ + int count = connectionCount_.fetch_sub(1, std::memory_order_seq_cst); + if (count <= 0) { + LOGF("Decrease db connection counter failed, count <= 0."); + return; + } + if (count != 1) { + return; + } + + LockObj(); + auto notifiers = std::move(closeNotifiers_); + UnlockObj(); + + for (auto ¬ifier : notifiers) { + if (notifier) { + notifier(); + } + } + + // Sync Close + syncEngine_->Close(); + + if (sqliteStorageEngine_ != nullptr) { + delete sqliteStorageEngine_; + sqliteStorageEngine_ = nullptr; + } + // close will dec sync ref of storageEngine_ + DecObjRef(storageEngine_); +} + +void SQLiteRelationalStore::ReleaseDBConnection(RelationalStoreConnection *connection) +{ + if (connectionCount_.load() == 1) { + sqliteStorageEngine_->SetConnectionFlag(false); + } + + connectMutex_.lock(); + if (connection != nullptr) { + KillAndDecObjRef(connection); + DecreaseConnectionCounter(); + connectMutex_.unlock(); + KillAndDecObjRef(this); + } else { + connectMutex_.unlock(); + } +} + +void SQLiteRelationalStore::WakeUpSyncer() +{ + syncEngine_->WakeUpSyncer(); +} +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h new file mode 100644 index 0000000000000000000000000000000000000000..f41dd0839819f9e02d4a1933561241f3121d9789 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SQLITE_RELATIONAL_STORE_H +#define SQLITE_RELATIONAL_STORE_H +#ifdef RELATIONAL_STORE + +#include +#include +#include + +#include "irelational_store.h" +#include "sqlite_single_relational_storage_engine.h" +#include "isyncer.h" +#include "sync_able_engine.h" +#include "relational_sync_able_storage.h" + +namespace DistributedDB { +class SQLiteRelationalStore : public IRelationalStore { +public: + SQLiteRelationalStore() = default; + ~SQLiteRelationalStore() override; + + RelationalStoreConnection *GetDBConnection(int &errCode) override; + int Open(const DBProperties &properties) override; + void OnClose(const std::function ¬ifier); + + SQLiteSingleVerRelationalStorageExecutor *GetHandle(bool isWrite, int &errCode) const; + void ReleaseHandle(SQLiteSingleVerRelationalStorageExecutor *&handle) const; + + int Sync(const ISyncer::SyncParma &syncParam); + + void ReleaseDBConnection(RelationalStoreConnection *connection); + + void WakeUpSyncer(); + + // for test mock + const RelationalSyncAbleStorage *GetStorageEngine() + { + return storageEngine_; + } +private: + // 1 store 1 connection + void DecreaseConnectionCounter(); + + // use for sync Interactive + std::shared_ptr syncEngine_ = nullptr; // For storage operate sync function + // use ref obj same as kv + RelationalSyncAbleStorage *storageEngine_ = nullptr; // For storage operate data + SQLiteSingleRelationalStorageEngine *sqliteStorageEngine_ = nullptr; + + void IncreaseConnectionCounter(); + int InitStorageEngine(const DBProperties &kvDBProp); + std::mutex connectMutex_; + std::atomic connectionCount_ = 0; + std::vector> closeNotifiers_; +}; +} // namespace DistributedDB +#endif +#endif // SQLITE_RELATIONAL_STORE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d322575d709847e339e745318ea1d06e5b383aff --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.cpp @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "sqlite_relational_store_connection.h" +#include "db_errno.h" +#include "log_print.h" + +namespace DistributedDB { +SQLiteRelationalStoreConnection::SQLiteRelationalStoreConnection(SQLiteRelationalStore *store) + : RelationalStoreConnection(store) {} +// Close and release the connection. +int SQLiteRelationalStoreConnection::Close() +{ + if (store_ == nullptr) { + return -E_INVALID_CONNECTION; + } + + if (isExclusive_.load()) { + return -E_BUSY; + } + + // check if transaction closed + { + std::lock_guard transactionLock(transactionMutex_); + if (writeHandle_ != nullptr) { + LOGW("Transaction started, need to rollback before close."); + int errCode = RollBack(); + if (errCode != E_OK) { + LOGE("Rollback transaction failed, %d.", errCode); + } + ReleaseExecutor(writeHandle_); + } + } + + static_cast(store_)->ReleaseDBConnection(this); + return E_OK; +} + +std::string SQLiteRelationalStoreConnection::GetIdentifier() +{ + return std::string(); +} + +SQLiteSingleVerRelationalStorageExecutor *SQLiteRelationalStoreConnection::GetExecutor(bool isWrite, int &errCode) const +{ + auto *store = GetDB(); + if (store == nullptr) { + errCode = -E_NOT_INIT; + LOGE("[SingleVerConnection] store is null, get executor failed! errCode = [%d]", errCode); + return nullptr; + } + + return store->GetHandle(isWrite, errCode); +} + +void SQLiteRelationalStoreConnection::ReleaseExecutor(SQLiteSingleVerRelationalStorageExecutor *&executor) const +{ + auto *store = GetDB(); + if (store != nullptr) { + store->ReleaseHandle(executor); + } +} + +int SQLiteRelationalStoreConnection::StartTransaction() +{ + std::lock_guard lock(transactionMutex_); + if (writeHandle_ != nullptr) { + LOGD("Transaction started already."); + return -E_TRANSACT_STATE; + } + + int errCode = E_OK; + auto *handle = GetExecutor(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->StartTransaction(TransactType::DEFERRED); + if (errCode != E_OK) { + ReleaseExecutor(handle); + return errCode; + } + + LOGD("[SingleVerConnection] Start transaction finish."); + writeHandle_ = handle; + transactingFlag_.store(true); + return E_OK; +} + +// Commit the transaction +int SQLiteRelationalStoreConnection::Commit() +{ + std::lock_guard lock(transactionMutex_); + if (writeHandle_ == nullptr) { + LOGE("single version database is null or the transaction has not been started"); + return -E_INVALID_DB; + } + + int errCode = writeHandle_->Commit(); + ReleaseExecutor(writeHandle_); + if (errCode == E_OK) { + transactingFlag_.store(false); + } + LOGD("connection commit transaction!"); + return errCode; +} + +// Roll back the transaction +int SQLiteRelationalStoreConnection::RollBack() +{ + std::lock_guard lock(transactionMutex_); + if (writeHandle_ == nullptr) { + LOGE("Invalid handle for rollback or the transaction has not been started."); + return -E_INVALID_DB; + } + + int errCode = writeHandle_->Rollback(); + ReleaseExecutor(writeHandle_); + if (errCode == E_OK) { + transactingFlag_.store(false); + } + LOGI("connection rollback transaction!"); + return errCode; +} + +int SQLiteRelationalStoreConnection::CreateDistributedTable(const std::string &tableName, + const RelationalStoreDelegate::TableOption &option) +{ + int errCode = StartTransaction(); + if (errCode != E_OK && errCode != E_TRANSACT_STATE) { + return errCode; + } + + errCode = writeHandle_->CreateDistributedTable(tableName, option); + if (errCode == E_OK) { + errCode = Commit(); + } else { + int innerCode = RollBack(); + errCode = (innerCode != E_OK) ? innerCode : errCode; + } + return errCode; +} + +int SQLiteRelationalStoreConnection::Pragma(int cmd, void *parameter) // reserve for interface function fix +{ + return E_OK; +} + +int SQLiteRelationalStoreConnection::TriggerAutoSync() +{ + return E_OK; +} + +int SQLiteRelationalStoreConnection::SyncToDevice(SyncInfo &info) +{ + auto *store = GetDB(); + if (store == nullptr) { + LOGE("[SingleVerConnection] store is null, get executor failed!"); + return -E_INVALID_CONNECTION; + } + + { + AutoLock lockGuard(this); + IncObjRef(this); + ISyncer::SyncParma syncParam; + syncParam.devices = info.devices; + syncParam.mode = info.mode; + syncParam.wait = info.wait; + syncParam.isQuerySync = true; + syncParam.relationOnComplete = info.onComplete; + syncParam.syncQuery = QuerySyncObject(info.query); + syncParam.onFinalize = [this]() { DecObjRef(this); }; + + int errCode = store->Sync(syncParam); + if (errCode != E_OK) { + DecObjRef(this); + } + } + return E_OK; +} +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h new file mode 100644 index 0000000000000000000000000000000000000000..42184112fc0917c0a31fdbf19fc1e440db58044c --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_relational_store_connection.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SQLITE_RELATIONAL_STORE_CONNECTION_H +#define SQLITE_RELATIONAL_STORE_CONNECTION_H +#ifdef RELATIONAL_STORE + +#include +#include +#include "macro_utils.h" +#include "relational_store_connection.h" +#include "sqlite_single_ver_relational_storage_executor.h" +#include "sqlite_relational_store.h" + +namespace DistributedDB { +class SQLiteRelationalStoreConnection : public RelationalStoreConnection { +public: + explicit SQLiteRelationalStoreConnection(SQLiteRelationalStore *store); + + ~SQLiteRelationalStoreConnection() override = default; + + DISABLE_COPY_ASSIGN_MOVE(SQLiteRelationalStoreConnection); + + // Close and release the connection. + int Close() override; + int TriggerAutoSync() override; + int SyncToDevice(SyncInfo &info) override; + std::string GetIdentifier() override; + int CreateDistributedTable(const std::string &tableName, + const RelationalStoreDelegate::TableOption &option) override; + +protected: + + int Pragma(int cmd, void *parameter); +private: + + SQLiteSingleVerRelationalStorageExecutor *GetExecutor(bool isWrite, int &errCode) const; + void ReleaseExecutor(SQLiteSingleVerRelationalStorageExecutor *&executor) const; + int StartTransaction(); + // Commit the transaction + int Commit(); + + // Roll back the transaction + int RollBack(); + + SQLiteSingleVerRelationalStorageExecutor *writeHandle_ = nullptr; + mutable std::mutex transactionMutex_; // used for transaction + std::atomic transactingFlag_; +}; +} // namespace DistributedDB +#endif +#endif // SQLITE_RELATIONAL_STORE_CONNECTION_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ab12fd5d85fe01e9b000b2c3c335c35299da09ba --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "sqlite_single_relational_storage_engine.h" + +#include "sqlite_single_ver_relational_storage_executor.h" +#include "db_errno.h" + +namespace DistributedDB { +SQLiteSingleRelationalStorageEngine::SQLiteSingleRelationalStorageEngine() {}; + +SQLiteSingleRelationalStorageEngine::~SQLiteSingleRelationalStorageEngine() {}; + +StorageExecutor *SQLiteSingleRelationalStorageEngine::NewSQLiteStorageExecutor(sqlite3 *dbHandle, bool isWrite, + bool isMemDb) +{ + return new (std::nothrow) SQLiteSingleVerRelationalStorageExecutor(dbHandle, isWrite); +} + +int SQLiteSingleRelationalStorageEngine::Upgrade(sqlite3 *db) +{ + return SQLiteUtils::CreateRelationalMetaTable(db); +} + +int SQLiteSingleRelationalStorageEngine::RegisterFunction(sqlite3 *db) const +{ + int errCode = SQLiteUtils::RegisterCalcHash(db); + if (errCode != E_OK) { + LOGE("[engine] register calculate hash failed!"); + return errCode; + } + + errCode = SQLiteUtils::RegisterGetSysTime(db); + if (errCode != E_OK) { + LOGE("[engine] register get sys time failed!"); + } + return E_OK; +} + +int SQLiteSingleRelationalStorageEngine::CreateNewExecutor(bool isWrite, StorageExecutor *&handle) +{ + sqlite3 *db = nullptr; + int errCode = SQLiteUtils::OpenDatabase(option_, db); + if (errCode != E_OK) { + return errCode; + } + do { + errCode = Upgrade(db); + if (errCode != E_OK) { + break; + } + + errCode = RegisterFunction(db); + if (errCode != E_OK) { + break; + } + handle = NewSQLiteStorageExecutor(db, isWrite, false); + if (handle == nullptr) { + LOGE("[Relational] New SQLiteStorageExecutor[%d] for the pool failed.", isWrite); + errCode = -E_OUT_OF_MEMORY; + break; + } + return E_OK; + } while (false); + + (void)sqlite3_close_v2(db); + db = nullptr; + return errCode; +} +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h new file mode 100644 index 0000000000000000000000000000000000000000..28ea03efbf03f16fd3f4e4215b6bd46486572099 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/relational/sqlite_single_relational_storage_engine.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SQLITE_RELATIONAL_ENGINE_H +#define SQLITE_RELATIONAL_ENGINE_H +#ifdef RELATIONAL_STORE + +#include "macro_utils.h" +#include "sqlite_storage_engine.h" +#include "sqlite_single_ver_relational_storage_executor.h" + +namespace DistributedDB { +class SQLiteSingleRelationalStorageEngine : public SQLiteStorageEngine { +public: + SQLiteSingleRelationalStorageEngine(); + ~SQLiteSingleRelationalStorageEngine() override; + + // Delete the copy and assign constructors + DISABLE_COPY_ASSIGN_MOVE(SQLiteSingleRelationalStorageEngine); + +protected: + StorageExecutor *NewSQLiteStorageExecutor(sqlite3 *db, bool isWrite, bool isMemDb) override; + int Upgrade(sqlite3 *db) override; + int CreateNewExecutor(bool isWrite, StorageExecutor *&handle) override; + +private: + // For executor. + + // For engine. + + // For db. + int RegisterFunction(sqlite3 *db) const; +}; +} // namespace DistributedDB + +#endif +#endif // SQLITE_RELATIONAL_ENGINE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_local_kvdb_snapshot.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_local_kvdb_snapshot.h index c94962f757a95afaca0bb75022364bbc35081664..cc8b7a1ce46f84eaff624f41e83ce2a71a90c2bd 100644 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_local_kvdb_snapshot.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_local_kvdb_snapshot.h @@ -37,7 +37,7 @@ public: int Get(const Key &key, Value &value) const override; // Get the entries of the key set - int GetEntries(const Key &keyPrefix, std::vector &entry) const override; + int GetEntries(const Key &keyPrefix, std::vector &entries) const override; void Close(); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2053fb76b9990f533e89313d0e03b4bcbf4e407b --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.cpp @@ -0,0 +1,798 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "sqlite_query_helper.h" + +#include + +#include "db_common.h" +#include "db_constant.h" +#include "db_errno.h" +#include "log_print.h" +#include "macro_utils.h" +#include "sqlite_utils.h" + +namespace DistributedDB { +using namespace TriggerMode; +namespace { +const std::string PRE_QUERY_KV_SQL = "SELECT key, value FROM sync_data "; +const std::string PRE_QUERY_ITEM_SQL = "SELECT * FROM "; +const std::string PRE_QUERY_ROWID_SQL = "SELECT rowid FROM sync_data "; +const std::string PRE_GET_COUNT_SQL = "SELECT count(*) FROM sync_data "; +const std::string FILTER_NATIVE_DATA_SQL = "WHERE (flag&0x01=0) "; +const std::string FILTER_REMOTE_QUERY = "WHERE (flag&0x03=0x02)"; +const std::string USING_INDEX = "INDEXED BY "; +const int MAX_SQL_LEN = 1024 * 1024; // 1M bytes +const int SINGLE_FIELD_VALUE_SIZE = 1; +const int MAX_CONDITIONS_SIZE = 128; +const int MAX_SQLITE_BIND_SIZE = 50000; +const uint32_t SYMBOL_TYPE_MASK = 0xff00; + +const std::map RELATIONAL_SYMBOL_TO_SQL { + {QueryObjType::EQUALTO, "= "}, + {QueryObjType::NOT_EQUALTO, "!= "}, + {QueryObjType::GREATER_THAN, "> "}, + {QueryObjType::LESS_THAN, "< "}, + {QueryObjType::GREATER_THAN_OR_EQUALTO, ">= "}, + {QueryObjType::LESS_THAN_OR_EQUALTO, "<= "}, + {QueryObjType::LIKE, " LIKE "}, + {QueryObjType::NOT_LIKE, " NOT LIKE "}, + {QueryObjType::IS_NULL, " IS NULL "}, + {QueryObjType::IS_NOT_NULL, " IS NOT NULL "}, + {QueryObjType::IN, " IN ("}, + {QueryObjType::NOT_IN, " NOT IN ("}, +}; + +const std::map LOGIC_SYMBOL_TO_SQL { + {QueryObjType::AND, " AND "}, + {QueryObjType::OR, " OR "}, + {QueryObjType::BEGIN_GROUP, "("}, + {QueryObjType::END_GROUP, ")"}, +}; + +std::string FieldValue2String(const FieldValue &val, QueryValueType type) +{ + std::stringstream ss; + switch (type) { + case QueryValueType::VALUE_TYPE_NULL: + return "NULL"; + case QueryValueType::VALUE_TYPE_BOOL: + return val.boolValue ? "1" : "0"; + case QueryValueType::VALUE_TYPE_INTEGER: + return std::to_string(val.integerValue); + case QueryValueType::VALUE_TYPE_LONG: + return std::to_string(val.longValue); + case QueryValueType::VALUE_TYPE_DOUBLE: + ss << std::setprecision(DBConstant::DOUBLE_PRECISION) << val.doubleValue; + return ss.str(); + case QueryValueType::VALUE_TYPE_STRING: + return "'" + val.stringValue + "'"; + case QueryValueType::VALUE_TYPE_INVALID: + default: + return ""; + } +} +} + +SqliteQueryHelper::SqliteQueryHelper(const QueryObjInfo &info) + : schema_(info.schema_), + queryObjNodes_(info.queryObjNodes_), + prefixKey_(info.prefixKey_), + suggestIndex_(info.suggestIndex_), + tableName_(info.tableName_), + orderByCounts_(info.orderByCounts_), + isValid_(info.isValid_), + transformed_(false), + hasOrderBy_(info.hasOrderBy_), + hasLimit_(info.hasLimit_), + isOrderByAppeared_(false), + hasPrefixKey_(info.hasPrefixKey_), + isNeedOrderbyKey_(false) +{} + +SymbolType SqliteQueryHelper::GetSymbolType(const QueryObjType &queryObjType) +{ + return static_cast(static_cast(queryObjType) & SYMBOL_TYPE_MASK); +} + +bool SqliteQueryHelper::FilterSymbolToAddBracketLink(std::string &querySql, bool isNeedLink) const +{ + bool isNeedEndBracket = false; + for (const auto &iter : queryObjNodes_) { + SymbolType symbolType = GetSymbolType(iter.operFlag); + if (symbolType == COMPARE_SYMBOL || symbolType == RELATIONAL_SYMBOL || symbolType == RANGE_SYMBOL) { + querySql += isNeedLink ? " AND (" : " ("; + isNeedEndBracket = true; + break; + } else if (symbolType == LOGIC_SYMBOL || symbolType == PREFIXKEY_SYMBOL) { + continue; + } else { + break; + } + } + return isNeedEndBracket; +} + + +int SqliteQueryHelper::ParseQueryObjNodeToSQL() +{ + if (queryObjNodes_.empty()) { + querySql_ += ";"; + return E_OK; + } + + bool isNeedEndBracket = FilterSymbolToAddBracketLink(querySql_); + + int errCode = E_OK; + for (const QueryObjNode &objNode : queryObjNodes_) { + SymbolType symbolType = GetSymbolType(objNode.operFlag); + if (symbolType == SPECIAL_SYMBOL && isNeedEndBracket) { + querySql_ += ") "; + isNeedEndBracket = false; + } + errCode = ParseQueryExpression(objNode, querySql_); + if (errCode != E_OK) { + querySql_.clear(); + return errCode; + } + } + + if (isNeedEndBracket) { + querySql_ += ") "; + } + + return errCode; +} + +int SqliteQueryHelper::ToQuerySql() +{ + int errCode = ParseQueryObjNodeToSQL(); + if (errCode != E_OK) { + return errCode; + } + + // Limit needs to be placed after orderBy and processed separately in the limit branch + if (hasPrefixKey_ && !hasOrderBy_ && !hasLimit_ && isNeedOrderbyKey_) { + LOGD("Need add order by key at last when has prefixKey no need order by value and limit!"); + querySql_ += "ORDER BY key ASC"; + } + querySql_ += ";"; + return errCode; +} + +int SqliteQueryHelper::ToQuerySyncSql(bool hasSubQuery) +{ + int errCode = ParseQueryObjNodeToSQL(); + if (errCode != E_OK) { + return errCode; + } + + // Order by time when no order by and no limit and no need order by key. + if (!hasOrderBy_ && !hasLimit_ && !isNeedOrderbyKey_) { + querySql_ += "ORDER BY timestamp ASC"; + } + + if (!hasSubQuery) { + querySql_ += ";"; + } + return errCode; +} + +int SqliteQueryHelper::ToGetCountSql() +{ + countSql_.clear(); + if (queryObjNodes_.empty()) { + countSql_ += ";"; + return E_OK; + } + bool isNeedEndBracket = FilterSymbolToAddBracketLink(countSql_); + + int errCode = E_OK; + for (const QueryObjNode &objNode : queryObjNodes_) { + SymbolType symbolType = GetSymbolType(objNode.operFlag); + if (symbolType == SPECIAL_SYMBOL && isNeedEndBracket) { + countSql_ += ") "; + isNeedEndBracket = false; + } + + if (objNode.operFlag == QueryObjType::LIMIT) { + hasLimit_ = true; + continue; + } + if (objNode.operFlag == QueryObjType::ORDERBY) { + hasOrderBy_ = true; + continue; + } + errCode = ParseQueryExpression(objNode, countSql_); + if (errCode != E_OK) { + countSql_.clear(); + return errCode; + } + } + + if (isNeedEndBracket) { + countSql_ += ") "; + } + + // Limit needs to be placed after orderBy and processed separately in the limit branch + if (hasPrefixKey_ && !hasOrderBy_ && !hasLimit_ && isNeedOrderbyKey_) { + LOGD("Need add order by key at last when has prefixKey no need order by value and limit!"); + countSql_ += "ORDER BY key ASC"; + } + countSql_ += ";"; + return errCode; +} + +int SqliteQueryHelper::GetQuerySql(std::string &sql, bool onlyRowid) +{ + if (!isValid_) { + return -E_INVALID_QUERY_FORMAT; + } + + const std::string &querySqlForUse = (onlyRowid ? PRE_QUERY_ROWID_SQL : PRE_QUERY_KV_SQL); + sql = AssembleSqlForSuggestIndex(querySqlForUse, FILTER_NATIVE_DATA_SQL); + sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); + if (transformed_) { + LOGD("This query object has been parsed."); + sql += querySql_; + return E_OK; + } + int errCode = ToQuerySql(); + if (errCode != E_OK) { + LOGE("Transfer to query sql failed! errCode[%d]", errCode); + return errCode; + } + transformed_ = true; + sql += querySql_; + return errCode; +} + +int SqliteQueryHelper::GetSyncDataCheckSql(std::string &sql) +{ + int errCode = E_OK; + if (!isValid_) { + return -E_INVALID_QUERY_FORMAT; + } + sql = PRE_QUERY_ITEM_SQL + tableName_ + " WHERE hash_key=? AND (flag&0x01=0) "; + sql += hasPrefixKey_ ? " AND (key>=? AND key<=?) " : ""; + if (!transformed_) { + errCode = ToQuerySql(); + if (errCode != E_OK) { + LOGE("Transfer query to sync data check sql failed! errCode[%d]", errCode); + return errCode; + } + transformed_ = true; + } + sql += querySql_; + return errCode; +} + +int SqliteQueryHelper::BindSyncDataCheckStmt(sqlite3_stmt *statement, const Key &hashKey) const +{ + if (statement == nullptr) { + return -E_INVALID_ARGS; + } + int index = 1; // bind statement start index 1 + int errCode = SQLiteUtils::BindBlobToStatement(statement, index++, hashKey, false); + if (errCode != E_OK) { + LOGE("Get sync data check statement failed when bind hash key, errCode = %d", errCode); + return errCode; + } + if (hasPrefixKey_) { + // bind the prefix key for the first and second args. + errCode = SQLiteUtils::BindPrefixKey(statement, index, prefixKey_); + if (errCode != E_OK) { + LOGE("Get sync data check statement failed when bind prefix key, errCode = %d", errCode); + return errCode; + } + index += 2; // prefixKey takes 2 position + } + for (const QueryObjNode &objNode : queryObjNodes_) { + errCode = BindFieldValue(statement, objNode, index); + if (errCode != E_OK) { + LOGE("Get sync data check statement failed when bind field value, errCode = %d", errCode); + return errCode; + } + } + return errCode; +} + +int SqliteQueryHelper::GetCountQuerySql(std::string &sql) +{ + if (!isValid_) { + return -E_INVALID_QUERY_FORMAT; + } + + int errCode = ToGetCountSql(); + if (errCode != E_OK) { + return errCode; + } + sql = AssembleSqlForSuggestIndex(PRE_GET_COUNT_SQL, FILTER_NATIVE_DATA_SQL); + sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); + sql += countSql_; + return E_OK; +} + +int SqliteQueryHelper::GetQuerySqlStatement(sqlite3 *dbHandle, const std::string &sql, sqlite3_stmt *&statement) +{ + int errCode = SQLiteUtils::GetStatement(dbHandle, sql, statement); + if (errCode != E_OK) { + LOGE("[Query] Get statement fail!"); + return -E_INVALID_QUERY_FORMAT; + } + int index = 1; + if (hasPrefixKey_) { + // bind the prefix key for the first and second args. + errCode = SQLiteUtils::BindPrefixKey(statement, 1, prefixKey_); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + LOGE("[Query] Get statement when bind prefix key, errCode = %d", errCode); + return errCode; + } + index = 3; // begin from 3rd args + } + + for (const QueryObjNode &objNode : queryObjNodes_) { + errCode = BindFieldValue(statement, objNode, index); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + LOGE("[Query] Get statement fail when bind field value, errCode = %d", errCode); + return errCode; + } + } + return errCode; +} + +int SqliteQueryHelper::GetQuerySqlStatement(sqlite3 *dbHandle, bool onlyRowid, sqlite3_stmt *&statement) +{ + std::string sql; + int errCode = GetQuerySql(sql, onlyRowid); + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::GetStatement(dbHandle, sql, statement); + if (errCode != E_OK) { + LOGE("[Query] Get statement fail!"); + return -E_INVALID_QUERY_FORMAT; + } + int index = 1; + if (hasPrefixKey_) { + // bind the prefix key for the first and second args. + errCode = SQLiteUtils::BindPrefixKey(statement, 1, prefixKey_); + if (errCode != E_OK) { + LOGE("[Query] Get statement when bind prefix key, errCode = %d", errCode); + return errCode; + } + index = 3; // begin from 3rd args + } + + for (const QueryObjNode &objNode : queryObjNodes_) { + errCode = BindFieldValue(statement, objNode, index); + if (errCode != E_OK) { + LOGE("[Query] Get statement fail when bind field value, errCode = %d", errCode); + return errCode; + } + } + return errCode; +} + +int SqliteQueryHelper::GetCountSqlStatement(sqlite3 *dbHandle, sqlite3_stmt *&countStmt) +{ + std::string countSql; + int errCode = GetCountQuerySql(countSql); + if (errCode != E_OK) { + return errCode; + } + + // bind statement for count + errCode = SQLiteUtils::GetStatement(dbHandle, countSql, countStmt); + if (errCode != E_OK) { + LOGE("Get count statement error:%d", errCode); + return -E_INVALID_QUERY_FORMAT; + } + int index = 1; + if (hasPrefixKey_) { + // bind the prefix key for the first and second args. + errCode = SQLiteUtils::BindPrefixKey(countStmt, 1, prefixKey_); + if (errCode != E_OK) { + LOGE("[Query] Get count statement fail when bind prefix key, errCode = %d", errCode); + return errCode; + } + index = 3; // begin from 3rd args + } + + for (const QueryObjNode &objNode : queryObjNodes_) { + if (GetSymbolType(objNode.operFlag) == SPECIAL_SYMBOL) { + continue; + } + errCode = BindFieldValue(countStmt, objNode, index); + if (errCode != E_OK) { + LOGE("[Query] Get count statement fail when bind field value, errCode = %d", errCode); + return errCode; + } + } + return errCode; +} + +int SqliteQueryHelper::GetSyncDataQuerySql(std::string &sql, bool hasSubQuery) +{ + if (!isValid_) { + return -E_INVALID_QUERY_FORMAT; + } + + if (hasLimit_) { + hasSubQuery = true; // Need sub query. + } else { + isNeedOrderbyKey_ = false; // Need order by timestamp. + } + + sql = AssembleSqlForSuggestIndex(PRE_QUERY_ITEM_SQL + tableName_ + " ", FILTER_REMOTE_QUERY); + sql = !hasPrefixKey_ ? sql : (sql + " AND (key>=? AND key<=?) "); + sql = hasSubQuery ? sql : (sql + " AND (timestamp>=? AND timestamp= ? AND timestamp < ?) ORDER BY timestamp;"; + } + return errCode; +} + +int SqliteQueryHelper::BindTimeRange(sqlite3_stmt *&statement, int &index, uint64_t beginTime, uint64_t endTime) const +{ + int errCode = SQLiteUtils::BindInt64ToStatement(statement, index++, beginTime); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; + } + + errCode = SQLiteUtils::BindInt64ToStatement(statement, index++, endTime); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + } + return errCode; +} + +int SqliteQueryHelper::BindObjNodes(sqlite3_stmt *&statement, int &index) const +{ + int errCode = E_OK; + for (const QueryObjNode &objNode : queryObjNodes_) { + errCode = BindFieldValue(statement, objNode, index); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + LOGE("[Query] Get statement fail when bind field value, errCode = %d", errCode); + break; + } + } + return errCode; +} + +int SqliteQueryHelper::GetQuerySyncStatement(sqlite3 *dbHandle, uint64_t beginTime, uint64_t endTime, + sqlite3_stmt *&statement) +{ + bool hasSubQuery = false; + if (hasLimit_) { + hasSubQuery = true; // Need sub query. + } else { + isNeedOrderbyKey_ = false; // Need order by timestamp. + } + std::string sql; + int errCode = GetSyncDataQuerySql(sql, hasSubQuery); + if (errCode != E_OK) { + LOGE("[Query] Get SQL fail!"); + return -E_INVALID_QUERY_FORMAT; + } + + errCode = SQLiteUtils::GetStatement(dbHandle, sql, statement); + if (errCode != E_OK) { + LOGE("[Query] Get statement fail!"); + return -E_INVALID_QUERY_FORMAT; + } + + int index = 1; // begin with 1. + if (hasPrefixKey_) { + // bind the prefix key for the first and second args. + errCode = SQLiteUtils::BindPrefixKey(statement, index, prefixKey_); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + LOGE("[Query] Get statement when bind prefix key, errCode = %d", errCode); + return errCode; + } + index = 3; // begin with 3 next if prefix key exists. + } + + if (hasSubQuery) { + // For sub query SQL, timestamp must be last : (prefix key), (objNodes), timestamp. + // SQL: SELECT * FROM ( SELECT * FROM sync_data WHERE (flag&0x03=0x02) LIMIT 10 OFFSET 0 ) WHERE (timestamp>=? + // AND timestamp=? AND timestamp MAX_SQLITE_BIND_SIZE) { + return -E_MAX_LIMITS; + } + errCode = sqlite3_bind_text(statement, index, queryNode.fieldValue[i].stringValue.c_str(), + queryNode.fieldValue[i].stringValue.size(), SQLITE_TRANSIENT); + } + if (errCode != SQLITE_OK) { + break; + } + index++; + } + return SQLiteUtils::MapSQLiteErrno(errCode); +} + +std::string SqliteQueryHelper::MapCastTypeSql(const FieldType &type) const +{ + switch (type) { + case FieldType::LEAF_FIELD_BOOL: + case FieldType::LEAF_FIELD_INTEGER: + case FieldType::LEAF_FIELD_LONG: + return "INT"; + case FieldType::LEAF_FIELD_DOUBLE: + return "REAL"; + case FieldType::LEAF_FIELD_STRING: + return "TEXT"; + case FieldType::LEAF_FIELD_NULL: + return "NULL"; + default: + return ""; + } +} + +int SqliteQueryHelper::ParseQueryExpression(const QueryObjNode &queryNode, std::string &querySql, + const std::string &accessStr, bool placeholder) +{ + SymbolType symbolType = GetSymbolType(queryNode.operFlag); + if (symbolType == RANGE_SYMBOL && queryNode.fieldValue.size() > MAX_CONDITIONS_SIZE) { + LOGE("[Query][Parse][Expression] conditions is too many!"); + return -E_MAX_LIMITS; + } + + if (symbolType == COMPARE_SYMBOL || symbolType == RELATIONAL_SYMBOL || symbolType == RANGE_SYMBOL) { + querySql += MapCastFuncSql(queryNode, accessStr); + querySql += MapRelationalSymbolToSql(queryNode, placeholder); + } else if (symbolType == LOGIC_SYMBOL || symbolType == LINK_SYMBOL) { + querySql += MapLogicSymbolToSql(queryNode); + } else { + querySql += MapKeywordSymbolToSql(queryNode); + } + + if (querySql.size() > MAX_SQL_LEN) { + LOGE("[Query][Parse][Expression] Sql is too long!"); + return -E_MAX_LIMITS; + } + return E_OK; +} + +std::string SqliteQueryHelper::AssembleSqlForSuggestIndex(const std::string &baseSql, const std::string &filter) const +{ + std::string formatIndex = CheckAndFormatSuggestIndex(); + if (formatIndex.empty()) { + return baseSql + filter; + } + + return baseSql + USING_INDEX + "'" + formatIndex + "' " + filter; +} + +std::string SqliteQueryHelper::CheckAndFormatSuggestIndex() const +{ + if (suggestIndex_.empty()) { + return ""; + } + IndexName indexName; + int errCode = SchemaUtils::ParseAndCheckFieldPath(suggestIndex_, indexName); + if (errCode != E_OK) { + LOGW("Check and format suggest index failed! %d", errCode); + return ""; + } + + if (!schema_.IsIndexExist(indexName)) { + LOGW("The suggest index not exist!"); + return ""; + } + return SchemaUtils::FieldPathString(indexName); +} + +int SqliteQueryHelper::GetSubscribeCondition(const std::string &accessStr, std::string &conditionStr) +{ + if (queryObjNodes_.empty()) { + conditionStr += " (1 = 1) "; + return E_OK; + } + conditionStr += "("; + if (hasPrefixKey_) { + conditionStr += "(hex(" + accessStr + "key) LIKE '" + DBCommon::VectorToHexString(prefixKey_) + "%')"; + } + bool isNeedEndBracket = FilterSymbolToAddBracketLink(conditionStr, hasPrefixKey_); + int errCode = E_OK; + for (const QueryObjNode &objNode : queryObjNodes_) { + SymbolType symbolType = GetSymbolType(objNode.operFlag); + if (symbolType == SPECIAL_SYMBOL && isNeedEndBracket) { + conditionStr += ") "; + isNeedEndBracket = false; + } + errCode = ParseQueryExpression(objNode, conditionStr, accessStr, false); + if (errCode != E_OK) { + conditionStr.clear(); + return errCode; + } + } + + if (isNeedEndBracket) { + conditionStr += ") "; + } + conditionStr += ")"; + return errCode; +} + +int SqliteQueryHelper::GetSubscribeSql(const std::string &subscribeId, TriggerModeEnum mode, + std::string &subscribeCondition) +{ + if (!isValid_) { + return -E_INVALID_QUERY_FORMAT; + } + int errCode = E_OK; + switch (mode) { + case TriggerModeEnum::INSERT: + errCode = GetSubscribeCondition(DBConstant::TRIGGER_REFERENCES_NEW, subscribeCondition); + break; + case TriggerModeEnum::UPDATE: + errCode = GetSubscribeCondition(DBConstant::TRIGGER_REFERENCES_OLD, subscribeCondition); + if (errCode != E_OK) { + break; + } + subscribeCondition += " OR "; + errCode = GetSubscribeCondition(DBConstant::TRIGGER_REFERENCES_NEW, subscribeCondition); + break; + case TriggerModeEnum::DELETE: + errCode = GetSubscribeCondition(DBConstant::TRIGGER_REFERENCES_OLD, subscribeCondition); + break; + default: + errCode = -INVALID_ARGS; + } + if (errCode != E_OK) { + LOGD("Get subscribe query condition failed. %d", errCode); + } + return errCode; +} +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.h new file mode 100644 index 0000000000000000000000000000000000000000..c0fe47db27ec4d3843d9c5e9a89025c5fba49c92 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_query_helper.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SQLITE_QUERY_HELPER_H +#define SQLITE_QUERY_HELPER_H + +#include +#include +#include + +#include "query_expression.h" +#include "schema_utils.h" +#include "sqlite_import.h" + +namespace DistributedDB { +namespace TriggerMode { +enum class TriggerModeEnum; +} +struct QueryObjInfo { + SchemaObject schema_; + std::list queryObjNodes_; + std::vector prefixKey_; + std::string suggestIndex_; + int orderByCounts_ = 0; // Record processing to which orderBy node + bool isValid_ = true; + bool hasOrderBy_ = false; + bool hasLimit_ = false; + bool hasPrefixKey_ = false; + std::string tableName_; +}; + +enum SymbolType : uint32_t { + INVALID_SYMBOL = 0x0000, + COMPARE_SYMBOL = 0x0100, // relation symbol use to compare + RELATIONAL_SYMBOL = 0x0200, + RANGE_SYMBOL = 0x0300, + PREFIXKEY_SYMBOL = 0x0400, + LOGIC_SYMBOL = 0x0500, + LINK_SYMBOL = 0x0600, // use to link relatonal symbol + SPECIAL_SYMBOL = 0x0700, // need special precess and need at the last + SUGGEST_INDEX_SYMBOL = 0x0800, +}; + +class SqliteQueryHelper final { +public: + explicit SqliteQueryHelper(const QueryObjInfo &info); + + // forbidden move constructor. + SqliteQueryHelper(SqliteQueryHelper &&) = delete; + SqliteQueryHelper &operator=(SqliteQueryHelper &&) = delete; + // forbidden copy constructor. + SqliteQueryHelper(const SqliteQueryHelper &) = delete; + SqliteQueryHelper &operator=(const SqliteQueryHelper &) = delete; + + ~SqliteQueryHelper() = default; + + int GetQuerySqlStatement(sqlite3 *dbHandle, bool onlyRowid, sqlite3_stmt *&statement); + int GetQuerySqlStatement(sqlite3 *dbHandle, const std::string &sql, sqlite3_stmt *&statement); + int GetCountSqlStatement(sqlite3 *dbHandle, sqlite3_stmt *&countStmt); + + // For query Sync + int GetQuerySyncStatement(sqlite3 *dbHandle, uint64_t beginTime, uint64_t endTime, sqlite3_stmt *&statement); + int GetSyncDataCheckSql(std::string &sql); + int BindSyncDataCheckStmt(sqlite3_stmt *statement, const Key &hashKey) const; + + int GetSubscribeSql(const std::string &subscribeId, TriggerMode::TriggerModeEnum mode, + std::string &subscribeCondition); + + static SymbolType GetSymbolType(const QueryObjType &queryObjType); + + // public for unit test + int GetQuerySql(std::string &sql, bool onlyRowid); + int GetCountQuerySql(std::string &sql); + + const std::string &GetTableName() + { + return tableName_; + } + +private: + int ToQuerySql(); + int ToQuerySyncSql(bool hasSubQuery); + int ToGetCountSql(); + int ParseQueryExpression(const QueryObjNode &queryNode, std::string &querySql, + const std::string &accessStr = "", bool placeholder = true); + std::string MapRelationalSymbolToSql(const QueryObjNode &queryNode, bool placeholder = false) const; + std::string MapKeywordSymbolToSql(const QueryObjNode &queryNode); + std::string MapLogicSymbolToSql(const QueryObjNode &queryNode) const; + std::string MapValueToSql(const QueryObjNode &queryNode, bool placeholder) const; + std::string MapCastFuncSql(const QueryObjNode &queryNode, const std::string &accessStr = ""); + std::string MapCastTypeSql(const FieldType &type) const; + int BindFieldValue(sqlite3_stmt *statement, const QueryObjNode &queryNode, int &index) const; + bool FilterSymbolToAddBracketLink(std::string &querySql, bool isNeedLink = true) const; + std::string AssembleSqlForSuggestIndex(const std::string &baseSql, const std::string &filter) const; + std::string CheckAndFormatSuggestIndex() const; + int GetSyncDataQuerySql(std::string &sql, bool hasSubQuery); + int ParseQueryObjNodeToSQL(); + int BindTimeRange(sqlite3_stmt *&statement, int &index, uint64_t beginTime, uint64_t endTime) const; + int BindObjNodes(sqlite3_stmt *&statement, int &index) const; + int GetSubscribeCondition(const std::string &accessStr, std::string &conditionStr); + + SchemaObject schema_; + std::list queryObjNodes_; + std::vector prefixKey_; + std::string suggestIndex_; + std::string tableName_; + + std::string querySql_; + std::string countSql_; + + int orderByCounts_; // Record processing to which orderBy node + bool isValid_; + bool transformed_; + bool hasOrderBy_; + bool hasLimit_; + bool isOrderByAppeared_; + bool hasPrefixKey_; + bool isNeedOrderbyKey_; // The tag field is used for prefix query filtering key sorting +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6f7a8482667b97c60069ef49db3f45f847e26c85 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sqlite_single_ver_continue_token.h" + +namespace DistributedDB { +SQLiteSingleVerContinueToken::SQLiteSingleVerContinueToken(TimeStamp begin, TimeStamp end) + : timeRanges_(MulDevTimeRanges{{"", {begin, end}}}) +{} + +SQLiteSingleVerContinueToken::SQLiteSingleVerContinueToken( + const SyncTimeRange &timeRange, const QueryObject &queryObject) + : queryObject_(std::map{{"", queryObject}}), + timeRanges_(MulDevTimeRanges{{"", {timeRange.beginTime, timeRange.endTime}}}), + deleteTimeRanges_(MulDevTimeRanges{{"", {timeRange.deleteBeginTime, timeRange.deleteEndTime}}}) +{} + +SQLiteSingleVerContinueToken::SQLiteSingleVerContinueToken(MulDevTimeRanges timeRanges) + : timeRanges_(timeRanges) +{} + +SQLiteSingleVerContinueToken::~SQLiteSingleVerContinueToken() +{} + +bool SQLiteSingleVerContinueToken::CheckValid() const +{ + return ((magicBegin_ == MAGIC_BEGIN) && (magicEnd_ == MAGIC_END)); +} + +TimeStamp SQLiteSingleVerContinueToken::GetQueryBeginTime() const +{ + return GetBeginTimeStamp(timeRanges_); +} + +TimeStamp SQLiteSingleVerContinueToken::GetQueryEndTime() const +{ + return GetEndTimeStamp(timeRanges_); +} + +TimeStamp SQLiteSingleVerContinueToken::GetDeletedBeginTime() const +{ + return GetBeginTimeStamp(deleteTimeRanges_); +} + +TimeStamp SQLiteSingleVerContinueToken::GetDeletedEndTime() const +{ + return GetEndTimeStamp(deleteTimeRanges_); +} + +void SQLiteSingleVerContinueToken::SetNextBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime) +{ + RemovePrevDevAndSetBeginTime(deviceID, nextBeginTime, timeRanges_); +} + +const MulDevTimeRanges& SQLiteSingleVerContinueToken::GetTimeRanges() +{ + return timeRanges_; +} + +void SQLiteSingleVerContinueToken::SetDeletedNextBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime) +{ + RemovePrevDevAndSetBeginTime(deviceID, nextBeginTime, deleteTimeRanges_); +} + +const MulDevTimeRanges& SQLiteSingleVerContinueToken::GetDeletedTimeRanges() const +{ + return deleteTimeRanges_; +} + +void SQLiteSingleVerContinueToken::FinishGetQueryData() +{ + timeRanges_.clear(); +} + +void SQLiteSingleVerContinueToken::FinishGetDeletedData() +{ + deleteTimeRanges_.clear(); +} + +bool SQLiteSingleVerContinueToken::IsGetQueryDataFinished() const +{ + return timeRanges_.empty(); +} + +bool SQLiteSingleVerContinueToken::IsGetDeletedDataFinished() const +{ + return deleteTimeRanges_.empty(); +} + +bool SQLiteSingleVerContinueToken::IsQuerySync() const +{ + return !queryObject_.empty(); +} + +QueryObject SQLiteSingleVerContinueToken::GetQuery() const +{ + if (!queryObject_.empty()) { + return queryObject_.begin()->second; + } + return QueryObject{}; +} + +void SQLiteSingleVerContinueToken::RemovePrevDevAndSetBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime, + MulDevTimeRanges &timeRanges) +{ + auto iter = timeRanges.find(deviceID); + if (iter == timeRanges.end()) { + return; + } + iter = timeRanges.erase(timeRanges.begin(), iter); + iter->second.first = nextBeginTime; +} + +TimeStamp SQLiteSingleVerContinueToken::GetBeginTimeStamp(const MulDevTimeRanges &timeRanges) const +{ + if (timeRanges.empty()) { + return 0; + } + return timeRanges.begin()->second.first; +} + +TimeStamp SQLiteSingleVerContinueToken::GetEndTimeStamp(const MulDevTimeRanges &timeRanges) const +{ + if (timeRanges.empty()) { + return static_cast(INT64_MAX); + } + return timeRanges.begin()->second.second; +} +} // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.h new file mode 100644 index 0000000000000000000000000000000000000000..a4fa7751703c653c6359deb7beed06c398b45c0a --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_continue_token.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SQLITE_SINGLE_VER_CONTINUE_TOKEN_H +#define SQLITE_SINGLE_VER_CONTINUE_TOKEN_H + +#include + +#include "db_types.h" +#include "query_object.h" +#include "single_ver_kvdb_sync_interface.h" + +namespace DistributedDB { +class SQLiteSingleVerContinueToken { +public: + // For one device. + SQLiteSingleVerContinueToken(TimeStamp begin, TimeStamp end); + + // For one device in query sync. + SQLiteSingleVerContinueToken(const SyncTimeRange &timeRange, const QueryObject &queryObject); + + // For multiple device. + explicit SQLiteSingleVerContinueToken(MulDevTimeRanges timeRanges); + + ~SQLiteSingleVerContinueToken(); + + /* + * function: Check the magic number at the beginning and end of the SingleVerContinueToken. + * returnValue: Return true if the begin and end magic number is OK. + * Return false if the begin or end magic number is error. + */ + bool CheckValid() const; + + TimeStamp GetQueryBeginTime() const; + TimeStamp GetQueryEndTime() const; + TimeStamp GetDeletedBeginTime() const; + TimeStamp GetDeletedEndTime() const; + + void SetNextBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime); + const MulDevTimeRanges &GetTimeRanges(); + void SetDeletedNextBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime); + const MulDevTimeRanges &GetDeletedTimeRanges() const; + + void FinishGetQueryData(); + void FinishGetDeletedData(); + + bool IsGetQueryDataFinished() const; + bool IsGetDeletedDataFinished() const; + + bool IsQuerySync() const; + QueryObject GetQuery() const; + +private: + void RemovePrevDevAndSetBeginTime(const DeviceID &deviceID, TimeStamp nextBeginTime, MulDevTimeRanges &timeRanges); + + TimeStamp GetBeginTimeStamp(const MulDevTimeRanges &timeRanges) const; + TimeStamp GetEndTimeStamp(const MulDevTimeRanges &timeRanges) const; + + static const unsigned int MAGIC_BEGIN = 0x600D0AC7; // for token guard + static const unsigned int MAGIC_END = 0x0AC7600D; // for token guard + unsigned int magicBegin_ = MAGIC_BEGIN; + std::map queryObject_; + MulDevTimeRanges timeRanges_; + MulDevTimeRanges deleteTimeRanges_; + unsigned int magicEnd_ = MAGIC_END; + + std::map>> mulDevTableTimeRange_; + std::map>> mulDevTableDeletedTimeRange_; +}; +} // namespace DistributedDB +#endif // SQLITE_SINGLE_VER_CONTINUE_TOKEN_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp index 3ad49870751a96ed4353a538519a87cdf817ab57..70d7cec2c01fea10bbc0c8b1300bb5b9c77eb5d5 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp @@ -238,9 +238,9 @@ void SQLiteSingleVerDatabaseUpgrader::SetMetaUpgrade(const SecurityOption &curre } } -void SQLiteSingleVerDatabaseUpgrader::SetSubdir(const std::string &parentDir) +void SQLiteSingleVerDatabaseUpgrader::SetSubdir(const std::string &subDir) { - subDir_ = parentDir; + subDir_ = subDir; } int SQLiteSingleVerDatabaseUpgrader::SetPathSecOptWithCheck(const std::string &path, const SecurityOption &secOption, diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.h index 28ce76587db8dbda9cf3e61d2b1ade0b1564a1c2..3d5190e4458590a7654ca821f24497ef6790bb76 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_database_upgrader.h @@ -32,7 +32,7 @@ public: static int CreateDbDir(); void SetMetaUpgrade(const SecurityOption ¤tOpt, const SecurityOption &expectOpt, const std::string &subDir); - void SetSubdir(const std::string &subdir); + void SetSubdir(const std::string &subDir); static int SetPathSecOptWithCheck(const std::string &path, const SecurityOption &secOption, const std::string &dbStore, bool isWithChecked = false); static int SetSecOption(const std::string &path, const SecurityOption &secOption, bool isWithChecked); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp index 8f38ccbb006d6005019f4b90a8d99d9239a3f3e4..80a98f372fd1b178679779820e63017a6b98af8c 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.cpp @@ -19,23 +19,20 @@ #include #include +#include "data_compression.h" #include "db_common.h" #include "db_constant.h" -#include "sqlite_utils.h" -#include "storage_engine_manager.h" -#include "sqlite_single_ver_storage_engine.h" -#include "sqlite_single_ver_natural_store_connection.h" #include "db_errno.h" +#include "generic_single_ver_kv_entry.h" +#include "intercepted_data_impl.h" +#include "kvdb_utils.h" #include "log_print.h" -#include "value_hash_calc.h" -#include "hash.h" #include "platform_specific.h" -#include "db_constant.h" -#include "package_file.h" -#include "generic_single_ver_kv_entry.h" #include "schema_object.h" -#include "kvdb_utils.h" #include "single_ver_database_oper.h" +#include "storage_engine_manager.h" +#include "sqlite_single_ver_natural_store_connection.h" +#include "value_hash_calc.h" namespace DistributedDB { #define CHECK_STORAGE_ENGINE do { \ @@ -45,72 +42,83 @@ namespace DistributedDB { } while (0) namespace { - constexpr int MAX_SYNC_BLOCK_SIZE = 31457280; // 30MB constexpr int DEF_LIFE_CYCLE_TIME = 60000; // 60s constexpr int WAIT_DELEGATE_CALLBACK_TIME = 100; - - // Currently this function only suitable to be call from sync in insert_record_from_sync procedure - // Take attention if future coder attempt to call it in other situation procedure - int SaveSyncItems(SQLiteSingleVerStorageExecutor *handle, std::vector &dataItems, - const DeviceInfo &deviceInfo, TimeStamp &maxTimestamp, SingleVerNaturalStoreCommitNotifyData *commitData) + // In querySync, when getting query data finished, + // if the block size reach the half of max block size, will get deleted data next; + // if the block size not reach the half of max block size, will not get deleted data. + constexpr float QUERY_SYNC_THRESHOLD = 0.50; + + const std::string CREATE_DB_TIME = "createDBTime"; + + // Called when get multiple dev data. + // deviceID is the device which currently being getting. When getting one dev data, deviceID is "". + // dataItems is the DataItems which already be get from DB sorted by timestamp. + // token must not be null. + void ProcessContinueToken(const DeviceID &deviceID, const std::vector &dataItems, int &errCode, + SQLiteSingleVerContinueToken *&token) { - int errCode = handle->StartTransaction(TransactType::IMMEDIATE); - if (errCode != E_OK) { - return errCode; - } - - int innerCode; - errCode = handle->PrepareForSavingData(SingleVerDataType::SYNC_TYPE); - if (errCode != E_OK) { - goto END; + if (errCode != -E_UNFINISHED) { // Error happened or get data finished. Token should be cleared. + delete token; + token = nullptr; + return; } - for (auto &item : dataItems) { - if (item.neglect) { // Do not save this record if it is neglected - continue; - } - errCode = handle->SaveSyncDataItem(item, deviceInfo, maxTimestamp, commitData); - if (errCode != E_OK && errCode != -E_NOT_FOUND) { - break; - } + if (dataItems.empty()) { + errCode = -E_INTERNAL_ERROR; + LOGE("Get data unfinished but dataitems is empty."); + delete token; + token = nullptr; + return; } - if (errCode == -E_NOT_FOUND) { - errCode = E_OK; + TimeStamp nextBeginTime = dataItems.back().timeStamp + 1; + if (nextBeginTime > INT64_MAX) { + nextBeginTime = INT64_MAX; } + token->SetNextBeginTime(deviceID, nextBeginTime); + return; + } - innerCode = handle->ResetForSavingData(SingleVerDataType::SYNC_TYPE); - if (innerCode != E_OK) { - errCode = innerCode; - } - END: - if (errCode == E_OK) { - errCode = handle->Commit(); - } else { - (void)handle->Rollback(); // Keep the error code of the first scene - } - return errCode; + // Called when get one dev data. + void ProcessContinueToken(const std::vector &dataItems, int &errCode, + SQLiteSingleVerContinueToken *&token) + { + ProcessContinueToken("", dataItems, errCode, token); } - void ProcessContinueToken(int &errCode, TimeStamp end, const std::vector &dataItems, - ContinueTokenStruct *&token) + // Called when get query sync data. + // dataItems is the DataItems which already be get from DB sorted by timestamp. + // token must not be null. + void ProcessContinueTokenForQuerySync(const std::vector &dataItems, int &errCode, + SQLiteSingleVerContinueToken *&token) { - if (token == nullptr) { + if (errCode != -E_UNFINISHED) { // Error happened or get data finished. Token should be cleared. + delete token; + token = nullptr; return; } - if (!dataItems.empty()) { - TimeStamp timestamp = dataItems.back().timeStamp; - if (timestamp > INT64_MAX - 1) { - token->SetBeginTimeStamp(INT64_MAX); - } else { - token->SetBeginTimeStamp(timestamp + 1); // Add 1 for the timestamp. - } - token->SetEndTimeStamp(end); - } else { + + if (dataItems.empty()) { + errCode = -E_INTERNAL_ERROR; + LOGE("Get data unfinished but dataitems is empty."); delete token; token = nullptr; - errCode = -E_INTERNAL_ERROR; + return; + } + + TimeStamp nextBeginTime = dataItems.back().timeStamp + 1; + if (nextBeginTime > INT64_MAX) { + nextBeginTime = INT64_MAX; + } + bool getDeleteData = ((dataItems.back().flag & DataItem::DELETE_FLAG) != 0); + if (getDeleteData) { + token->FinishGetQueryData(); + token->SetDeletedNextBeginTime("", nextBeginTime); + } else { + token->SetNextBeginTime("", nextBeginTime); } + return; } void UpdateSecProperties(KvDBProperties &properties, bool isReadOnly, const SchemaObject &savedSchemaObj, @@ -129,6 +137,34 @@ namespace { properties.SetIntProp(KvDBProperties::SECURITY_LABEL, engine->GetSecurityOption().securityLabel); properties.SetIntProp(KvDBProperties::SECURITY_FLAG, engine->GetSecurityOption().securityFlag); } + + int GetKvEntriesByDataItems(std::vector &entries, std::vector &dataItems) + { + int errCode = E_OK; + for (auto &item : dataItems) { + auto entry = new (std::nothrow) GenericSingleVerKvEntry(); + if (entry == nullptr) { + errCode = -E_OUT_OF_MEMORY; + LOGE("GetKvEntries failed, errCode:%d", errCode); + SingleVerKvEntry::Release(entries); + break; + } + entry->SetEntryData(std::move(item)); + entries.push_back(entry); + } + return errCode; + } + + bool CanHoldDeletedData(const std::vector &dataItems, const DataSizeSpecInfo &dataSizeInfo, + size_t appendLen) + { + bool reachThreshold = false; + for (size_t i = 0, blockSize = 0; !reachThreshold && i < dataItems.size(); i++) { + blockSize += SQLiteSingleVerStorageExecutor::GetDataItemSerialSize(dataItems[i], appendLen); + reachThreshold = (blockSize >= dataSizeInfo.blockSize * QUERY_SYNC_THRESHOLD); + } + return !reachThreshold; + } } SQLiteSingleVerNaturalStore::SQLiteSingleVerNaturalStore() @@ -140,7 +176,9 @@ SQLiteSingleVerNaturalStore::SQLiteSingleVerNaturalStore() isReadOnly_(false), lifeCycleNotifier_(nullptr), lifeTimerId_(0), - autoLifeTime_(DEF_LIFE_CYCLE_TIME) + autoLifeTime_(DEF_LIFE_CYCLE_TIME), + createDBTime_(0), + dataInterceptor_(nullptr) {} SQLiteSingleVerNaturalStore::~SQLiteSingleVerNaturalStore() @@ -150,9 +188,7 @@ SQLiteSingleVerNaturalStore::~SQLiteSingleVerNaturalStore() std::string SQLiteSingleVerNaturalStore::GetDatabasePath(const KvDBProperties &kvDBProp) { - std::string dataDir = kvDBProp.GetStringProp(KvDBProperties::DATA_DIR, ""); - std::string identifierDir = kvDBProp.GetStringProp(KvDBProperties::IDENTIFIER_DIR, ""); - std::string filePath = dataDir + "/" + identifierDir + "/" + DBConstant::SINGLE_SUB_DIR + "/" + + std::string filePath = GetSubDirPath(kvDBProp) + "/" + DBConstant::MAINDB_DIR + "/" + DBConstant::SINGLE_VER_DATA_STORE + DBConstant::SQLITE_DB_EXTENSION; return filePath; } @@ -206,7 +242,7 @@ int SQLiteSingleVerNaturalStore::RegisterLifeCycleCallback(const DatabaseLifeCyc return E_OK; } - if (notifier && lifeTimerId_ != 0) { + if (lifeTimerId_ != 0) { errCode = StopLifeCycleTimer(); if (errCode != E_OK) { LOGE("Stop the life cycle timer failed:%d", errCode); @@ -254,11 +290,6 @@ int SQLiteSingleVerNaturalStore::GetSecurityOption(SecurityOption &option) const return E_OK; } -bool SQLiteSingleVerNaturalStore::IsReadable() const -{ - return true; -} - namespace { inline bool OriValueCanBeUse(int errCode) { @@ -270,14 +301,11 @@ inline bool AmendValueShouldBeUse(int errCode) return (errCode == -E_VALUE_MATCH_AMENDED); } -inline bool ValueIsSomehowWrong(int errCode) +inline bool IsValueMismatched(int errCode) { - if (errCode == -E_VALUE_MISMATCH_FEILD_COUNT || - errCode == -E_VALUE_MISMATCH_FEILD_TYPE || - errCode == -E_VALUE_MISMATCH_CONSTRAINT) { - return true; - } - return false; + return (errCode == -E_VALUE_MISMATCH_FEILD_COUNT || + errCode == -E_VALUE_MISMATCH_FEILD_TYPE || + errCode == -E_VALUE_MISMATCH_CONSTRAINT); } } @@ -315,7 +343,7 @@ int SQLiteSingleVerNaturalStore::CheckValueAndAmendIfNeed(ValueSource sourceType useAmendValue = true; return E_OK; } - if (ValueIsSomehowWrong(errCode)) { + if (IsValueMismatched(errCode)) { return errCode; } } else { @@ -339,7 +367,6 @@ int SQLiteSingleVerNaturalStore::ClearIncompleteDatabase(const KvDBProperties &k return -E_REMOVE_FILE; } } - return E_OK; } @@ -378,6 +405,27 @@ int SQLiteSingleVerNaturalStore::CheckDatabaseRecovery(const KvDBProperties &kvD return errCode; } +int SQLiteSingleVerNaturalStore::GetAndInitStorageEngine(const KvDBProperties &kvDBProp) +{ + int errCode = E_OK; + storageEngine_ = + static_cast(StorageEngineManager::GetStorageEngine(kvDBProp, errCode)); + if (storageEngine_ == nullptr) { + return errCode; + } + + if (storageEngine_->IsEngineCorrupted()) { + LOGE("[SqlSinStore][GetAndInitStorageEngine] database engine is corrupted, not need continue to open!"); + return -E_INVALID_PASSWD_OR_CORRUPTED_DB; + } + + errCode = InitDatabaseContext(kvDBProp); + if (errCode != E_OK) { + LOGE("[SqlSinStore][Open] Init database context fail! errCode = [%d]", errCode); + } + return errCode; +} + int SQLiteSingleVerNaturalStore::Open(const KvDBProperties &kvDBProp) { std::lock_guard lock(initialMutex_); @@ -392,28 +440,24 @@ int SQLiteSingleVerNaturalStore::Open(const KvDBProperties &kvDBProp) bool isReadOnly = false; SchemaObject savedSchemaObj; - storageEngine_ = - static_cast(StorageEngineManager::GetStorageEngine(kvDBProp, errCode)); - if (errCode != E_OK) { - goto ERROR; - } - if (storageEngine_->IsEngineCorrupted()) { - errCode = -E_INVALID_PASSWD_OR_CORRUPTED_DB; - LOGE("[SqlSinStore][Open] database engine is corrupted, not need continue to open! errCode = [%d]", errCode); + errCode = GetAndInitStorageEngine(kvDBProp); + if (errCode != E_OK) { goto ERROR; } - errCode = InitDatabaseContext(kvDBProp); + errCode = RegisterNotification(); if (errCode != E_OK) { - LOGE("[SqlSinStore][Open] Init database context fail! errCode = [%d]", errCode); + LOGE("Register notification failed:%d", errCode); goto ERROR; } - errCode = RegisterNotification(); + + errCode = RemoveAllSubscribe(); if (errCode != E_OK) { - LOGE("Register notification failed:%d", errCode); + LOGE("[SqlSinStore][Open] remove subscribe fail! errCode = [%d]", errCode); goto ERROR; } + // Here, the dbfile is created or opened, and upgrade of table structure has done. // More, Upgrade of schema is also done in upgrader call in InitDatabaseContext, schema in dbfile updated if need. // If inputSchema is empty, upgrader do nothing of schema, isReadOnly will be true if dbfile contain schema before. @@ -429,6 +473,13 @@ int SQLiteSingleVerNaturalStore::Open(const KvDBProperties &kvDBProp) UpdateSecProperties(MyProp(), isReadOnly, savedSchemaObj, storageEngine_); StartSyncer(); + OnKill([this]() { ReleaseResources(); }); + + errCode = SaveCreateDBTimeIfNotExisted(); + if (errCode != E_OK) { + goto ERROR; + } + InitialLocalDataTimestamp(); isInitialized_ = true; isReadOnly_ = isReadOnly; @@ -440,7 +491,6 @@ ERROR: void SQLiteSingleVerNaturalStore::Close() { - SyncAbleKvDB::Close(); ReleaseResources(); } @@ -529,6 +579,34 @@ int SQLiteSingleVerNaturalStore::PutMetaData(const Key &key, const Value &value) return errCode; } +// Delete multiple meta data records in a transaction. +int SQLiteSingleVerNaturalStore::DeleteMetaData(const std::vector &keys) +{ + for (const auto &key : keys) { + if (key.empty() || key.size() > DBConstant::MAX_KEY_SIZE) { + return -E_INVALID_ARGS; + } + } + int errCode = E_OK; + auto handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + handle->StartTransaction(TransactType::IMMEDIATE); + errCode = handle->DeleteMetaData(keys); + if (errCode != E_OK) { + handle->Rollback(); + LOGE("[SinStore] DeleteMetaData failed, errCode = %d", errCode); + } else { + handle->Commit(); + } + + ReleaseHandle(handle); + HeartBeatForLifeCycle(); + return errCode; +} + int SQLiteSingleVerNaturalStore::GetAllMetaKeys(std::vector &keys) const { CHECK_STORAGE_ENGINE; @@ -592,11 +670,7 @@ int SQLiteSingleVerNaturalStore::GetSyncData(TimeStamp begin, TimeStamp end, std ERROR: if (errCode != E_OK && errCode != -E_UNFINISHED) { - for (auto &itemEntry : entries) { - delete itemEntry; - itemEntry = nullptr; - } - entries.clear(); + SingleVerKvEntry::Release(entries); } HeartBeatForLifeCycle(); return errCode; @@ -605,11 +679,11 @@ ERROR: int SQLiteSingleVerNaturalStore::GetSyncData(TimeStamp begin, TimeStamp end, std::vector &dataItems, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const { - if (begin >= end || dataSizeInfo.blockSize > MAX_SYNC_BLOCK_SIZE) { + if (begin >= end || dataSizeInfo.blockSize > DBConstant::MAX_SYNC_BLOCK_SIZE) { return -E_INVALID_ARGS; } - auto token = new (std::nothrow) ContinueTokenStruct; + auto token = new (std::nothrow) SQLiteSingleVerContinueToken(begin, end); if (token == nullptr) { LOGE("[SQLiteSingleVerNaturalStore][NewToken] Bad alloc."); return -E_OUT_OF_MEMORY; @@ -627,23 +701,105 @@ int SQLiteSingleVerNaturalStore::GetSyncData(TimeStamp begin, TimeStamp end, std } ERROR: - if (errCode != -E_UNFINISHED) { - if (errCode != E_OK) { - dataItems.clear(); - } + if (errCode != -E_UNFINISHED && errCode != E_OK) { + dataItems.clear(); + } + ProcessContinueToken(dataItems, errCode, token); + continueStmtToken = static_cast(token); + + ReleaseHandle(handle); + return errCode; +} + +int SQLiteSingleVerNaturalStore::GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const +{ + if (!timeRange.IsValid()) { + return -E_INVALID_ARGS; + } + int errCode = CheckReadDataControlled(); + if (errCode != E_OK) { + LOGE("[GetEntries] Existed cache prevents the reading from query sync[%d]!", errCode); + return errCode; + } + + query.SetSchema(GetSchemaObject()); + auto token = new (std::nothrow) SQLiteSingleVerContinueToken(timeRange, query); + if (token == nullptr) { + LOGE("[SingleVerNStore] Allocate continue token failed."); + return -E_OUT_OF_MEMORY; + } + + int innerCode; + std::vector dataItems; + errCode = GetSyncDataForQuerySync(dataItems, token, dataSizeInfo); + if (errCode != E_OK && errCode != -E_UNFINISHED) { // The code need be sent to outside except new error happened. + goto ERROR; + } + + innerCode = GetKvEntriesByDataItems(entries, dataItems); + if (innerCode != E_OK) { + errCode = innerCode; delete token; token = nullptr; - } else { - ProcessContinueToken(errCode, end, dataItems, token); } - if (handle != nullptr) { - ReleaseHandle(handle); - } +ERROR: continueStmtToken = static_cast(token); return errCode; } +/** + * Caller must ensure that parameter continueStmtToken is valid. + * If error happened, token will be deleted here. + */ +int SQLiteSingleVerNaturalStore::GetSyncDataForQuerySync(std::vector &dataItems, + SQLiteSingleVerContinueToken *&continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const +{ + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetHandle(false, errCode); + if (handle == nullptr) { + goto ERROR; + } + + // Get query data. + if (!continueStmtToken->IsGetQueryDataFinished()) { + LOGD("[SingleVerNStore] Get query data between %llu and %llu.", continueStmtToken->GetQueryBeginTime(), + continueStmtToken->GetQueryEndTime()); + errCode = handle->GetSyncDataWithQuery(continueStmtToken->GetQuery(), GetAppendedLen(), dataSizeInfo, + std::make_pair(continueStmtToken->GetQueryBeginTime(), continueStmtToken->GetQueryEndTime()), dataItems); + } + + // Get query data finished. + if (errCode == E_OK || errCode == -E_FINISHED) { + // Clear query timeRange. + continueStmtToken->FinishGetQueryData(); + if (!continueStmtToken->IsGetDeletedDataFinished()) { + errCode = -E_UNFINISHED; + // Get delete time next. + if (CanHoldDeletedData(dataItems, dataSizeInfo, GetAppendedLen())) { + LOGD("[SingleVerNStore] Get deleted data between %llu and %llu.", + continueStmtToken->GetDeletedBeginTime(), continueStmtToken->GetDeletedEndTime()); + errCode = handle->GetDeletedSyncDataByTimestamp(dataItems, GetAppendedLen(), + continueStmtToken->GetDeletedBeginTime(), continueStmtToken->GetDeletedEndTime(), dataSizeInfo); + } + } + } + + if (errCode == -E_FINISHED) { + errCode = E_OK; + } + +ERROR: + if (errCode != -E_UNFINISHED && errCode != E_OK) { // Error happened. + dataItems.clear(); + } + ProcessContinueTokenForQuerySync(dataItems, errCode, continueStmtToken); + ReleaseHandle(handle); + return errCode; +} + int SQLiteSingleVerNaturalStore::GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const { @@ -654,30 +810,23 @@ int SQLiteSingleVerNaturalStore::GetSyncDataNext(std::vector } std::vector dataItems; - errCode = GetSyncDataNext(dataItems, continueStmtToken, dataSizeInfo); - if (errCode != E_OK && errCode != -E_UNFINISHED) { - LOGE("GetSyncDataNext errCode:%d", errCode); - goto ERROR; + auto token = static_cast(continueStmtToken); + if (token->IsQuerySync()) { + errCode = GetSyncDataForQuerySync(dataItems, token, dataSizeInfo); + continueStmtToken = static_cast(token); + } else { + errCode = GetSyncDataNext(dataItems, continueStmtToken, dataSizeInfo); } - for (auto &item : dataItems) { - GenericSingleVerKvEntry *entry = new (std::nothrow) GenericSingleVerKvEntry(); - if (entry == nullptr) { - errCode = -E_OUT_OF_MEMORY; - LOGE("GetSyncDataNext errCode:%d", errCode); - goto ERROR; - } - entry->SetEntryData(std::move(item)); - entries.push_back(entry); + if (errCode != E_OK && errCode != -E_UNFINISHED) { + LOGE("GetSyncDataNext errCode:%d", errCode); + return errCode; } -ERROR: - if (errCode != E_OK && errCode != -E_UNFINISHED) { - for (auto &itemEntry : entries) { - delete itemEntry; - itemEntry = nullptr; - } - entries.clear(); + int innerErrCode = GetKvEntriesByDataItems(entries, dataItems); + if (innerErrCode != E_OK) { + errCode = innerErrCode; + ReleaseContinueToken(continueStmtToken); } return errCode; } @@ -685,11 +834,11 @@ ERROR: int SQLiteSingleVerNaturalStore::GetSyncDataNext(std::vector &dataItems, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const { - if (dataSizeInfo.blockSize > MAX_SYNC_BLOCK_SIZE) { + if (dataSizeInfo.blockSize > DBConstant::MAX_SYNC_BLOCK_SIZE) { return -E_INVALID_ARGS; } - auto token = static_cast(continueStmtToken); + auto token = static_cast(continueStmtToken); if (token == nullptr || !(token->CheckValid())) { LOGE("[SingleVerNaturalStore][GetSyncDataNext] invalid continue token."); return -E_INVALID_ARGS; @@ -702,50 +851,46 @@ int SQLiteSingleVerNaturalStore::GetSyncDataNext(std::vector &dataItem return errCode; } - errCode = handle->GetSyncDataByTimestamp(dataItems, GetAppendedLen(), token->GetBeginTimeStamp(), - token->GetEndTimeStamp(), dataSizeInfo); - if (errCode == -E_UNFINISHED) { - // update the begin timestamp of the next fetch. - if (!dataItems.empty()) { - TimeStamp timestamp = dataItems.back().timeStamp; - if (timestamp >= INT64_MAX) { - token->SetBeginTimeStamp(INT64_MAX); - } else { - token->SetBeginTimeStamp(timestamp + 1); // Add 1 for the next fetch. - } - continueStmtToken = static_cast(token); - } else { - ReleaseContinueToken(continueStmtToken); - errCode = -E_INTERNAL_ERROR; - } - } else { - ReleaseContinueToken(continueStmtToken); - if (errCode == -E_FINISHED) { - errCode = E_OK; - } + errCode = handle->GetSyncDataByTimestamp(dataItems, GetAppendedLen(), token->GetQueryBeginTime(), + token->GetQueryEndTime(), dataSizeInfo); + if (errCode == -E_FINISHED) { + errCode = E_OK; } + + ProcessContinueToken(dataItems, errCode, token); + continueStmtToken = static_cast(token); + ReleaseHandle(handle); return errCode; } void SQLiteSingleVerNaturalStore::ReleaseContinueToken(ContinueToken &continueStmtToken) const { - auto token = static_cast(continueStmtToken); + auto token = static_cast(continueStmtToken); if (token == nullptr || !(token->CheckValid())) { LOGE("[SQLiteSingleVerNaturalStore][ReleaseContinueToken] Input is not a continue token."); return; } delete token; continueStmtToken = nullptr; - return; } -int SQLiteSingleVerNaturalStore::PutSyncData(const std::vector &entries, - const std::string &deviceName) +int SQLiteSingleVerNaturalStore::PutSyncDataWithQuery(const QueryObject &query, + const std::vector &entries, const std::string &deviceName) { + if (deviceName.length() > DBConstant::MAX_DEV_LENGTH) { + LOGW("Device length is invalid for sync put"); + return -E_INVALID_ARGS; + } + HeartBeatForLifeCycle(); + DeviceInfo deviceInfo = {false, deviceName}; + if (deviceName.empty()) { + deviceInfo.deviceName = "Unknown"; + } + std::vector dataItems; - for (auto itemEntry : entries) { - GenericSingleVerKvEntry *entry = static_cast(itemEntry); + for (const auto itemEntry : entries) { + auto *entry = static_cast(itemEntry); if (entry != nullptr) { DataItem item; item.origDev = entry->GetOrigDevice(); @@ -757,37 +902,15 @@ int SQLiteSingleVerNaturalStore::PutSyncData(const std::vector &dataItems, const std::string &deviceName) -{ - if (deviceName.length() > DBConstant::MAX_DEV_LENGTH) { - LOGW("Device length is invalid for sync put"); - return -E_INVALID_ARGS; - } - DeviceInfo deviceInfo = {false, deviceName}; - if (deviceName.empty()) { - deviceInfo.deviceName = "Unknown"; - } - int errCode = SaveSyncDataItems(dataItems, deviceInfo, true); // Currently true to check value content + int errCode = SaveSyncDataItems(query, dataItems, deviceInfo, true); // Current is true to check value content if (errCode != E_OK) { - LOGE("PutSyncData errCode:%d", errCode); + LOGE("PutSyncData failed:%d", errCode); } return errCode; } -void SQLiteSingleVerNaturalStore::ReleaseKvEntry(const SingleVerKvEntry *entry) -{ - if (entry != nullptr) { - delete entry; - entry = nullptr; - } -} - void SQLiteSingleVerNaturalStore::GetMaxTimeStamp(TimeStamp &stamp) const { std::lock_guard lock(maxTimeStampMutex_); @@ -820,7 +943,7 @@ int SQLiteSingleVerNaturalStore::RemoveDeviceData(const std::string &deviceName, return -E_NOT_PERMIT; } // Call the syncer module to erase the water mark. - int errCode = EraseDeviceWaterMark(deviceName); + int errCode = EraseDeviceWaterMark(deviceName, true); if (errCode != E_OK) { LOGE("[SingleVerNStore] erase water mark failed:%d", errCode); return errCode; @@ -956,33 +1079,22 @@ void SQLiteSingleVerNaturalStore::ReleaseHandle(SQLiteSingleVerStorageExecutor * int SQLiteSingleVerNaturalStore::RegisterNotification() { - int errCode = RegisterNotificationEventType(SQLITE_GENERAL_NS_LOCAL_PUT_EVENT); - if (errCode != E_OK) { - LOGE("Register single version local event failed:%d!", errCode); - return errCode; - } - - errCode = RegisterNotificationEventType(SQLITE_GENERAL_NS_PUT_EVENT); - if (errCode != E_OK) { - LOGE("Register single version put event failed:%d!", errCode); - UnRegisterNotificationEventType(SQLITE_GENERAL_NS_LOCAL_PUT_EVENT); - return errCode; - } + static const std::vector events { + static_cast(SQLITE_GENERAL_NS_LOCAL_PUT_EVENT), + static_cast(SQLITE_GENERAL_NS_PUT_EVENT), + static_cast(SQLITE_GENERAL_NS_SYNC_EVENT), + static_cast(SQLITE_GENERAL_CONFLICT_EVENT), + }; - errCode = RegisterNotificationEventType(SQLITE_GENERAL_NS_SYNC_EVENT); - if (errCode != E_OK) { - LOGE("Register single version sync event failed:%d!", errCode); - UnRegisterNotificationEventType(SQLITE_GENERAL_NS_PUT_EVENT); - UnRegisterNotificationEventType(SQLITE_GENERAL_NS_LOCAL_PUT_EVENT); - return errCode; - } - - errCode = RegisterNotificationEventType(SQLITE_GENERAL_CONFLICT_EVENT); - if (errCode != E_OK) { - LOGE("Register single version sync event failed:%d!", errCode); - UnRegisterNotificationEventType(SQLITE_GENERAL_NS_SYNC_EVENT); - UnRegisterNotificationEventType(SQLITE_GENERAL_NS_PUT_EVENT); - UnRegisterNotificationEventType(SQLITE_GENERAL_NS_LOCAL_PUT_EVENT); + for (auto event = events.begin(); event != events.end(); ++event) { + int errCode = RegisterNotificationEventType(*event); + if (errCode == E_OK) { + continue; + } + LOGE("Register single version event %d failed:%d!", *event, errCode); + for (auto iter = events.begin(); iter != event; ++iter) { + UnRegisterNotificationEventType(*iter); + } return errCode; } @@ -993,6 +1105,7 @@ int SQLiteSingleVerNaturalStore::RegisterNotification() void SQLiteSingleVerNaturalStore::ReleaseResources() { + SyncAbleKvDB::Close(); if (notificationEventsRegistered_) { UnRegisterNotificationEventType(static_cast(SQLITE_GENERAL_NS_SYNC_EVENT)); UnRegisterNotificationEventType(static_cast(SQLITE_GENERAL_NS_PUT_EVENT)); @@ -1049,8 +1162,8 @@ void SQLiteSingleVerNaturalStore::InitConflictNotifiedFlag(SingleVerNaturalStore // Currently this function only suitable to be call from sync in insert_record_from_sync procedure // Take attention if future coder attempt to call it in other situation procedure -int SQLiteSingleVerNaturalStore::SaveSyncDataItems(std::vector &dataItems, const DeviceInfo &deviceInfo, - bool checkValueContent) +int SQLiteSingleVerNaturalStore::SaveSyncDataItems(const QueryObject &query, std::vector &dataItems, + const DeviceInfo &deviceInfo, bool checkValueContent) { // Sync procedure does not care readOnly Flag CHECK_STORAGE_ENGINE; @@ -1065,11 +1178,12 @@ int SQLiteSingleVerNaturalStore::SaveSyncDataItems(std::vector &dataIt if (checkValueContent) { CheckAmendValueContentForSyncProcedure(dataItems); } - + QueryObject queryInner = query; + queryInner.SetSchema(GetSchemaObjectConstRef()); if (IsExtendedCacheDBMode()) { - errCode = SaveSyncDataToCacheDB(dataItems, deviceInfo); + errCode = SaveSyncDataToCacheDB(queryInner, dataItems, deviceInfo); } else { - errCode = SaveSyncDataToMain(dataItems, deviceInfo); + errCode = SaveSyncDataToMain(queryInner, dataItems, deviceInfo); } if (errCode != E_OK) { LOGE("[SingleVerNStore] SaveSyncDataItems failed:%d", errCode); @@ -1077,37 +1191,80 @@ int SQLiteSingleVerNaturalStore::SaveSyncDataItems(std::vector &dataIt return errCode; } -int SQLiteSingleVerNaturalStore::SaveSyncDataToMain(std::vector &dataItems, const DeviceInfo &deviceInfo) +int SQLiteSingleVerNaturalStore::SaveSyncDataToMain(const QueryObject &query, std::vector &dataItems, + const DeviceInfo &deviceInfo) { - int errCode = E_OK; - LOGD("[SQLiteSingleVerNaturalStore::SaveSyncData] Get write handle."); - SQLiteSingleVerStorageExecutor *handle = GetHandle(true, errCode); - if (handle == nullptr) { - return errCode; - } - auto *committedData = new (std::nothrow) SingleVerNaturalStoreCommitNotifyData; if (committedData == nullptr) { LOGE("[SingleVerNStore] Failed to alloc single version notify data"); - ReleaseHandle(handle); return -E_OUT_OF_MEMORY; } - InitConflictNotifiedFlag(committedData); TimeStamp maxTimestamp = 0; bool isNeedCommit = false; - errCode = SaveSyncItems(handle, dataItems, deviceInfo, maxTimestamp, committedData); + int errCode = SaveSyncItems(query, dataItems, deviceInfo, maxTimestamp, committedData); if (errCode == E_OK) { isNeedCommit = true; (void)SetMaxTimeStamp(maxTimestamp); } CommitAndReleaseNotifyData(committedData, isNeedCommit, SQLITE_GENERAL_NS_SYNC_EVENT); + return errCode; +} + +// Currently, this function only suitable to be call from sync in insert_record_from_sync procedure +// Take attention if future coder attempt to call it in other situation procedure +int SQLiteSingleVerNaturalStore::SaveSyncItems(const QueryObject &query, std::vector &dataItems, + const DeviceInfo &deviceInfo, TimeStamp &maxTimestamp, SingleVerNaturalStoreCommitNotifyData *commitData) const +{ + int errCode = E_OK; + int innerCode = E_OK; + LOGD("[SQLiteSingleVerNaturalStore::SaveSyncData] Get write handle."); + SQLiteSingleVerStorageExecutor *handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + errCode = handle->CheckDataWithQuery(query, dataItems, deviceInfo); + if (errCode != E_OK) { + goto END; + } + errCode = handle->PrepareForSavingData(SingleVerDataType::SYNC_TYPE); + if (errCode != E_OK) { + goto END; + } + for (auto &item: dataItems) { + if (item.neglect) { // Do not save this record if it is neglected + continue; + } + errCode = handle->SaveSyncDataItem(item, deviceInfo, maxTimestamp, commitData); + if (errCode != E_OK && errCode != -E_NOT_FOUND) { + break; + } + } + if (errCode == -E_NOT_FOUND) { + errCode = E_OK; + } + innerCode = handle->ResetForSavingData(SingleVerDataType::SYNC_TYPE); + if (innerCode != E_OK) { + errCode = innerCode; + } +END: + if (errCode == E_OK) { + errCode = handle->Commit(); + } else { + (void)handle->Rollback(); // Keep the error code of the first scene + } ReleaseHandle(handle); return errCode; } -int SQLiteSingleVerNaturalStore::SaveSyncDataToCacheDB(std::vector &dataItems, const DeviceInfo &deviceInfo) +int SQLiteSingleVerNaturalStore::SaveSyncDataToCacheDB(const QueryObject &query, std::vector &dataItems, + const DeviceInfo &deviceInfo) { int errCode = E_OK; SQLiteSingleVerStorageExecutor *handle = GetHandle(true, errCode); @@ -1116,7 +1273,7 @@ int SQLiteSingleVerNaturalStore::SaveSyncDataToCacheDB(std::vector &da } TimeStamp maxTimestamp = 0; - errCode = SaveSyncItemsInCacheMode(handle, dataItems, deviceInfo, maxTimestamp); + errCode = SaveSyncItemsInCacheMode(handle, query, dataItems, deviceInfo, maxTimestamp); if (errCode != E_OK) { LOGE("[SingleVerNStore] Failed to save sync data in cache mode, err : %d", errCode); } else { @@ -1282,6 +1439,14 @@ int SQLiteSingleVerNaturalStore::Import(const std::string &filePath, const Ciphe operation = std::make_unique(this, storageEngine_); operation->SetLocalDevId(localDev); errCode = operation->Import(filePath, passwd); + if (errCode != E_OK) { + goto END; + } + + // Save create db time. + storageEngine_->Enable(OperatePerm::IMPORT_MONOPOLIZE_PERM); + errCode = SaveCreateDBTime(); + END: // restore the storage engine and the syncer. storageEngine_->Enable(OperatePerm::IMPORT_MONOPOLIZE_PERM); @@ -1365,8 +1530,7 @@ void SQLiteSingleVerNaturalStore::InitDataBaseOption(const KvDBProperties &kvDBP securityOpt.securityFlag = kvDBProp.GetSecFlag(); } - option = {uri, isCreateNecessary, isMemoryDb, createTableSqls, cipherType, passwd, schemaStr, - subDir, securityOpt}; + option = {uri, isCreateNecessary, isMemoryDb, createTableSqls, cipherType, passwd, schemaStr, subDir, securityOpt}; option.conflictReslovePolicy = kvDBProp.GetIntProp(KvDBProperties::CONFLICT_RESOLVE_POLICY, DEFAULT_LAST_WIN); option.createDirByStoreIdOnly = kvDBProp.GetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, false); } @@ -1374,40 +1538,34 @@ void SQLiteSingleVerNaturalStore::InitDataBaseOption(const KvDBProperties &kvDBP int SQLiteSingleVerNaturalStore::TransObserverTypeToRegisterFunctionType( int observerType, RegisterFuncType &type) const { - switch (observerType) { - case static_cast(SQLITE_GENERAL_NS_PUT_EVENT): - type = OBSERVER_SINGLE_VERSION_NS_PUT_EVENT; - return E_OK; - case static_cast(SQLITE_GENERAL_NS_SYNC_EVENT): - type = OBSERVER_SINGLE_VERSION_NS_SYNC_EVENT; - return E_OK; - case static_cast(SQLITE_GENERAL_NS_LOCAL_PUT_EVENT): - type = OBSERVER_SINGLE_VERSION_NS_LOCAL_EVENT; - return E_OK; - case static_cast(SQLITE_GENERAL_CONFLICT_EVENT): - type = OBSERVER_SINGLE_VERSION_NS_CONFLICT_EVENT; - return E_OK; - default: - return -E_NOT_SUPPORT; + static const std::map transMap { + { static_cast(SQLITE_GENERAL_NS_PUT_EVENT), OBSERVER_SINGLE_VERSION_NS_PUT_EVENT }, + { static_cast(SQLITE_GENERAL_NS_SYNC_EVENT), OBSERVER_SINGLE_VERSION_NS_SYNC_EVENT }, + { static_cast(SQLITE_GENERAL_NS_LOCAL_PUT_EVENT), OBSERVER_SINGLE_VERSION_NS_LOCAL_EVENT }, + { static_cast(SQLITE_GENERAL_CONFLICT_EVENT), OBSERVER_SINGLE_VERSION_NS_CONFLICT_EVENT }, + }; + auto iter = transMap.find(observerType); + if (iter == transMap.end()) { + return -E_NOT_SUPPORT; } + type = iter->second; + return E_OK; } int SQLiteSingleVerNaturalStore::TransConflictTypeToRegisterFunctionType( int conflictType, RegisterFuncType &type) const { - switch (conflictType) { - case static_cast(SQLITE_GENERAL_NS_FOREIGN_KEY_ONLY): - type = CONFLICT_SINGLE_VERSION_NS_FOREIGN_KEY_ONLY; - return E_OK; - case static_cast(SQLITE_GENERAL_NS_FOREIGN_KEY_ORIG): - type = CONFLICT_SINGLE_VERSION_NS_FOREIGN_KEY_ORIG; - return E_OK; - case static_cast(SQLITE_GENERAL_NS_NATIVE_ALL): - type = CONFLICT_SINGLE_VERSION_NS_NATIVE_ALL; - return E_OK; - default: - return -E_NOT_SUPPORT; + static const std::map transMap { + { static_cast(SQLITE_GENERAL_NS_FOREIGN_KEY_ONLY), CONFLICT_SINGLE_VERSION_NS_FOREIGN_KEY_ONLY }, + { static_cast(SQLITE_GENERAL_NS_FOREIGN_KEY_ORIG), CONFLICT_SINGLE_VERSION_NS_FOREIGN_KEY_ORIG }, + { static_cast(SQLITE_GENERAL_NS_NATIVE_ALL), CONFLICT_SINGLE_VERSION_NS_NATIVE_ALL }, + }; + auto iter = transMap.find(conflictType); + if (iter == transMap.end()) { + return -E_NOT_SUPPORT; } + type = iter->second; + return E_OK; } int SQLiteSingleVerNaturalStore::GetSchema(SchemaObject &schema) const @@ -1491,27 +1649,24 @@ int SQLiteSingleVerNaturalStore::RemoveKvDB(const KvDBProperties &properties) std::string storeDir; GenericKvDB::GetStoreDirectory(properties, KvDBProperties::SINGLE_VER_TYPE, storeDir, storeOnlyDir); - std::string currentDir = storeDir + DBConstant::MAINDB_DIR + "/"; - std::string currentOnlyDir = storeOnlyDir + DBConstant::MAINDB_DIR + "/"; - int removeMainErrCode = KvDBUtils::RemoveKvDB(currentDir, currentOnlyDir, DBConstant::SINGLE_VER_DATA_STORE); - if (removeMainErrCode != -E_NOT_FOUND && removeMainErrCode != E_OK) { - return removeMainErrCode; - } - currentDir = storeDir + DBConstant::METADB_DIR + "/"; - currentOnlyDir = storeOnlyDir + DBConstant::METADB_DIR + "/"; - int removeMetaErrCode = KvDBUtils::RemoveKvDB(currentDir, currentOnlyDir, DBConstant::SINGLE_VER_META_STORE); - if (removeMetaErrCode != -E_NOT_FOUND && removeMetaErrCode != E_OK) { - return removeMetaErrCode; - } - currentDir = storeDir + DBConstant::CACHEDB_DIR + "/"; - currentOnlyDir = storeOnlyDir + DBConstant::CACHEDB_DIR + "/"; - int removeCacheErrCode = KvDBUtils::RemoveKvDB(currentDir, currentOnlyDir, DBConstant::SINGLE_VER_CACHE_STORE); - if (removeCacheErrCode != -E_NOT_FOUND && removeCacheErrCode != E_OK) { - return removeCacheErrCode; - } + const std::vector> dbDir { + {DBConstant::MAINDB_DIR, DBConstant::SINGLE_VER_DATA_STORE}, + {DBConstant::METADB_DIR, DBConstant::SINGLE_VER_META_STORE}, + {DBConstant::CACHEDB_DIR, DBConstant::SINGLE_VER_CACHE_STORE}}; - // Signed numbers can not use bit operations - if (removeMainErrCode == -E_NOT_FOUND && removeMetaErrCode == -E_NOT_FOUND && removeCacheErrCode == -E_NOT_FOUND) { + bool isAllNotFound = true; + for (const auto &item : dbDir) { + std::string currentDir = storeDir + item.first + "/"; + std::string currentOnlyDir = storeOnlyDir + item.first + "/"; + int errCode = KvDBUtils::RemoveKvDB(currentDir, currentOnlyDir, item.second); + if (errCode != -E_NOT_FOUND) { + if (errCode != E_OK) { + return errCode; + } + isAllNotFound = false; + } + }; + if (isAllNotFound) { return -E_NOT_FOUND; } @@ -1531,7 +1686,8 @@ int SQLiteSingleVerNaturalStore::GetKvDBSize(const KvDBProperties &properties, u std::string storeOnlyIdentDir; std::string storeIdentDir; GenericKvDB::GetStoreDirectory(properties, KvDBProperties::SINGLE_VER_TYPE, storeIdentDir, storeOnlyIdentDir); - std::vector> dbDir {{DBConstant::MAINDB_DIR, DBConstant::SINGLE_VER_DATA_STORE}, + const std::vector> dbDir { + {DBConstant::MAINDB_DIR, DBConstant::SINGLE_VER_DATA_STORE}, {DBConstant::METADB_DIR, DBConstant::SINGLE_VER_META_STORE}, {DBConstant::CACHEDB_DIR, DBConstant::SINGLE_VER_CACHE_STORE}}; int errCode = -E_NOT_FOUND; @@ -1581,11 +1737,11 @@ int SQLiteSingleVerNaturalStore::StartLifeCycleTimer(const DatabaseLifeCycleNoti return 0; }, [this]() { - int errCode = RuntimeContext::GetInstance()->ScheduleTask([this]() { + int ret = RuntimeContext::GetInstance()->ScheduleTask([this]() { RefObject::DecObjRef(this); }); - if (errCode != E_OK) { - LOGE("SQLiteSingleVerNaturalStore timer finalizer ScheduleTask, errCode %d", errCode); + if (ret != E_OK) { + LOGE("SQLiteSingleVerNaturalStore timer finalizer ScheduleTask, errCode %d", ret); } }, timerId); @@ -1631,11 +1787,13 @@ int SQLiteSingleVerNaturalStore::StopLifeCycleTimer() const bool SQLiteSingleVerNaturalStore::IsDataMigrating() const { - if (storageEngine_ != nullptr) { - EngineState state = storageEngine_->GetEngineState(); - if (state == EngineState::MIGRATING || state == EngineState::ENGINE_BUSY) { - return true; - } + if (storageEngine_ == nullptr) { + return false; + } + + if (storageEngine_->IsMigrating()) { + LOGD("Migrating now."); + return true; } return false; } @@ -1740,7 +1898,8 @@ void SQLiteSingleVerNaturalStore::CheckAmendValueContentForSyncProcedure(std::ve uint32_t amendCount = 0; uint32_t neglectCount = 0; for (auto &eachItem : dataItems) { - if ((eachItem.flag & DataItem::DELETE_FLAG) != 0) { + if ((eachItem.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG || + (eachItem.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) == DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) { // Delete record not concerned deleteCount++; continue; @@ -1761,7 +1920,8 @@ void SQLiteSingleVerNaturalStore::CheckAmendValueContentForSyncProcedure(std::ve } int SQLiteSingleVerNaturalStore::SaveSyncItemsInCacheMode(SQLiteSingleVerStorageExecutor *handle, - std::vector &dataItems, const DeviceInfo &deviceInfo, TimeStamp &maxTimestamp) const + const QueryObject &query, std::vector &dataItems, const DeviceInfo &deviceInfo, + TimeStamp &maxTimestamp) const { int errCode = handle->StartTransaction(TransactType::IMMEDIATE); if (errCode != E_OK) { @@ -1776,7 +1936,7 @@ int SQLiteSingleVerNaturalStore::SaveSyncItemsInCacheMode(SQLiteSingleVerStorage } for (auto &item : dataItems) { - errCode = handle->SaveSyncDataItemInCacheMode(item, deviceInfo, maxTimestamp, recordVersion); + errCode = handle->SaveSyncDataItemInCacheMode(item, deviceInfo, maxTimestamp, recordVersion, query); if (errCode != E_OK && errCode != -E_NOT_FOUND) { break; } @@ -1807,5 +1967,299 @@ void SQLiteSingleVerNaturalStore::NotifyRemotePushFinished(const std::string &ta NotifyRemotePushFinishedInner(targetId); } +int SQLiteSingleVerNaturalStore::GetDatabaseCreateTimeStamp(TimeStamp &outTime) const +{ + // Found in memory. + { + std::lock_guard autoLock(createDBTimeMutex_); + if (createDBTime_ != 0) { + outTime = createDBTime_; + return E_OK; + } + } + + const Key key(CREATE_DB_TIME.begin(), CREATE_DB_TIME.end()); + Value value; + int errCode = GetMetaData(key, value); + if (errCode != E_OK) { + LOGD("GetDatabaseCreateTimeStamp failed, errCode = %d.", errCode); + return errCode; + } + + TimeStamp createDBTime = 0; + Parcel parcel(value.data(), value.size()); + (void)parcel.ReadUInt64(createDBTime); + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + outTime = createDBTime; + std::lock_guard autoLock(createDBTimeMutex_); + createDBTime_ = createDBTime; + return E_OK; +} + +int SQLiteSingleVerNaturalStore::CheckIntegrity() const +{ + int errCode = E_OK; + auto handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->CheckIntegrity(); + ReleaseHandle(handle); + return errCode; +} + +int SQLiteSingleVerNaturalStore::SaveCreateDBTime() +{ + TimeStamp createDBTime = GetCurrentTimeStamp(); + const Key key(CREATE_DB_TIME.begin(), CREATE_DB_TIME.end()); + Value value(Parcel::GetUInt64Len()); + Parcel parcel(value.data(), Parcel::GetUInt64Len()); + (void)parcel.WriteUInt64(createDBTime); + if (parcel.IsError()) { + LOGE("SaveCreateDBTime failed, something wrong in parcel."); + return -E_PARSE_FAIL; + } + + int errCode = PutMetaData(key, value); + if (errCode != E_OK) { + LOGE("SaveCreateDBTime failed, errCode = %d", errCode); + return errCode; + } + + // save in memory. + std::lock_guard autoLock(createDBTimeMutex_); + createDBTime_ = createDBTime; + return errCode; +} + +int SQLiteSingleVerNaturalStore::SaveCreateDBTimeIfNotExisted() +{ + TimeStamp createDBTime = 0; + int errCode = GetDatabaseCreateTimeStamp(createDBTime); + if (errCode == -E_NOT_FOUND) { + errCode = SaveCreateDBTime(); + } + if (errCode != E_OK) { + LOGE("SaveCreateDBTimeIfNotExisted failed, errCode=%d.", errCode); + } + return errCode; +} + +int SQLiteSingleVerNaturalStore::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + if (keyPrefix.empty() || keyPrefix.size() > DBConstant::MAX_KEY_SIZE) { + return -E_INVALID_ARGS; + } + + int errCode = E_OK; + auto handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->DeleteMetaDataByPrefixKey(keyPrefix); + if (errCode != E_OK) { + LOGE("[SinStore] DeleteMetaData by prefix key failed, errCode = %d", errCode); + } + + ReleaseHandle(handle); + HeartBeatForLifeCycle(); + return errCode; +} + +int SQLiteSingleVerNaturalStore::GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const +{ + needCompressOnSync = GetDbProperties().GetBoolProp(KvDBProperties::COMPRESS_ON_SYNC, false); + compressionRate = GetDbProperties().GetIntProp(KvDBProperties::COMPRESSION_RATE, + DBConstant::DEFAULT_COMPTRESS_RATE); + return E_OK; +} + +int SQLiteSingleVerNaturalStore::GetCompressionAlgo(std::set &algorithmSet) const +{ + algorithmSet.clear(); + DataCompression::GetCompressionAlgo(algorithmSet); + return E_OK; +} + +int SQLiteSingleVerNaturalStore::CheckAndInitQueryCondition(QueryObject &query) const +{ + const SchemaObject &localSchema = MyProp().GetSchemaConstRef(); + if (localSchema.GetSchemaType() != SchemaType::NONE && localSchema.GetSchemaType() != SchemaType::JSON) { + // Flatbuffer schema is not support subscribe + return -E_NOT_SUPPORT; + } + query.SetSchema(localSchema); + + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetHandle(false, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->CheckQueryObjectLegal(query); + if (errCode != E_OK) { + LOGE("Check query condition failed [%d]!", errCode); + } + ReleaseHandle(handle); + return errCode; +} + +void SQLiteSingleVerNaturalStore::SetDataInterceptor(const PushDataInterceptor &interceptor) +{ + std::unique_lock lock(dataInterceptorMutex_); + dataInterceptor_ = interceptor; +} + +int SQLiteSingleVerNaturalStore::InterceptData(std::vector &entries, const std::string &sourceID, + const std::string &targetID) const +{ + PushDataInterceptor interceptor = nullptr; + { + std::shared_lock lock(dataInterceptorMutex_); + if (dataInterceptor_ == nullptr) { + return E_OK; + } + interceptor = dataInterceptor_; + } + + InterceptedDataImpl data(entries, [this](const Value &newValue) -> int { + bool useAmendValue = false; + Value amendValue = newValue; + return this->CheckValueAndAmendIfNeed(ValueSource::FROM_LOCAL, newValue, amendValue, useAmendValue); + } + ); + + int errCode = interceptor(data, sourceID, targetID); + if (data.IsError()) { + SingleVerKvEntry::Release(entries); + LOGE("Intercept data failed:%d.", errCode); + return -E_INTERCEPT_DATA_FAIL; + } + return E_OK; +} + +int SQLiteSingleVerNaturalStore::AddSubscribe(const std::string &subscribeId, const QueryObject &query, + bool needCacheSubscribe) +{ + const SchemaObject &localSchema = MyProp().GetSchemaConstRef(); + if (localSchema.GetSchemaType() != SchemaType::NONE && localSchema.GetSchemaType() != SchemaType::JSON) { + // Flatbuffer schema is not support subscribe + return -E_NOT_SUPPORT; + } + QueryObject queryInner = query; + queryInner.SetSchema(localSchema); + if (IsExtendedCacheDBMode() && needCacheSubscribe) { // cache auto subscribe when engine state is in CACHEDB mode + LOGI("Cache subscribe query and return ok when in cacheDB."); + storageEngine_->CacheSubscribe(subscribeId, queryInner); + return E_OK; + } + + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + + errCode = handle->AddSubscribeTrigger(queryInner, subscribeId); + if (errCode != E_OK) { + LOGE("Add subscribe trigger failed: %d", errCode); + (void)handle->Rollback(); + } else { + errCode = handle->Commit(); + } + ReleaseHandle(handle); + return errCode; +} + +int SQLiteSingleVerNaturalStore::RemoveSubscribe(const std::vector &subscribeIds) +{ + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + errCode = handle->RemoveSubscribeTrigger(subscribeIds); + if (errCode != E_OK) { + LOGE("Remove subscribe trigger failed: %d", errCode); + goto ERR; + } + errCode = handle->RemoveSubscribeTriggerWaterMark(subscribeIds); + if (errCode != E_OK) { + LOGE("Remove subscribe data water mark failed: %d", errCode); + } +ERR: + if (errCode == E_OK) { + errCode = handle->Commit(); + } else { + (void)handle->Rollback(); + } + ReleaseHandle(handle); + return errCode; +} + +int SQLiteSingleVerNaturalStore::RemoveSubscribe(const std::string &subscribeId) +{ + return RemoveSubscribe(std::vector {subscribeId}); +} + +int SQLiteSingleVerNaturalStore::RemoveAllSubscribe() +{ + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetHandle(true, errCode); + if (handle == nullptr) { + return errCode; + } + std::vector triggers; + errCode = handle->GetTriggers(DBConstant::SUBSCRIBE_QUERY_PREFIX, triggers); + if (errCode != E_OK) { + LOGE("Get all subscribe triggers failed. %d", errCode); + ReleaseHandle(handle); + return errCode; + } + + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + ReleaseHandle(handle); + return errCode; + } + + Key prefixKey; + errCode = handle->RemoveTrigger(triggers); + if (errCode != E_OK) { + LOGE("remove all subscribe triggers failed. %d", errCode); + goto END; + } + + DBCommon::StringToVector(DBConstant::SUBSCRIBE_QUERY_PREFIX, prefixKey); + errCode = handle->DeleteMetaDataByPrefixKey(prefixKey); + if (errCode != E_OK) { + LOGE("remove all subscribe water mark failed. %d", errCode); + } +END: + if (errCode == E_OK) { + errCode = handle->Commit(); + } else { + (void)handle->Rollback(); + } + ReleaseHandle(handle); + return errCode; +} + DEFINE_OBJECT_TAG_FACILITIES(SQLiteSingleVerNaturalStore) } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h index 71b488d5b27a3d6e3d99004cfafa525a023b8b2e..bc4b5795f0af2173f905232d40c67aa9458a1b82 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store.h @@ -24,44 +24,9 @@ #include "single_ver_kvdb_sync_interface.h" #include "kv_store_nb_conflict_data_impl.h" #include "runtime_context.h" +#include "sqlite_single_ver_continue_token.h" namespace DistributedDB { -struct ContinueTokenStruct { - /* - * function: Check the magic number at the beginning and end of the ContinueTokenStruct. - * returnValue: Return true if the begin and end magic number is OK. - * Return false if the begin or end magic number is error. - */ - bool CheckValid() const - { - return ((magicBegin_ == MAGIC_BEGIN) && (magicEnd_ == MAGIC_END)); - } - TimeStamp GetBeginTimeStamp() const - { - return begin_; - } - void SetBeginTimeStamp(TimeStamp begin) - { - begin_ = begin; - } - TimeStamp GetEndTimeStamp() const - { - return end_; - } - void SetEndTimeStamp(TimeStamp end) - { - end_ = end; - } - -private: - static const unsigned int MAGIC_BEGIN = 0x600D0AC7; // for token guard - static const unsigned int MAGIC_END = 0x0AC7600D; // for token guard - unsigned int magicBegin_ = MAGIC_BEGIN; - TimeStamp begin_ = 0; - TimeStamp end_ = 0; - unsigned int magicEnd_ = MAGIC_END; -}; - class SQLiteSingleVerNaturalStore : public SyncAbleKvDB, public SingleVerKvDBSyncInterface { public: SQLiteSingleVerNaturalStore(); @@ -98,6 +63,11 @@ public: int PutMetaData(const Key &key, const Value &value) override; + // Delete multiple meta data records in a transaction. + int DeleteMetaData(const std::vector &keys) override; + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + int GetAllMetaKeys(std::vector &keys) const override; int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &dataItems, ContinueToken &continueStmtToken, @@ -106,6 +76,9 @@ public: int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; + int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, const DataSizeSpecInfo &dataSizeInfo, + ContinueToken &continueStmtToken, std::vector &entries) const override; + int GetSyncDataNext(std::vector &dataItems, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; @@ -114,11 +87,8 @@ public: void ReleaseContinueToken(ContinueToken &continueStmtToken) const override; - int PutSyncData(std::vector &dataItems, const std::string &deviceName) override; - - int PutSyncData(const std::vector &entries, const std::string &deviceName) override; - - void ReleaseKvEntry(const SingleVerKvEntry *entry) override; + int PutSyncDataWithQuery(const QueryObject &query, const std::vector &entries, + const std::string &deviceName) override; void GetMaxTimeStamp(TimeStamp &stamp) const override; @@ -172,8 +142,6 @@ public: int GetSecurityOption(SecurityOption &option) const override; - bool IsReadable() const override; - bool IsDataMigrating() const override; void SetConnectionFlag(bool isExisted) const override; @@ -193,6 +161,28 @@ public: void NotifyRemotePushFinished(const std::string &targetId) const override; + int GetDatabaseCreateTimeStamp(TimeStamp &outTime) const override; + + int CheckIntegrity() const override; + + int GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const override; + int GetCompressionAlgo(std::set &algorithmSet) const override; + + // Check and init query object for query sync and subscribe, flatbuffer schema will always return E_NOT_SUPPORT. + // return E_OK if subscribe is legal, ERROR on exception. + int CheckAndInitQueryCondition(QueryObject &query) const override; + + int InterceptData(std::vector &entries, const std::string &sourceID, + const std::string &targetID) const override; + + void SetDataInterceptor(const PushDataInterceptor &interceptor) override; + + int AddSubscribe(const std::string &subscribeId, const QueryObject &query, bool needCacheSubscribe) override; + + int RemoveSubscribe(const std::string &subscribeId) override; + + int RemoveSubscribe(const std::vector &subscribeIds) override; + private: int CheckDatabaseRecovery(const KvDBProperties &kvDBProp); @@ -205,9 +195,8 @@ private: void InitCurrentMaxStamp(); - int SaveSyncDataItems(std::vector &dataItems, const DeviceInfo &deviceInfo, bool checkValueContent); - - int GetData(const SQLiteStorageExecutor* handle, const std::string &sql, const Key &key, Value &value) const; + int SaveSyncDataItems(const QueryObject &query, std::vector &dataItems, const DeviceInfo &deviceInfo, + bool checkValueContent); int InitStorageEngine(const KvDBProperties &kvDBProp, bool isNeedUpdateSecOpt); @@ -237,6 +226,7 @@ private: void InitConflictNotifiedFlag(SingleVerNaturalStoreCommitNotifyData *committedData); void AsyncDataMigration() const; + // Change value that should be amended, and neglect value that is incompatible void CheckAmendValueContentForSyncProcedure(std::vector &dataItems) const; @@ -244,15 +234,31 @@ private: int RemoveDeviceDataNormally(const std::string &deviceName, bool isNeedNotify); - int SaveSyncDataToMain(std::vector &dataItems, const DeviceInfo &deviceInfo); + int SaveSyncDataToMain(const QueryObject &query, std::vector &dataItems, const DeviceInfo &deviceInfo); - int SaveSyncDataToCacheDB(std::vector &dataItems, const DeviceInfo &deviceInfo); + // Currently, this function only suitable to be call from sync in insert_record_from_sync procedure + // Take attention if future coder attempt to call it in other situation procedure + int SaveSyncItems(const QueryObject& query, std::vector &dataItems, const DeviceInfo &deviceInfo, + TimeStamp &maxTimestamp, SingleVerNaturalStoreCommitNotifyData *commitData) const; - int SaveSyncItemsInCacheMode(SQLiteSingleVerStorageExecutor *handle, + int SaveSyncDataToCacheDB(const QueryObject &query, std::vector &dataItems, + const DeviceInfo &deviceInfo); + + int SaveSyncItemsInCacheMode(SQLiteSingleVerStorageExecutor *handle, const QueryObject &query, std::vector &dataItems, const DeviceInfo &deviceInfo, TimeStamp &maxTimestamp) const; int ClearIncompleteDatabase(const KvDBProperties &kvDBPro) const; + int GetSyncDataForQuerySync(std::vector &dataItems, SQLiteSingleVerContinueToken *&continueStmtToken, + const DataSizeSpecInfo &dataSizeInfo) const; + + int SaveCreateDBTime(); + int SaveCreateDBTimeIfNotExisted(); + + int GetAndInitStorageEngine(const KvDBProperties &kvDBProp); + + int RemoveAllSubscribe(); + DECLARE_OBJECT_TAG(SQLiteSingleVerNaturalStore); TimeStamp currentMaxTimeStamp_ = 0; @@ -268,6 +274,11 @@ private: mutable DatabaseLifeCycleNotifier lifeCycleNotifier_; mutable TimerId lifeTimerId_; uint32_t autoLifeTime_; + mutable TimeStamp createDBTime_; + mutable std::mutex createDBTimeMutex_; + + mutable std::shared_mutex dataInterceptorMutex_; + PushDataInterceptor dataInterceptor_; }; } #endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp index cf37553e44f574bfdb246d89b99682e9dfac9301..fc54546f8160acbf6b8724f19370f345557465a4 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.cpp @@ -689,14 +689,16 @@ int SQLiteSingleVerNaturalStoreConnection::PreClose() } // check if transaction closed - std::lock_guard transactionLock(transactionMutex_); - if (writeHandle_ != nullptr) { - LOGW("Transaction started, need to rollback before close."); - int errCode = RollbackInner(); - if (errCode != E_OK) { - LOGE("Rollback transaction failed, %d.", errCode); + { + std::lock_guard transactionLock(transactionMutex_); + if (writeHandle_ != nullptr) { + LOGW("Transaction started, need to rollback before close."); + int errCode = RollbackInner(); + if (errCode != E_OK) { + LOGE("Rollback transaction failed, %d.", errCode); + } + ReleaseExecutor(writeHandle_); } - ReleaseExecutor(writeHandle_); } // Clear the conflict type function. @@ -708,6 +710,20 @@ int SQLiteSingleVerNaturalStoreConnection::PreClose() return E_OK; } +int SQLiteSingleVerNaturalStoreConnection::CheckIntegrity() const +{ + int errCode = E_OK; + SQLiteSingleVerStorageExecutor *handle = GetExecutor(true, errCode); + if (handle == nullptr) { + LOGW("Failed to get the executor for the integrity check."); + return errCode; + } + + errCode = handle->CheckIntegrity(); + ReleaseExecutor(handle); + return errCode; +} + int SQLiteSingleVerNaturalStoreConnection::CheckMonoStatus(OperatePerm perm) { // 1. Get the connection number @@ -1038,7 +1054,8 @@ int SQLiteSingleVerNaturalStoreConnection::SaveEntryInCacheMode(DataItem &dataIt TimeStamp maxTimestamp = 0; DeviceInfo deviceInfo = {true, ""}; - errCode = writeHandle_->SaveSyncDataItemInCacheMode(dataItem, deviceInfo, maxTimestamp, recordVersion); + QueryObject query(Query::Select()); + errCode = writeHandle_->SaveSyncDataItemInCacheMode(dataItem, deviceInfo, maxTimestamp, recordVersion, query); if (errCode == E_OK) { if (maxTimestamp > currentMaxTimeStamp_) { currentMaxTimeStamp_ = maxTimestamp; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h index 8a619ff698c480af34299b5be8d0ab1cc603ca8d..1bd8f40081d790006a7314f9005e7be6a10eaa7c 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_natural_store_connection.h @@ -50,6 +50,7 @@ public: int GetEntries(const IOption &option, const Query &query, std::vector &entries) const override; int GetCount(const IOption &option, const Query &query, int &count) const override; + // Put the batch values to the database. int PutBatch(const IOption &option, const std::vector &entries) override; @@ -102,6 +103,8 @@ public: // Called when Close and delete the connection. int PreClose() override; + int CheckIntegrity() const override; + private: int CheckMonoStatus(OperatePerm perm); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..167adfdb290ac6848db05b7e2b227d4a85c24812 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relation_storage_executor.cpp @@ -0,0 +1,811 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "sqlite_single_ver_relational_storage_executor.h" +#include "data_transformer.h" +#include "db_common.h" + +namespace DistributedDB { +SQLiteSingleVerRelationalStorageExecutor::SQLiteSingleVerRelationalStorageExecutor(sqlite3 *dbHandle, bool writable) + : SQLiteStorageExecutor(dbHandle, writable, false) {}; + +int SQLiteSingleVerRelationalStorageExecutor::CreateDistributedTable(const std::string &tableName, + const RelationalStoreDelegate::TableOption &option) +{ + if (dbHandle_ == nullptr) { + LOGE("Begin transaction failed, dbHandle is null."); + return -E_INVALID_DB; + } + + TableInfo table; + int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); + if (errCode != E_OK) { + LOGE("[CreateDistributedTable] analysis table schema failed"); + return errCode; + } + + // create log table + errCode = SQLiteUtils::CreateRelationalLogTable(dbHandle_, tableName); + if (errCode != E_OK) { + LOGE("[CreateDistributedTable] create log table failed"); + return errCode; + } + + // add trigger + errCode = SQLiteUtils::AddRelationalLogTableTrigger(dbHandle_, table); + if (errCode != E_OK) { + LOGE("[CreateDistributedTable] create log table failed"); + return errCode; + } + return E_OK; +} + +int SQLiteSingleVerRelationalStorageExecutor::StartTransaction(TransactType type) +{ + if (dbHandle_ == nullptr) { + LOGE("Begin transaction failed, dbHandle is null."); + return -E_INVALID_DB; + } + int errCode = SQLiteUtils::BeginTransaction(dbHandle_, type); + if (errCode != E_OK) { + LOGE("Begin transaction failed, errCode = %d", errCode); + } + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::Commit() +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + + return SQLiteUtils::CommitTransaction(dbHandle_); +} + +int SQLiteSingleVerRelationalStorageExecutor::Rollback() +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + int errCode = SQLiteUtils::RollbackTransaction(dbHandle_); + if (errCode != E_OK) { + LOGE("sqlite single ver storage executor rollback fail! errCode = [%d]", errCode); + } + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::SetTableInfo(QueryObject query) +{ + int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, query.GetTableName(), table_); + if (errCode != E_OK) { + LOGE("[CreateDistributedTable] analysis table schema failed"); + } + return errCode; +} + +// binding index just for the get sync data sql +static const int BIND_BEGIN_STAMP_INDEX = 1; +static const int BIND_END_STAMP_INDEX = 2; + +int SQLiteSingleVerRelationalStorageExecutor::PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, + sqlite3_stmt *&statement, bool getDeletedData) const +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + + const std::string SELECT_SYNC_DELETED_ENTRIES_SQL = + "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + + "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x03) ORDER BY timestamp ASC;"; + const std::string SELECT_SYNC_ENTRIES_SQL = + "SELECT * FROM distributeddatamgr_aux_" + table_.GetTableName() + + "_log WHERE timestamp >= ? AND timestamp < ? AND (flag&0x02=0x02) ORDER BY timestamp ASC;"; + + const std::string sql = (getDeletedData ? SELECT_SYNC_DELETED_ENTRIES_SQL : SELECT_SYNC_ENTRIES_SQL); + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); + if (errCode != E_OK) { + LOGE("Prepare the sync entries statement error:%d", errCode); + return errCode; + } + + errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_BEGIN_STAMP_INDEX, begin); + if (errCode != E_OK) { + goto ERROR; + } + + errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_END_STAMP_INDEX, end); + if (errCode != E_OK) { + goto ERROR; + } + +ERROR: + if (errCode != E_OK) { + LOGE("Bind the timestamp for getting sync data error:%d", errCode); + SQLiteUtils::ResetStatement(statement, true, errCode); + } + + return CheckCorruptedStatus(errCode); +} + +static void GetDataValueByType(sqlite3_stmt *statement, DataValue &value, StorageType type, int cid) +{ + switch (type) { + case StorageType::STORAGE_TYPE_BOOL: { + value = static_cast(sqlite3_column_int64(statement, cid)); + break; + } + + case StorageType::STORAGE_TYPE_INTEGER: { + int64_t v = static_cast(sqlite3_column_int64(statement, cid)); + value = v; + break; + } + + case StorageType::STORAGE_TYPE_REAL: { + value = sqlite3_column_double(statement, cid); + break; + } + + case StorageType::STORAGE_TYPE_TEXT: { + const char *colValue = reinterpret_cast(sqlite3_column_text(statement, cid)); + if (colValue == nullptr) { + value = std::string(); + } else { + value = std::string(colValue); + } + break; + } + + case StorageType::STORAGE_TYPE_BLOB: { + std::vector blobValue; + (void)SQLiteUtils::GetColumnBlobValue(statement, cid, blobValue); + Blob blob; + blob.WriteBlob(blobValue.data(), static_cast(blobValue.size())); + value = blob; + break; + } + + default: + return; + } +} + +static void BindDataValueByType(sqlite3_stmt *statement, + const std::optional &value, StorageType type, int cid) +{ + if (value == std::nullopt) { + sqlite3_bind_null(statement, cid); + return; + } + + switch (type) { + case StorageType::STORAGE_TYPE_BOOL: { + bool boolData = false; + value.value().GetBool(boolData); + sqlite3_bind_int(statement, cid, boolData); + break; + } + + case StorageType::STORAGE_TYPE_INTEGER: { + int64_t intData = 0; + value.value().GetInt64(intData); + sqlite3_bind_int64(statement, cid, intData); + break; + } + + case StorageType::STORAGE_TYPE_REAL: { + double doubleData = 0; + value.value().GetDouble(doubleData); + sqlite3_bind_double(statement, cid, doubleData); + break; + } + + case StorageType::STORAGE_TYPE_TEXT: { + std::string strData; + value.value().GetText(strData); + SQLiteUtils::BindTextToStatement(statement, cid, strData); + break; + } + + case StorageType::STORAGE_TYPE_BLOB: { + Blob blob; + value.value().GetBlob(blob); + std::vector blobData(blob.GetData(), blob.GetData() + blob.GetSize()); + SQLiteUtils::BindBlobToStatement(statement, cid, blobData, true); + break; + } + + default: + return; + } +} + +static int GetLogData(sqlite3_stmt *logStatement, LogInfo &logInfo) +{ + logInfo.dataKey = sqlite3_column_int(logStatement, 0); + + std::vector dev; + int errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 1, dev); + if (errCode != E_OK) { + return errCode; + } + logInfo.device = std::string(dev.begin(), dev.end()); + + std::vector oriDev; + errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 2, oriDev); + if (errCode != E_OK) { + return errCode; + } + logInfo.originDev = std::string(oriDev.begin(), oriDev.end()); + + logInfo.timestamp = static_cast(sqlite3_column_int64(logStatement, 3)); + logInfo.wTimeStamp = static_cast(sqlite3_column_int64(logStatement, 4)); + logInfo.flag = static_cast(sqlite3_column_int64(logStatement, 5)); + logInfo.flag &= (~DataItem::LOCAL_FLAG); + + std::vector hashKey; + errCode = SQLiteUtils::GetColumnBlobValue(logStatement, 6, hashKey); + if (errCode != E_OK) { + return errCode; + } + + logInfo.hashKey = std::string(hashKey.begin(), hashKey.end()); + return errCode; +} + +static sqlite3_stmt *GetDataStmtByPK(sqlite3 *db, const std::string &tableName, int rowId) +{ + std::string sql = "select * from " + tableName + " where rowId=?;"; + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + LOGE("[data by rowid statement] Get statement fail!"); + return nullptr; + } + + errCode = SQLiteUtils::BindInt64ToStatement(statement, 1, rowId); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(statement, true, errCode); + return nullptr; + } + + return statement; +} + + +int SQLiteSingleVerRelationalStorageExecutor::GetDataItemForSync(sqlite3_stmt *logStatement, DataItem &dataItem) const +{ + std::map colInfos = table_.GetFields(); + RowDataWithLog data; + + int errCode = GetLogData(logStatement, data.logInfo); + if (errCode != E_OK) { + LOGE("relational data value transfer to kv fail"); + return errCode; + } + + // dataKey is rowid by pk + // only get local device data + sqlite3_stmt *stmt = GetDataStmtByPK(dbHandle_, table_.GetTableName(), data.logInfo.dataKey); + + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW) && errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + LOGE("[SQLiteSingleVerRelationalStorageExecutor] Step fail errCode:%d", errCode); + return errCode; + } + + std::vector fieldInfos; + for (const auto &col: colInfos) { + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + break; + } + DataValue value; + GetDataValueByType(stmt, value, col.second.GetStorageType(), col.second.GetColumnId()); + + fieldInfos.push_back(col.second); + data.rowData.push_back(value); + } + + SQLiteUtils::ResetStatement(stmt, true, errCode); + + errCode = DataTransformer::SerializeDataItem(data, fieldInfos, dataItem); + if (errCode != E_OK) { + LOGE("relational data value transfer to kv fail"); + } + return errCode; +} + +static size_t GetDataItemSerialSize(DataItem &item, size_t appendLen) +{ + // timestamp and local flag: 3 * uint64_t, version(uint32_t), key, value, origin dev and the padding size. + // the size would not be very large. + static const size_t maxOrigDevLength = 40; + size_t devLength = std::max(maxOrigDevLength, item.origDev.size()); + size_t dataSize = (Parcel::GetUInt64Len() * 3 + Parcel::GetUInt32Len() + Parcel::GetVectorCharLen(item.key) + + Parcel::GetVectorCharLen(item.value) + devLength + appendLen); + return dataSize; +} + +int SQLiteSingleVerRelationalStorageExecutor::GetSyncDataItems(std::vector &dataItems, + sqlite3_stmt *logStatement, size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const +{ + int errCode; + size_t dataTotalSize = 0; + do { + DataItem item; + errCode = SQLiteUtils::StepWithRetry(logStatement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + errCode = GetDataItemForSync(logStatement, item); + } else { + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); + errCode = -E_FINISHED; + } else { + LOGE("Get sync data error:%d", errCode); + } + break; + } + + // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. + dataTotalSize += GetDataItemSerialSize(item, appendLength); + if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || + dataItems.size() >= dataSizeInfo.packetSize) { + errCode = -E_UNFINISHED; + break; + } else { + dataItems.push_back(std::move(item)); + } + } while (true); + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::GetDeletedSyncDataByTimestamp(std::vector &dataItems, + size_t appendLength, TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const +{ + sqlite3_stmt *logStatement = nullptr; + int errCode = PrepareForSyncDataByTime(begin, end, logStatement, true); + if (errCode != E_OK) { + return errCode; + } + + errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); + SQLiteUtils::ResetStatement(logStatement, true, errCode); + return CheckCorruptedStatus(errCode); +} + +static const std::string SELECT_META_VALUE_SQL = + "SELECT value FROM distributeddatamgr_aux_metadata WHERE key=?;"; +int SQLiteSingleVerRelationalStorageExecutor::GetKvData(const Key &key, Value &value) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_META_VALUE_SQL, statement); + if (errCode != E_OK) { + goto END; + } + + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // first arg. + if (errCode != E_OK) { + goto END; + } + + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = -E_NOT_FOUND; + goto END; + } else if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + goto END; + } + + errCode = SQLiteUtils::GetColumnBlobValue(statement, 0, value); // only one result. + END: + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +static const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO distributeddatamgr_aux_metadata VALUES(?,?);"; +int SQLiteSingleVerRelationalStorageExecutor::PutKvData(const Key &key, const Value &value) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, INSERT_META_SQL, statement); + if (errCode != E_OK) { + goto ERROR; + } + + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // BIND_KV_KEY_INDEX + if (errCode != E_OK) { + LOGE("[SingleVerExe][BindPutKv]Bind key error:%d", errCode); + goto ERROR; + } + + errCode = SQLiteUtils::BindBlobToStatement(statement, 2, value, true); // BIND_KV_VAL_INDEX + if (errCode != E_OK) { + LOGE("[SingleVerExe][BindPutKv]Bind value error:%d", errCode); + goto ERROR; + } + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } +ERROR: + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +static const std::string REMOVE_META_VALUE_SQL = "DELETE FROM distributeddatamgr_aux_metadata WHERE key=?;"; +int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaData(const std::vector &keys) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_SQL, statement); + if (errCode != E_OK) { + return errCode; + } + + for (const auto &key : keys) { + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // first arg. + if (errCode != E_OK) { + break; + } + + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + break; + } + errCode = E_OK; + SQLiteUtils::ResetStatement(statement, false, errCode); + } + SQLiteUtils::ResetStatement(statement, true, errCode); + return CheckCorruptedStatus(errCode); +} + +static const std::string REMOVE_META_VALUE_BY_KEY_PREFIX_SQL = + "DELETE FROM distributeddatamgr_aux_metadata WHERE key>=? AND key<=?;"; +int SQLiteSingleVerRelationalStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, REMOVE_META_VALUE_BY_KEY_PREFIX_SQL, statement); + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::BindPrefixKey(statement, 1, keyPrefix); // 1 is first arg. + if (errCode == E_OK) { + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + } + SQLiteUtils::ResetStatement(statement, true, errCode); + return CheckCorruptedStatus(errCode); +} + +static int GetAllKeys(sqlite3_stmt *statement, std::vector &keys) +{ + if (statement == nullptr) { + return -E_INVALID_DB; + } + int errCode; + do { + errCode = SQLiteUtils::StepWithRetry(statement, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + Key key; + errCode = SQLiteUtils::GetColumnBlobValue(statement, 0, key); + if (errCode != E_OK) { + break; + } + + keys.push_back(std::move(key)); + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + break; + } else { + LOGE("SQLite step for getting all keys failed:%d", errCode); + break; + } + } while (true); + return errCode; +} + + +static const std::string SELECT_ALL_META_KEYS = "SELECT key FROM distributeddatamgr_aux_metadata;"; +int SQLiteSingleVerRelationalStorageExecutor::GetAllMetaKeys(std::vector &keys) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_ALL_META_KEYS, statement); + if (errCode != E_OK) { + LOGE("[Relational][GetAllKey] Get statement failed:%d", errCode); + return errCode; + } + errCode = GetAllKeys(statement, keys); + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingLog(const QueryObject &object, + const std::string &deviceName, sqlite3_stmt *&logStmt) const +{ + std::string devName = DBCommon::TransferHashString(deviceName); + const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_log"; + std::string dataFormat = "?, '" + deviceName + "', ?, ?, ?, ?, ?"; + + std::string sql = "INSERT OR REPLACE INTO " + tableName + + " (data_key, device, ori_device, timestamp, wtimestamp, flag, hash_key) VALUES (" + dataFormat + ");"; + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, logStmt); + if (errCode != E_OK) { + LOGE("[info statement] Get statement fail!"); + return -E_INVALID_QUERY_FORMAT; + } + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::PrepareForSavingData(const QueryObject &object, + const std::string &deviceName, sqlite3_stmt *&statement) const +{ + // distributeddatamgr_aux_userTableName_deviceHash + // tableName + std::string devName = DBCommon::TransferHashString(deviceName); + const std::string tableName = "distributeddatamgr_aux_" + object.GetTableName() + "_" + + DBCommon::TransferStringToHex(devName); + TableInfo table; + int errCode = SQLiteUtils::AnalysisSchema(dbHandle_, tableName, table); + if (errCode == -E_NOT_FOUND) { + errCode = SQLiteUtils::CreateSameStuTable(dbHandle_, object.GetTableName(), tableName, false); + } + + if (errCode != E_OK) { + LOGE("[PrepareForSavingData] analysis table schema failed"); + return errCode; + } + + std::string colName; + std::string dataFormat; + const std::map fields = table_.GetFields(); + for (const auto &field : fields) { + colName += field.first + ","; + dataFormat += "?,"; + } + colName.pop_back(); + dataFormat.pop_back(); + + std::string sql = "INSERT OR REPLACE INTO " + tableName + " (" + colName + ") VALUES (" + dataFormat + ");"; + errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); + if (errCode != E_OK) { + LOGE("[info statement] Get statement fail!"); + errCode = -E_INVALID_QUERY_FORMAT; + } + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::SaveSyncLog(sqlite3_stmt *statement, + const DataItem &dataItem, TimeStamp &maxTimestamp) +{ + Key hashKey; + (void)DBCommon::CalcValueHash(dataItem.key, hashKey); + std::string hash = std::string(hashKey.begin(), hashKey.end()); + std::string sql = "select * from distributeddatamgr_aux_" + table_.GetTableName() + "_log where hash_key = ?;"; + sqlite3_stmt *queryStmt = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, queryStmt); + if (errCode != E_OK) { + LOGE("[info statement] Get statement fail!"); + return -E_INVALID_QUERY_FORMAT; + } + SQLiteUtils::BindTextToStatement(queryStmt, 1, hash); + + LogInfo logInfoGet; + errCode = SQLiteUtils::StepWithRetry(queryStmt, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + errCode = -E_NOT_FOUND; + } else { + errCode = GetLogData(queryStmt, logInfoGet); + } + SQLiteUtils::ResetStatement(queryStmt, true, errCode); + + LogInfo logInfoBind; + std::string key = std::string(dataItem.key.begin(), dataItem.key.end()); + logInfoBind.hashKey = hash; + logInfoBind.device = dataItem.dev; + logInfoBind.timestamp = dataItem.timeStamp; + int dataKeyBind = -1; + logInfoBind.flag = dataItem.flag; + logInfoBind.wTimeStamp = maxTimestamp; + + if (errCode == -E_NOT_FOUND) { // insert + logInfoBind.originDev = dataItem.dev; + } else if (errCode == E_OK) { // update + logInfoBind.wTimeStamp = logInfoGet.wTimeStamp; + logInfoBind.originDev = logInfoGet.originDev; + } else { + return errCode; + } + + // bind + SQLiteUtils::BindInt64ToStatement(statement, 1, dataKeyBind); + + std::vector originDev(logInfoBind.originDev.begin(), logInfoBind.originDev.end()); + SQLiteUtils::BindBlobToStatement(statement, 2, originDev); + + SQLiteUtils::BindInt64ToStatement(statement, 3, logInfoBind.timestamp); + SQLiteUtils::BindInt64ToStatement(statement, 4, logInfoBind.wTimeStamp); + SQLiteUtils::BindInt64ToStatement(statement, 5, logInfoBind.flag); + + SQLiteUtils::BindTextToStatement(statement, 6, logInfoBind.hashKey); + + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + return E_OK; + } + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::DeleteSyncDataItem(const DataItem &dataItem) +{ + std::string devName = DBCommon::TransferHashString(dataItem.dev); + const std::string tableName = "distributeddatamgr_aux_" + table_.GetTableName() + "_" + + DBCommon::TransferStringToHex(devName); + std::string hashKey = std::string(dataItem.hashKey.begin(), dataItem.hashKey.end()); + std::string sql = "DELETE FROM " + tableName + " WHERE calc_hash(" + table_.GetPrimaryKey() + ")=" + hashKey + ";"; + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, stmt); + if (errCode != E_OK) { + LOGE("[info statement] Get statement fail!"); + return -E_INVALID_QUERY_FORMAT; + } + + errCode = SQLiteUtils::StepWithRetry(stmt, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::SaveSyncDataItem(sqlite3_stmt *statement, const DataItem &dataItem) +{ + if ((dataItem.flag & DataItem::DELETE_FLAG) != 0) { + return DeleteSyncDataItem(dataItem); + } + + std::map colInfos = table_.GetFields(); + std::vector fieldInfos; + for (const auto &col: colInfos) { + fieldInfos.push_back(col.second); + } + + std::vector indexMapping; + DataTransformer::ReduceMapping(fieldInfos, fieldInfos, indexMapping); + + OptRowDataWithLog data; + int errCode = DataTransformer::DeSerializeDataItem(dataItem, data, fieldInfos, indexMapping); + if (errCode != E_OK) { + LOGE("[RelationalStorageExecutor] DeSerialize dataItem failed! errCode = [%d]", errCode); + return errCode; + } + + for (size_t index = 0; index < data.optionalData.size(); index++) { + const auto &filedData = data.optionalData[index]; + if (filedData.has_value()) { + (void)BindDataValueByType(statement, filedData, + filedData.value().GetType(), fieldInfos[index].GetColumnId() + 1); + } else { + (void)BindDataValueByType(statement, filedData, + StorageType::STORAGE_TYPE_NULL, fieldInfos[index].GetColumnId() + 1); + } + } + + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::SaveSyncDataItems(const QueryObject &object, + std::vector &dataItems, const std::string &deviceName, TimeStamp &maxTimestamp) +{ + sqlite3_stmt *statement = nullptr; + int errCode = PrepareForSavingData(object, deviceName, statement); + if (errCode != E_OK) { + LOGE("[RelationalStorage] Get statement fail!"); + return errCode; + } + + sqlite3_stmt *logStmt = nullptr; + errCode = PrepareForSavingLog(object, deviceName, logStmt); + if (errCode != E_OK) { + LOGE("[RelationalStorage] Get statement fail!"); + return errCode; + } + + for (auto &item : dataItems) { + if (item.neglect) { // Do not save this record if it is neglected + continue; + } + errCode = SaveSyncDataItem(statement, item); + if (errCode != E_OK && errCode != -E_NOT_FOUND) { + break; + } + + item.dev = deviceName; + errCode = SaveSyncLog(logStmt, item, maxTimestamp); + if (errCode != E_OK) { + break; + } + maxTimestamp = std::max(item.timeStamp, maxTimestamp); + SQLiteUtils::ResetStatement(statement, false, errCode); + SQLiteUtils::ResetStatement(logStmt, false, errCode); + } + + if (errCode == -E_NOT_FOUND) { + errCode = E_OK; + } + SQLiteUtils::ResetStatement(logStmt, true, errCode); + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +int SQLiteSingleVerRelationalStorageExecutor::SaveSyncItems(const QueryObject &object, std::vector &dataItems, + const std::string &deviceName, TimeStamp &timeStamp) +{ + int errCode = StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + return errCode; + } + + errCode = SaveSyncDataItems(object, dataItems, deviceName, timeStamp); + if (errCode == E_OK) { + errCode = Commit(); + } else { + (void)Rollback(); // Keep the error code of the first scene + } + return errCode; +} + +static int GetLogInfoStatement(sqlite3 *dbHandle, const std::string &tableName, + uint64_t beginTime, uint64_t endTime, sqlite3_stmt *&statement) +{ + std::string sql = "select * from distributeddatamgr_aux_" + tableName + + "_log where flag=0x02 AND timestamp>=? AND timestamp &dataItems, size_t appendLength, + QueryObject query, const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const +{ + sqlite3_stmt *logStatement = nullptr; + int errCode = GetLogInfoStatement(dbHandle_, query.GetTableName(), timeRange.first, timeRange.second, logStatement); + if (errCode == E_OK) { + errCode = GetSyncDataItems(dataItems, logStatement, appendLength, dataSizeInfo); + } + SQLiteUtils::ResetStatement(logStatement, true, errCode); + return errCode; +} +} // namespace DistributedDB +#endif diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h new file mode 100644 index 0000000000000000000000000000000000000000..301204f61a909cb08b0af0d94146015c7b0295df --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_relational_storage_executor.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SQLITE_SINGLE_VER_RELATIONAL_STORAGE_EXECUTOR_H +#define SQLITE_SINGLE_VER_RELATIONAL_STORAGE_EXECUTOR_H +#ifdef RELATIONAL_STORE + +#include "macro_utils.h" +#include "db_types.h" +#include "sqlite_utils.h" +#include "sqlite_storage_executor.h" +#include "relational_store_delegate.h" +#include "query_object.h" + +namespace DistributedDB { +class SQLiteSingleVerRelationalStorageExecutor : public SQLiteStorageExecutor { +public: + SQLiteSingleVerRelationalStorageExecutor(sqlite3 *dbHandle, bool writable); + ~SQLiteSingleVerRelationalStorageExecutor() override = default; + + // Delete the copy and assign constructors + DISABLE_COPY_ASSIGN_MOVE(SQLiteSingleVerRelationalStorageExecutor); + + int CreateDistributedTable(const std::string &tableName, const RelationalStoreDelegate::TableOption &option); + + int StartTransaction(TransactType type); + int Commit(); + int Rollback(); + + int SetTableInfo(QueryObject query); + + // For Get sync data + int GetSyncDataByQuery(std::vector &dataItems, size_t appendLength, QueryObject query, + const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange) const; + int GetDeletedSyncDataByTimestamp(std::vector &dataItems, size_t appendLength, + TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const; + + // operation of meta data + int GetKvData(const Key &key, Value &value) const; + int PutKvData(const Key &key, const Value &value) const; + int DeleteMetaData(const std::vector &keys) const; + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const; + int GetAllMetaKeys(std::vector &keys) const; + + // For Put sync data + int SaveSyncItems(const QueryObject &object, std::vector &dataItems, + const std::string &deviceName, TimeStamp &timeStamp); + +private: + int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, + sqlite3_stmt *&statement, bool getDeletedData) const; + + int GetSyncDataItems(std::vector &dataItems, sqlite3_stmt *statement, + size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const; + + int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem) const; + + int SaveSyncDataItems(const QueryObject &object, std::vector &dataItems, + const std::string &deviceName, TimeStamp &timeStamp); + int SaveSyncDataItem(sqlite3_stmt *statement, const DataItem &dataItem); + + int DeleteSyncDataItem(const DataItem &dataItem); + + int SaveSyncLog(sqlite3_stmt *statement, const DataItem &dataItem, TimeStamp &maxTimestamp); + int PrepareForSavingData(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const; + int PrepareForSavingLog(const QueryObject &object, const std::string &deviceName, sqlite3_stmt *&statement) const; + + TableInfo table_; +}; +} // namespace DistributedDB +#endif +#endif // SQLITE_SINGLE_VER_RELATIONAL_STORAGE_EXECUTOR_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_result_set.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_result_set.h index 5fbe639543a2ab7d5dd4b08e633bfe9d1fda4ab8..d4c538304c52fe0d981c1edc79f01ffad129298a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_result_set.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_result_set.h @@ -80,20 +80,24 @@ private: void CloseForCacheEntryIdMode(); const Option option_; + // Common Part Of Two ResultSet Mode. bool isOpen_ = false; int count_ = 0; mutable int position_ = INIT_POSTION; // The position in the overall result mutable std::mutex mutex_; + // For KeyPrefix Type Or Query Type. const ResultSetType type_ = ResultSetType::KEYPREFIX; Key keyPrefix_; mutable QueryObject queryObj_; // Some QueryObject member function need to call is not a const function(BAD...) // Common Pointer For Use, Not Own it, Not Responsible To Release It. SQLiteSingleVerNaturalStore *kvDB_ = nullptr; + // Cache Full Entry Mode Using ResultEntriesWindow and IKvDBRawCursor, Own It, Responsible To Release It. ResultEntriesWindow *window_ = nullptr; IKvDBRawCursor *rawCursor_ = nullptr; + // Cache EntryId Mode Using StorageExecutor, Own It, Responsible To Release It. SQLiteSingleVerStorageExecutor *handle_ = nullptr; mutable std::vector cachedRowIds_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.h index 0f3d41a9d8587f83605fc1796db5fcc1e7ac79c3..4a534881a41d3e87a54942891d72c52d2bc2c57b 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_schema_database_upgrader.h @@ -30,6 +30,7 @@ public: protected: // Get an empty string with return_code E_OK indicate no schema but everything normally int GetDatabaseSchema(std::string &dbSchema) const override; + // Set or update schema into database file int SetDatabaseSchema(const std::string &dbSchema) override; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp index 61bfe157bc1acc210073c4a63743b2c010d9e02d..b99b34f90d97a77d2764ebe1e555ee647ce7993d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.cpp @@ -196,6 +196,37 @@ int SQLiteSingleVerStorageEngine::ReleaseHandleTransiently(SQLiteSingleVerStorag return errCode; } +int SQLiteSingleVerStorageEngine::AddSubscribeToMainDBInMigrate() +{ + LOGD("Add subscribe to mainDB from cache. %d", engineState_); + std::lock_guard lock(subscribeMutex_); + if (subscribeQuery_.empty()) { + return E_OK; + } + int errCode = E_OK; + auto handle = static_cast(FindExecutor(true, OperatePerm::NORMAL_PERM, errCode)); + if (errCode != E_OK || handle == nullptr) { + LOGE("Get available executor for add subscribe failed. %s", errCode); + return errCode; + } + errCode = handle->StartTransaction(TransactType::IMMEDIATE); + if (errCode != E_OK) { + goto END; + } + for (auto item : subscribeQuery_) { + errCode = handle->AddSubscribeTrigger(item.second, item.first); + if (errCode != E_OK) { + LOGE("Add subscribe trigger failed: %d id: %s", errCode, item.first.c_str()); + } + } + subscribeQuery_.clear(); + // Not rollback even if some triggers add failed. Users don’t perceive errors, add triggers as much as possible + (void)handle->Commit(); +END: + ReleaseExecutor(handle); + return errCode; +} + int SQLiteSingleVerStorageEngine::MigrateSyncData(SQLiteSingleVerStorageExecutor *&handle, bool &isNeedTriggerSync) { int errCode = E_OK; @@ -311,7 +342,7 @@ int SQLiteSingleVerStorageEngine::ReleaseExecutor(SQLiteSingleVerStorageExecutor int SQLiteSingleVerStorageEngine::FinishMigrateData(SQLiteSingleVerStorageExecutor *&handle, EngineState stateBeforeMigrate) { - LOGD("Begin to finish migrate and reinit db state!"); + LOGI("Begin to finish migrate and reinit db state!"); int errCode; if (handle == nullptr) { return -E_INVALID_ARGS; @@ -416,6 +447,8 @@ int SQLiteSingleVerStorageEngine::ExecuteMigrate() return errCode; } + isMigrating_.store(true); + LOGD("Migrate start."); bool isNeedTriggerSync = false; errCode = InitExecuteMigrate(handle, preState); if (errCode != E_OK) { @@ -423,9 +456,9 @@ int SQLiteSingleVerStorageEngine::ExecuteMigrate() goto END; } - LOGD("[SqlSingleVerEngine] Current enginState [%d] executorState [%d], begin to executing singleVer db migrate!", + LOGD("[SqlSingleVerEngine] Current engineState [%d] executorState [%d], begin to executing singleVer db migrate!", preState, executorState_); - // has been attach, Mark start of migration and it can migrating data + // has been attached, Mark start of migration and it can migrate data errCode = MigrateLocalData(handle); if (errCode != E_OK) { LOGE("Migrate local data fail, errCode = [%d]", errCode); @@ -450,6 +483,8 @@ int SQLiteSingleVerStorageEngine::ExecuteMigrate() END: // after FinishMigrateData, it will reset engine state // there is no need cover the errCode EndMigrate(handle, preState, errCode, isNeedTriggerSync); + isMigrating_.store(false); + LOGD("Migrate stop."); return errCode; } @@ -467,6 +502,11 @@ void SQLiteSingleVerStorageEngine::EndMigrate(SQLiteSingleVerStorageExecutor *&h if (errCode != E_OK) { LOGE("release executor after migrating! errCode = [%d]", errCode); } + + errCode = AddSubscribeToMainDBInMigrate(); + if (errCode != E_OK) { + LOGE("Add subscribe trigger after migrate sync data failed: %d", errCode); + } // Notify max timestamp offset for SyncEngine. // When time change offset equals 0, SyncEngine can adjust local time offset according to max timestamp. RuntimeContext::GetInstance()->NotifyTimeStampChanged(0); @@ -483,7 +523,7 @@ bool SQLiteSingleVerStorageEngine::IsEngineCorrupted() const StorageExecutor *SQLiteSingleVerStorageEngine::NewSQLiteStorageExecutor(sqlite3 *dbHandle, bool isWrite, bool isMemDb) { - auto executor = new (std::nothrow) SQLiteSingleVerStorageExecutor(dbHandle, isWrite, isMemDb); + auto executor = new (std::nothrow) SQLiteSingleVerStorageExecutor(dbHandle, isWrite, isMemDb, executorState_); if (executor == nullptr) { return executor; } @@ -491,7 +531,7 @@ StorageExecutor *SQLiteSingleVerStorageEngine::NewSQLiteStorageExecutor(sqlite3 return executor; } -int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(sqlite3 *&db) +int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(bool isWrite, sqlite3 *&db) { // Only could get the main database handle in the uninitialized and the main status. if (GetEngineState() != EngineState::INVALID && GetEngineState() != EngineState::MAINDB) { @@ -504,7 +544,12 @@ int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(sqlite3 *&db) DBConstant::SQLITE_DB_EXTENSION; } - int errCode = SQLiteUtils::OpenDatabase(option_, db); + OpenDbProperties optionTemp = option_; + if (!isWrite) { + optionTemp.createIfNecessary = false; + } + + int errCode = SQLiteUtils::OpenDatabase(optionTemp, db); if (errCode != E_OK) { if (errno == EKEYREVOKED) { LOGI("Failed to open the main database for key revoked[%d]", errCode); @@ -513,6 +558,7 @@ int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(sqlite3 *&db) return errCode; } + executorState_ = ExecutorState::MAINDB; // Set the engine state to main status for that the main database is valid. SetEngineState(EngineState::MAINDB); @@ -522,7 +568,6 @@ int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(sqlite3 *&db) errCode = AttachMainDbAndCacheDb(db, EngineState::MAINDB); if (errCode != E_OK) { LOGE("[SingleVerEngine][GetMain] Attach main db and cache db failed!, errCode = [%d]", errCode); - executorState_ = ExecutorState::MAINDB; return E_OK; // not care err to return, only use for print log } executorState_ = ExecutorState::MAIN_ATTACH_CACHE; @@ -535,9 +580,9 @@ int SQLiteSingleVerStorageEngine::TryToOpenMainDatabase(sqlite3 *&db) int SQLiteSingleVerStorageEngine::GetDbHandle(bool isWrite, const SecurityOption &secOpt, sqlite3 *&dbHandle) { - int errCode = TryToOpenMainDatabase(dbHandle); - LOGD("Finish to open the main database, write[%d], label[%d], flag[%d], errCode[%d]", - isWrite, secOpt.securityLabel, secOpt.securityFlag, errCode); + int errCode = TryToOpenMainDatabase(isWrite, dbHandle); + LOGD("Finish to open the main database, write[%d], label[%d], flag[%d], id[%.6s], errCode[%d]", isWrite, + secOpt.securityLabel, secOpt.securityFlag, DBCommon::TransferStringToHex(identifier_).c_str(), errCode); if (!(ParamCheckUtils::IsS3SECEOpt(secOpt) && errCode == -E_EKEYREVOKED)) { return errCode; } @@ -546,7 +591,7 @@ int SQLiteSingleVerStorageEngine::GetDbHandle(bool isWrite, const SecurityOption DBConstant::SQLITE_DB_EXTENSION; if (!isWrite || GetEngineState() != EngineState::INVALID || OS::CheckPathExistence(cacheDbPath)) { - LOGI("[SQLiteSingleStorageEng][GetDbHandle]Only use for first create cache db! [%d] [%d]", + LOGI("[SQLiteSingleStorageEng][GetDbHandle] Only use for first create cache db! [%d] [%d]", isWrite, GetEngineState()); return -E_EKEYREVOKED; } @@ -914,6 +959,12 @@ void SQLiteSingleVerStorageEngine::RegisterFunctionIfNeed(sqlite3 *dbHandle) con LOGW("[SqlSinEngine] RegisterFlatBufferExtractFunction fail, errCode = %d", errCode); } } + + // This function is used to update meta_data in triggers when it's attached to mainDB + int errCode = SQLiteUtils::RegisterMetaDataUpdateFunction(dbHandle); + if (errCode != E_OK) { + LOGW("[SqlSinEngine] RegisterMetaDataUpdateFunction fail, errCode = %d", errCode); + } } int SQLiteSingleVerStorageEngine::AttachMetaDatabase(sqlite3 *dbHandle, const OpenDbProperties &option) const @@ -1020,8 +1071,8 @@ void SQLiteSingleVerStorageEngine::SetMaxTimeStamp(TimeStamp maxTimeStamp) const void SQLiteSingleVerStorageEngine::CommitNotifyForMigrateCache(NotifyMigrateSyncData &syncData) const { - auto &isRemote = syncData.isRemote; - auto &isRemoveDeviceData = syncData.isRemoveDeviceData; + const auto &isRemote = syncData.isRemote; + const auto &isRemoveDeviceData = syncData.isRemoveDeviceData; auto &committedData = syncData.committedData; auto &entries = syncData.entries; @@ -1066,4 +1117,11 @@ void SQLiteSingleVerStorageEngine::CommitNotifyForMigrateCache(NotifyMigrateSync } return; } + +// Cache subscribe when engine state is CACHE mode, and its will be applied at the beginning of migrate. +void SQLiteSingleVerStorageEngine::CacheSubscribe(const std::string &subscribeId, const QueryObject &query) +{ + std::lock_guard lock(subscribeMutex_); + subscribeQuery_[subscribeId] = query; +} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.h index 87a7bbdaab15501e09aeefab651a5e3f5a9a6aa0..103f72e737f1206a2b26bfc766556298e1a2366d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_engine.h @@ -58,6 +58,8 @@ public: isNeedUpdateSecOpt_ = flag; } + void CacheSubscribe(const std::string &subscribeId, const QueryObject &query); + protected: StorageExecutor *NewSQLiteStorageExecutor(sqlite3 *dbHandle, bool isWrite, bool isMemDb) override; @@ -87,7 +89,7 @@ private: int EraseDeviceWaterMark(SQLiteSingleVerStorageExecutor *&handle, const std::vector &dataItems); // For db. - int TryToOpenMainDatabase(sqlite3 *&db); + int TryToOpenMainDatabase(bool isWrite, sqlite3 *&db); int GetCacheDbHandle(sqlite3 *&db); int GetDbHandle(bool isWrite, const SecurityOption &secOpt, sqlite3 *&dbHandle); int AttachMetaDatabase(sqlite3 *dbHandle, const OpenDbProperties &option) const; @@ -108,11 +110,17 @@ private: void InitConflictNotifiedFlag(SingleVerNaturalStoreCommitNotifyData *&committedData) const; void CommitNotifyForMigrateCache(NotifyMigrateSyncData &syncData) const; + // For subscribe + int AddSubscribeToMainDBInMigrate(); + mutable std::mutex migrateLock_; std::atomic cacheRecordVersion_; ExecutorState executorState_; bool isCorrupted_; bool isNeedUpdateSecOpt_; // update the option_ + + std::mutex subscribeMutex_; + std::map subscribeQuery_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp index 892bd7edf1175e54e7c2af84795bc0d0136958ac..9d3e9a2ff5d71d252f84c0a571eee3595046924b 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.cpp @@ -27,22 +27,38 @@ namespace DistributedDB { namespace { - void InitCommitNotifyDataKeyStatus(SingleVerNaturalStoreCommitNotifyData *committedData, const Key &hashKey, - const DataOperStatus &dataStatus) - { - if (committedData == nullptr) { - return; - } +void InitCommitNotifyDataKeyStatus(SingleVerNaturalStoreCommitNotifyData *committedData, const Key &hashKey, + const DataOperStatus &dataStatus) +{ + if (committedData == nullptr) { + return; + } - ExistStatus existedStatus = ExistStatus::NONE; - if (dataStatus.preStatus == DataStatus::DELETED) { - existedStatus = ExistStatus::DELETED; - } else if (dataStatus.preStatus == DataStatus::EXISTED) { - existedStatus = ExistStatus::EXIST; - } + ExistStatus existedStatus = ExistStatus::NONE; + if (dataStatus.preStatus == DataStatus::DELETED) { + existedStatus = ExistStatus::DELETED; + } else if (dataStatus.preStatus == DataStatus::EXISTED) { + existedStatus = ExistStatus::EXIST; + } - committedData->InitKeyPropRecord(hashKey, existedStatus); + committedData->InitKeyPropRecord(hashKey, existedStatus); +} + +int ResetOrRegetStmt(sqlite3 *db, sqlite3_stmt *&stmt, const std::string &sql) +{ + int errCode = E_OK; + SQLiteUtils::ResetStatement(stmt, false, errCode); + if (errCode != E_OK) { + LOGE("[ResetOrRegetStmt] reset stmt failed:%d.", errCode); + // Finish current statement and remade one + SQLiteUtils::ResetStatement(stmt, true, errCode); + errCode = SQLiteUtils::GetStatement(db, sql, stmt); + if (errCode != E_OK) { + LOGE("[ResetOrRegetStmt] reget failed:%d.", errCode); + } } + return errCode; +} } SQLiteSingleVerStorageExecutor::SQLiteSingleVerStorageExecutor(sqlite3 *dbHandle, bool writable, bool isMemDb) @@ -91,7 +107,7 @@ int SQLiteSingleVerStorageExecutor::GetKvData(SingleVerDataType type, const Key } else if (type == SingleVerDataType::SYNC_TYPE) { sql = SELECT_SYNC_VALUE_WTIMESTAMP_SQL; } else if (type == SingleVerDataType::META_TYPE) { - if (attachMetaMode_ == true) { + if (attachMetaMode_) { sql = SELECT_ATTACH_META_VALUE_SQL; } else { sql = SELECT_META_VALUE_SQL; @@ -283,7 +299,7 @@ int SQLiteSingleVerStorageExecutor::PutKvData(SingleVerDataType type, const Key if (isLocal && committedData != nullptr) { Entry entry = {key, value}; - committedData->InsertCommittedData(std::move(entry), (isExisted ? DataType::UPDATE : DataType::INSERT), true); + committedData->InsertCommittedData(std::move(entry), isExisted ? DataType::UPDATE : DataType::INSERT, true); } return E_OK; } @@ -317,18 +333,18 @@ END: int SQLiteSingleVerStorageExecutor::GetEntries(QueryObject &queryObj, std::vector &entries) const { - std::string sql; - int errCode = queryObj.GetQuerySql(sql); + int errCode = E_OK; + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); if (errCode != E_OK) { return errCode; } + sqlite3_stmt *statement = nullptr; - errCode = queryObj.GetQuerySqlStatement(dbHandle_, sql, statement); - if (errCode != E_OK) { - goto END; + errCode = helper.GetQuerySqlStatement(dbHandle_, false, statement); + if (errCode == E_OK) { + errCode = StepForResultEntries(statement, entries); } - errCode = StepForResultEntries(statement, entries); -END: + SQLiteUtils::ResetStatement(statement, true, errCode); return CheckCorruptedStatus(errCode); } @@ -339,8 +355,8 @@ int SQLiteSingleVerStorageExecutor::GetCount(QueryObject &queryObj, int &count) return -E_INVALID_DB; } - std::string countSql; - int errCode = queryObj.GetCountQuerySql(countSql); + int errCode = E_OK; + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); if (errCode != E_OK) { return errCode; } @@ -350,9 +366,15 @@ int SQLiteSingleVerStorageExecutor::GetCount(QueryObject &queryObj, int &count) return -E_INVALID_QUERY_FORMAT; } + std::string countSql; + errCode = helper.GetCountQuerySql(countSql); + if (errCode != E_OK) { + return errCode; + } + sqlite3_stmt *countStatement = nullptr; // get statement for count - errCode = queryObj.GetQuerySqlStatement(dbHandle_, countSql, countStatement); + errCode = helper.GetQuerySqlStatement(dbHandle_, countSql, countStatement); if (errCode != E_OK) { LOGE("Get count bind statement error:%d", errCode); goto END; @@ -389,7 +411,7 @@ void SQLiteSingleVerStorageExecutor::InitCurrentMaxStamp(TimeStamp &maxStamp) sqlite3_stmt *statement = nullptr; int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); if (errCode != E_OK) { - goto ERROR; + return; } errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); @@ -397,18 +419,18 @@ void SQLiteSingleVerStorageExecutor::InitCurrentMaxStamp(TimeStamp &maxStamp) maxStamp = static_cast(sqlite3_column_int64(statement, 0)); // get the first column LOGD("Max time stamp is %llu", maxStamp); } - -ERROR: SQLiteUtils::ResetStatement(statement, true, errCode); } int SQLiteSingleVerStorageExecutor::PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, - sqlite3_stmt *&statement) const + sqlite3_stmt *&statement, bool getDeletedData) const { if (dbHandle_ == nullptr) { return -E_INVALID_DB; } - int errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_SYNC_ENTRIES_SQL, statement); + + const std::string sql = (getDeletedData ? SELECT_SYNC_DELETED_ENTRIES_SQL : SELECT_SYNC_ENTRIES_SQL); + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); if (errCode != E_OK) { LOGE("Prepare the sync entries statement error:%d", errCode); return errCode; @@ -420,9 +442,6 @@ int SQLiteSingleVerStorageExecutor::PrepareForSyncDataByTime(TimeStamp begin, Ti } errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_END_STAMP_INDEX, end); - if (errCode != E_OK) { - goto ERROR; - } ERROR: if (errCode != E_OK) { @@ -444,17 +463,38 @@ void SQLiteSingleVerStorageExecutor::ReleaseContinueStatement() } } -int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector &dataItems, size_t appenedLength, - TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const +namespace { +int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem) { - size_t dataTotalSize = 0; - sqlite3_stmt *statement = nullptr; - int errCode = PrepareForSyncDataByTime(begin, end, statement); + dataItem.timeStamp = static_cast(sqlite3_column_int64(statement, SYNC_RES_TIME_INDEX)); + dataItem.writeTimeStamp = static_cast(sqlite3_column_int64(statement, SYNC_RES_W_TIME_INDEX)); + dataItem.flag = static_cast(sqlite3_column_int64(statement, SYNC_RES_FLAG_INDEX)); + dataItem.flag &= (~DataItem::LOCAL_FLAG); + std::vector devVect; + int errCode = SQLiteUtils::GetColumnBlobValue(statement, SYNC_RES_ORI_DEV_INDEX, devVect); if (errCode != E_OK) { return errCode; } + dataItem.origDev = std::string(devVect.begin(), devVect.end()); + int keyIndex = SYNC_RES_KEY_INDEX; + // If the data has been deleted, just use the hash key for sync. + if ((dataItem.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG) { + keyIndex = SYNC_RES_HASH_KEY_INDEX; + } + errCode = SQLiteUtils::GetColumnBlobValue(statement, keyIndex, dataItem.key); + if (errCode != E_OK) { + return errCode; + } + return SQLiteUtils::GetColumnBlobValue(statement, SYNC_RES_VAL_INDEX, dataItem.value); +} +} + +int SQLiteSingleVerStorageExecutor::GetSyncDataItems(std::vector &dataItems, sqlite3_stmt *statement, + size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const +{ + int errCode; + size_t dataTotalSize = 0; - dataItems.clear(); do { DataItem dataItem; errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); @@ -462,7 +502,7 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector errCode = GetDataItemForSync(statement, dataItem); } else { if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - LOGD("Get sync data finished, number: %zu, size: %zu", dataTotalSize, dataItems.size()); + LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); errCode = -E_FINISHED; } else { LOGE("Get sync data error:%d", errCode); @@ -471,7 +511,7 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector } // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. - dataTotalSize += GetDataItemSerialSize(dataItem, appenedLength); + dataTotalSize += GetDataItemSerialSize(dataItem, appendLength); if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || dataItems.size() >= dataSizeInfo.packetSize) { errCode = -E_UNFINISHED; @@ -480,11 +520,175 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector dataItems.push_back(std::move(dataItem)); } } while (true); + return errCode; +} + +int SQLiteSingleVerStorageExecutor::GetSyncDataByTimestamp(std::vector &dataItems, size_t appendLength, + TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = PrepareForSyncDataByTime(begin, end, statement); + if (errCode != E_OK) { + return errCode; + } + + errCode = GetSyncDataItems(dataItems, statement, appendLength, dataSizeInfo); + SQLiteUtils::ResetStatement(statement, true, errCode); + return CheckCorruptedStatus(errCode); +} + +int SQLiteSingleVerStorageExecutor::GetDeletedSyncDataByTimestamp(std::vector &dataItems, size_t appendLength, + TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const +{ + sqlite3_stmt *statement = nullptr; + int errCode = PrepareForSyncDataByTime(begin, end, statement, true); + if (errCode != E_OK) { + return errCode; + } + errCode = GetSyncDataItems(dataItems, statement, appendLength, dataSizeInfo); SQLiteUtils::ResetStatement(statement, true, errCode); return CheckCorruptedStatus(errCode); } +namespace { +int AppendDataItem(std::vector &dataItems, const DataItem &item, size_t &dataTotalSize, size_t appendLength, + const DataSizeSpecInfo &dataSizeInfo) +{ + // If dataTotalSize value is bigger than blockSize value , reserve the surplus data item. + dataTotalSize += SQLiteSingleVerStorageExecutor::GetDataItemSerialSize(item, appendLength); + if ((dataTotalSize > dataSizeInfo.blockSize && !dataItems.empty()) || + dataItems.size() >= dataSizeInfo.packetSize) { + return -E_UNFINISHED; + } else { + dataItems.push_back(item); + } + return E_OK; +} + +int GetFullDataStatement(sqlite3 *db, const std::pair &timeRange, sqlite3_stmt *&stmt) +{ + int errCode = SQLiteUtils::GetStatement(db, SELECT_SYNC_MODIFY_SQL, stmt); + if (errCode != E_OK) { + LOGE("Get statement failed. %d", errCode); + return errCode; + } + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 1, timeRange.first); // 1 : Bind time rang index start + if (errCode != E_OK) { + LOGE("Bind time range to statement failed. %d", errCode); + goto ERR; + } + errCode = SQLiteUtils::BindInt64ToStatement(stmt, 2, timeRange.second); // 2 : Bind time rang index end + if (errCode != E_OK) { + LOGE("Bind time range to statement failed. %d", errCode); + goto ERR; + } + return E_OK; // do not release statement when success +ERR: + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} + +int GetQueryDataStatement(sqlite3 *db, QueryObject query, const std::pair &timeRange, + sqlite3_stmt *&stmt) +{ + int errCode = E_OK; + SqliteQueryHelper helper = query.GetQueryHelper(errCode); + if (errCode != E_OK) { + return errCode; + } + return helper.GetQuerySyncStatement(db, timeRange.first, timeRange.second, stmt); +} + +int GetNextDataItem(sqlite3_stmt *stmt, bool isMemDB, DataItem &item) +{ + int errCode = SQLiteUtils::StepWithRetry(stmt, isMemDB); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + errCode = GetDataItemForSync(stmt, item); + } + return errCode; +} +} + +int SQLiteSingleVerStorageExecutor::GetSyncDataWithQuery(const QueryObject &query, size_t appendLength, + const DataSizeSpecInfo &dataSizeInfo, const std::pair &timeRange, + std::vector &dataItems) const +{ + sqlite3_stmt *fullStmt = nullptr; // statement for get all modified data in the time range + sqlite3_stmt *queryStmt = nullptr; // statement for get modified data which is matched query in the time range + int errCode = GetQueryDataStatement(dbHandle_, query, timeRange, queryStmt); + if (errCode != E_OK) { + LOGE("Get query matched data statement failed. %d", errCode); + goto END; + } + if (query.IsQueryOnlyByKey()) { + // Query sync by prefixKey only should not deal with REMOTE_DEVICE_DATA_MISS_QUERY. Get the data directly. + errCode = GetSyncDataItems(dataItems, queryStmt, appendLength, dataSizeInfo); + goto END; + } + errCode = GetFullDataStatement(dbHandle_, timeRange, fullStmt); + if (errCode != E_OK) { + LOGE("Get full changed data statement failed. %d", errCode); + goto END; + } + errCode = GetSyncDataWithQuery(fullStmt, queryStmt, appendLength, dataSizeInfo, dataItems); + if (errCode != E_OK && errCode != -E_UNFINISHED && errCode != -E_FINISHED) { + LOGE("Get sync data with query failed. %d", errCode); + } +END: + SQLiteUtils::ResetStatement(fullStmt, true, errCode); + SQLiteUtils::ResetStatement(queryStmt, true, errCode); + return CheckCorruptedStatus(errCode); +} + +int SQLiteSingleVerStorageExecutor::GetSyncDataWithQuery(sqlite3_stmt *fullStmt, sqlite3_stmt *queryStmt, + size_t appendLength, const DataSizeSpecInfo &dataSizeInfo, std::vector &dataItems) const +{ + int errCode = E_OK; + size_t dataTotalSize = 0; + DataItem fullItem; + DataItem matchItem; + bool isFullItemFinished = false; + bool isMatchItemFinished = false; + while (!isFullItemFinished || !isMatchItemFinished) { + errCode = GetNextDataItem(queryStmt, isMemDb_, matchItem); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { // query finished + isMatchItemFinished = true; + } else if (errCode != E_OK) { // step failed or get data failed + LOGE("Get next query matched data failed. %d", errCode); + return errCode; + } + while (!isFullItemFinished) { + errCode = GetNextDataItem(fullStmt, isMemDb_, fullItem); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { // queryStmt is a subset of fullStmt + isFullItemFinished = true; + break; + } else if (errCode != E_OK) { // step failed or get data failed + LOGE("Get next changed data failed. %d", errCode); + return errCode; + } + if (!isMatchItemFinished && matchItem.key == fullItem.key) { + errCode = AppendDataItem(dataItems, matchItem, dataTotalSize, appendLength, dataSizeInfo); + if (errCode == -E_UNFINISHED) { + goto END; + } + break; // step to next match data + } else { + DBCommon::CalcValueHash(fullItem.key, fullItem.key); + Value().swap(fullItem.value); // not send value when data miss query + fullItem.flag |= DataItem::REMOTE_DEVICE_DATA_MISS_QUERY; + errCode = AppendDataItem(dataItems, fullItem, dataTotalSize, appendLength, dataSizeInfo); + if (errCode == -E_UNFINISHED) { + goto END; + } + } + } + } +END: + LOGD("Get sync data finished, size of packet:%zu, number of item:%zu", dataTotalSize, dataItems.size()); + return (isFullItemFinished && isMatchItemFinished) ? -E_FINISHED : errCode; +} + int SQLiteSingleVerStorageExecutor::OpenResultSet(const Key &keyPrefix, int &count) { sqlite3_stmt *countStatement = nullptr; @@ -605,18 +809,19 @@ int SQLiteSingleVerStorageExecutor::OpenResultSetForCacheRowIdMode(QueryObject & if (dbHandle_ == nullptr) { return -E_INVALID_DB; } + int errCode = E_OK; - if (!queryObj.IsValid(errCode)) { - LOGE("[SqlSinExe][OpenResSetRowId][Query] Not Valid, errCode=%d", errCode); - return errCode; - } - std::string selectSql; - errCode = queryObj.GetQuerySql(selectSql, true); // only rowid sql + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); if (errCode != E_OK) { - LOGE("[SqlSinExe][OpenResSetRowId][Query] Get SQL fail, errCode=%d", errCode); return errCode; } - errCode = queryObj.GetQuerySqlStatement(dbHandle_, selectSql, getResultRowIdStatement_); + + if (!queryObj.IsValid()) { + LOGE("[SqlSinExe][OpenResSetRowId][Query] query object not Valid"); + return -E_INVALID_QUERY_FORMAT; + } + + errCode = helper.GetQuerySqlStatement(dbHandle_, true, getResultRowIdStatement_); if (errCode != E_OK) { LOGE("[SqlSinExe][OpenResSetRowId][Query] Get Stmt fail, errCode=%d", errCode); // The GetQuerySqlStatement does not self rollback(BAD...), so we have to reset the stmt here. @@ -632,16 +837,9 @@ int SQLiteSingleVerStorageExecutor::OpenResultSetForCacheRowIdMode(QueryObject & int SQLiteSingleVerStorageExecutor::ReloadResultSet(const Key &keyPrefix) { - int errCode = E_OK; - SQLiteUtils::ResetStatement(getResultRowIdStatement_, false, errCode); + int errCode = ResetOrRegetStmt(dbHandle_, getResultRowIdStatement_, SELECT_SYNC_ROWID_PREFIX_SQL); if (errCode != E_OK) { - LOGE("Reset result set rowid statement of keyPrefix error:%d", errCode); - SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); - errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_SYNC_ROWID_PREFIX_SQL, getResultRowIdStatement_); - if (errCode != E_OK) { - LOGE("Reset result set rowid statement of keyPrefix error:%d", errCode); - return CheckCorruptedStatus(errCode); - } + return CheckCorruptedStatus(errCode); } // No need to reset getResultEntryStatement_. Because the binding of it will be cleared in each get operation @@ -656,31 +854,29 @@ int SQLiteSingleVerStorageExecutor::ReloadResultSet(const Key &keyPrefix) int SQLiteSingleVerStorageExecutor::ReloadResultSet(QueryObject &queryObj) { int errCode = E_OK; - bool isValid = queryObj.IsValid(errCode); - if (!isValid) { + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); + if (errCode != E_OK) { return errCode; } + + if (!queryObj.IsValid()) { + return -E_INVALID_QUERY_FORMAT; + } + std::string sql; - errCode = queryObj.GetQuerySql(sql, true); // only rowid sql + errCode = helper.GetQuerySql(sql, true); // only rowid sql if (errCode != E_OK) { return errCode; } - SQLiteUtils::ResetStatement(getResultRowIdStatement_, false, errCode); + errCode = ResetOrRegetStmt(dbHandle_, getResultRowIdStatement_, sql); if (errCode != E_OK) { - LOGE("Reset result set rowid statement of query error:%d", errCode); - // Finish current statement and remade one - SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); - errCode = SQLiteUtils::GetStatement(dbHandle_, sql, getResultRowIdStatement_); - if (errCode != E_OK) { - LOGE("Reget result set rowid statement of query error:%d", errCode); - return CheckCorruptedStatus(errCode); - } + return CheckCorruptedStatus(errCode); } // No need to reset getResultEntryStatement_. Because the binding of it will be cleared in each get operation // GetQuerySqlStatement will not alter getResultRowIdStatement_ if it is not null - errCode = queryObj.GetQuerySqlStatement(dbHandle_, sql, getResultRowIdStatement_); + errCode = helper.GetQuerySqlStatement(dbHandle_, true, getResultRowIdStatement_); if (errCode != E_OK) { LOGE("Rebind result set rowid statement of query error:%d", errCode); return CheckCorruptedStatus(errCode); @@ -699,10 +895,9 @@ int SQLiteSingleVerStorageExecutor::ReloadResultSetForCacheRowIdMode(const Key & errCode = ResultSetLoadRowIdCache(rowIdCache, cacheLimit, cacheStartPos, count); if (errCode != E_OK) { LOGE("[SqlSinExe][ReloadResSet][KeyPrefix] Load fail, errCode=%d", errCode); - // We can just return, no need to reset the statement - return errCode; } - return E_OK; + // We can just return, no need to reset the statement + return errCode; } int SQLiteSingleVerStorageExecutor::ReloadResultSetForCacheRowIdMode(QueryObject &queryObj, @@ -716,10 +911,9 @@ int SQLiteSingleVerStorageExecutor::ReloadResultSetForCacheRowIdMode(QueryObject errCode = ResultSetLoadRowIdCache(rowIdCache, cacheLimit, cacheStartPos, count); if (errCode != E_OK) { LOGE("[SqlSinExe][ReloadResSet][Query] Load fail, errCode=%d", errCode); - // We can just return, no need to reset the statement - return errCode; } - return E_OK; + // We can just return, no need to reset the statement + return errCode; } int SQLiteSingleVerStorageExecutor::GetNextEntryFromResultSet(Key &key, Value &value, bool isCopy) @@ -908,8 +1102,7 @@ END: } void SQLiteSingleVerStorageExecutor::PutIntoCommittedData(const DataItem &itemPut, const DataItem &itemGet, - const DataOperStatus &status, const Key &hashKey, - SingleVerNaturalStoreCommitNotifyData *committedData) + const DataOperStatus &status, const Key &hashKey, SingleVerNaturalStoreCommitNotifyData *committedData) { if (committedData == nullptr) { return; @@ -933,18 +1126,29 @@ void SQLiteSingleVerStorageExecutor::PutIntoCommittedData(const DataItem &itemPu } } -int SQLiteSingleVerStorageExecutor::PrepareForSavingData(const std::string &readSql, const std::string &writeSql, - SaveRecordStatements &statements) const +int SQLiteSingleVerStorageExecutor::PrepareForSavingData(const std::string &readSql, const std::string &insertSql, + const std::string &updateSql, SaveRecordStatements &statements) const { int errCode = SQLiteUtils::GetStatement(dbHandle_, readSql, statements.queryStatement); if (errCode != E_OK) { - return errCode; + LOGE("Get query statement failed. errCode = [%d]", errCode); + goto ERR; } - errCode = SQLiteUtils::GetStatement(dbHandle_, writeSql, statements.putStatement); + errCode = SQLiteUtils::GetStatement(dbHandle_, insertSql, statements.insertStatement); if (errCode != E_OK) { - SQLiteUtils::ResetStatement(statements.queryStatement, true, errCode); + LOGE("Get insert statement failed. errCode = [%d]", errCode); + goto ERR; } + + errCode = SQLiteUtils::GetStatement(dbHandle_, updateSql, statements.updateStatement); + if (errCode != E_OK) { + LOGE("Get update statement failed. errCode = [%d]", errCode); + goto ERR; + } + return E_OK; +ERR: + (void)statements.ResetStatement(); return errCode; } @@ -952,9 +1156,10 @@ int SQLiteSingleVerStorageExecutor::PrepareForSavingData(SingleVerDataType type) { int errCode = -E_NOT_SUPPORT; if (type == SingleVerDataType::LOCAL_TYPE) { - errCode = PrepareForSavingData(SELECT_LOCAL_HASH_SQL, INSERT_LOCAL_SQL, saveLocalStatements_); + // currently, Local type has not been optimized, so pass updateSql parameter with INSERT_LOCAL_SQL + errCode = PrepareForSavingData(SELECT_LOCAL_HASH_SQL, INSERT_LOCAL_SQL, INSERT_LOCAL_SQL, saveLocalStatements_); } else if (type == SingleVerDataType::SYNC_TYPE) { - errCode = PrepareForSavingData(SELECT_SYNC_HASH_SQL, INSERT_SYNC_SQL, saveSyncStatements_); + errCode = PrepareForSavingData(SELECT_SYNC_HASH_SQL, INSERT_SYNC_SQL, UPDATE_SYNC_SQL, saveSyncStatements_); } return CheckCorruptedStatus(errCode); } @@ -963,10 +1168,12 @@ int SQLiteSingleVerStorageExecutor::ResetForSavingData(SingleVerDataType type) { int errCode = E_OK; if (type == SingleVerDataType::LOCAL_TYPE) { - SQLiteUtils::ResetStatement(saveLocalStatements_.putStatement, false, errCode); + SQLiteUtils::ResetStatement(saveLocalStatements_.insertStatement, false, errCode); + SQLiteUtils::ResetStatement(saveLocalStatements_.updateStatement, false, errCode); SQLiteUtils::ResetStatement(saveLocalStatements_.queryStatement, false, errCode); } else if (type == SingleVerDataType::SYNC_TYPE) { - SQLiteUtils::ResetStatement(saveSyncStatements_.putStatement, false, errCode); + SQLiteUtils::ResetStatement(saveSyncStatements_.insertStatement, false, errCode); + SQLiteUtils::ResetStatement(saveSyncStatements_.updateStatement, false, errCode); SQLiteUtils::ResetStatement(saveSyncStatements_.queryStatement, false, errCode); } return CheckCorruptedStatus(errCode); @@ -982,15 +1189,19 @@ std::string SQLiteSingleVerStorageExecutor::GetOriginDevName(const DataItem &dat } int SQLiteSingleVerStorageExecutor::SaveSyncDataToDatabase(const DataItem &dataItem, const Key &hashKey, - const std::string &origDev, const std::string &deviceName) + const std::string &origDev, const std::string &deviceName, bool isUpdate) { - auto statement = saveSyncStatements_.putStatement; + if ((dataItem.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) == DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) { + LOGD("Find query data missing, erase local data."); + return EraseSyncData(hashKey); + } + auto statement = saveSyncStatements_.GetDataSaveStatement(isUpdate); if (statement == nullptr) { return -E_INVALID_ARGS; } std::string devName = DBCommon::TransferHashString(deviceName); - int errCode = BindSavedSyncData(statement, dataItem, hashKey, {origDev, devName}); + int errCode = BindSavedSyncData(statement, dataItem, hashKey, {origDev, devName}, isUpdate); if (errCode != E_OK) { return errCode; } @@ -1006,7 +1217,8 @@ DataOperStatus SQLiteSingleVerStorageExecutor::JudgeSyncSaveType(DataItem &dataI const DataItem &itemGet, const std::string &devName, bool isHashKeyExisted) { DataOperStatus status; - status.isDeleted = ((dataItem.flag & DataItem::DELETE_FLAG) != 0); + status.isDeleted = ((dataItem.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG || + (dataItem.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) == DataItem::REMOTE_DEVICE_DATA_MISS_QUERY); if (isHashKeyExisted) { if ((itemGet.flag & DataItem::DELETE_FLAG) != 0) { status.preStatus = DataStatus::DELETED; @@ -1051,7 +1263,8 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataItemExt(const DataItem &dataItem, int SQLiteSingleVerStorageExecutor::ResetSaveSyncStatements(int errCode) { - SQLiteUtils::ResetStatement(saveSyncStatements_.putStatement, false, errCode); + SQLiteUtils::ResetStatement(saveSyncStatements_.insertStatement, false, errCode); + SQLiteUtils::ResetStatement(saveSyncStatements_.updateStatement, false, errCode); SQLiteUtils::ResetStatement(saveSyncStatements_.queryStatement, false, errCode); return CheckCorruptedStatus(errCode); } @@ -1137,13 +1350,14 @@ int SQLiteSingleVerStorageExecutor::SaveSyncDataItem(DataItem &dataItem, const D } PutConflictData(dataItem, notify.getData, deviceInfo, notify.dataStatus, committedData); - if (notify.dataStatus.isDefeated == true) { + if (notify.dataStatus.isDefeated) { LOGD("Data status is defeated:%d", errCode); return ResetSaveSyncStatements(errCode); } + bool isUpdate = (notify.dataStatus.preStatus != DataStatus::NOEXISTED); std::string origDev = GetOriginDevName(dataItem, notify.getData.origDev); - errCode = SaveSyncDataToDatabase(dataItem, notify.hashKey, origDev, deviceInfo.deviceName); + errCode = SaveSyncDataToDatabase(dataItem, notify.hashKey, origDev, deviceInfo.deviceName, isUpdate); if (errCode == E_OK) { PutIntoCommittedData(dataItem, notify.getData, notify.dataStatus, notify.hashKey, committedData); maxStamp = std::max(dataItem.timeStamp, maxStamp); @@ -1156,12 +1370,8 @@ int SQLiteSingleVerStorageExecutor::SaveSyncDataItem(DataItem &dataItem, const D int SQLiteSingleVerStorageExecutor::GetAllMetaKeys(std::vector &keys) const { sqlite3_stmt *statement = nullptr; - int errCode = -E_BASE; - if (attachMetaMode_ == true) { - errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_ATTACH_ALL_META_KEYS, statement); - } else { - errCode = SQLiteUtils::GetStatement(dbHandle_, SELECT_ALL_META_KEYS, statement); - } + const std::string &sqlStr = (attachMetaMode_ ? SELECT_ATTACH_ALL_META_KEYS : SELECT_ALL_META_KEYS); + int errCode = SQLiteUtils::GetStatement(dbHandle_, sqlStr, statement); if (errCode != E_OK) { LOGE("[SingleVerExe][GetAllKey] Get statement failed:%d", errCode); return errCode; @@ -1184,18 +1394,17 @@ int SQLiteSingleVerStorageExecutor::GetAllSyncedEntries(const std::string &devic return errCode; } - // When removing device data in cache mode, key is "remove", value is deviceID's hashstring. - // Therefore no need to transfer hashstring when migrating. + // When removing device data in cache mode, key is "remove", value is deviceID's hash string. + // Therefore, no need to transfer hash string when migrating. std::string devName = isSyncMigrating_ ? deviceName : DBCommon::TransferHashString(deviceName); std::vector devVect(devName.begin(), devName.end()); errCode = SQLiteUtils::BindBlobToStatement(statement, 1, devVect, true); // bind the 1st to device. if (errCode != E_OK) { LOGE("Failed to bind the synced device for all entries:%d", errCode); - goto ERROR; + } else { + errCode = GetAllEntries(statement, entries); } - errCode = GetAllEntries(statement, entries); -ERROR: SQLiteUtils::ResetStatement(statement, true, errCode); return errCode; } @@ -1261,9 +1470,10 @@ int SQLiteSingleVerStorageExecutor::GetAllKeys(sqlite3_stmt *statement, std::vec } int SQLiteSingleVerStorageExecutor::BindSavedSyncData(sqlite3_stmt *statement, const DataItem &dataItem, - const Key &hashKey, const SyncDataDevices &devices, int beginIndex) + const Key &hashKey, const SyncDataDevices &devices, bool isUpdate) { - int errCode = SQLiteUtils::BindBlobToStatement(statement, beginIndex + BIND_SYNC_HASH_KEY_INDEX, hashKey, false); + const int hashKeyIndex = isUpdate ? BIND_SYNC_UPDATE_HASH_KEY_INDEX : BIND_SYNC_HASH_KEY_INDEX; + int errCode = SQLiteUtils::BindBlobToStatement(statement, hashKeyIndex, hashKey, false); if (errCode != E_OK) { LOGE("Bind saved sync data hash key failed:%d", errCode); return errCode; @@ -1271,9 +1481,9 @@ int SQLiteSingleVerStorageExecutor::BindSavedSyncData(sqlite3_stmt *statement, c // if delete flag is set, just use the hash key instead of the key if ((dataItem.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG) { - errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_zeroblob(statement, beginIndex + BIND_SYNC_KEY_INDEX, -1)); + errCode = SQLiteUtils::MapSQLiteErrno(sqlite3_bind_zeroblob(statement, BIND_SYNC_KEY_INDEX, -1)); } else { - errCode = SQLiteUtils::BindBlobToStatement(statement, beginIndex + BIND_SYNC_KEY_INDEX, dataItem.key, false); + errCode = SQLiteUtils::BindBlobToStatement(statement, BIND_SYNC_KEY_INDEX, dataItem.key, false); } if (errCode != E_OK) { @@ -1281,20 +1491,20 @@ int SQLiteSingleVerStorageExecutor::BindSavedSyncData(sqlite3_stmt *statement, c return errCode; } - errCode = SQLiteUtils::BindBlobToStatement(statement, beginIndex + BIND_SYNC_VAL_INDEX, dataItem.value, true); + errCode = SQLiteUtils::BindBlobToStatement(statement, BIND_SYNC_VAL_INDEX, dataItem.value, true); if (errCode != E_OK) { LOGE("Bind saved sync data value failed:%d", errCode); return errCode; } - errCode = SQLiteUtils::BindInt64ToStatement(statement, beginIndex + BIND_SYNC_STAMP_INDEX, dataItem.timeStamp); + errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_SYNC_STAMP_INDEX, dataItem.timeStamp); if (errCode != E_OK) { LOGE("Bind saved sync data stamp failed:%d", errCode); return errCode; } - errCode = SQLiteUtils::BindInt64ToStatement(statement, beginIndex + BIND_SYNC_W_TIME_INDEX, - dataItem.writeTimeStamp); + const int writeTimeIndex = isUpdate ? BIND_SYNC_UPDATE_W_TIME_INDEX : BIND_SYNC_W_TIME_INDEX; + errCode = SQLiteUtils::BindInt64ToStatement(statement, writeTimeIndex, dataItem.writeTimeStamp); LOGD("Write timestamp:%llu timestamp:%llu, %llu", dataItem.writeTimeStamp, dataItem.timeStamp, dataItem.flag); if (errCode != E_OK) { LOGE("Bind saved sync data write stamp failed:%d", errCode); @@ -1323,7 +1533,8 @@ void SQLiteSingleVerStorageExecutor::PutConflictData(const DataItem &itemPut, co } Key origKey; - if ((itemPut.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG) { + if ((itemPut.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG || + (itemPut.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) == DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) { origKey = itemGet.key; } else { origKey = itemPut.key; @@ -1367,7 +1578,8 @@ int SQLiteSingleVerStorageExecutor::GetSyncDataItemPre(const DataItem &itemPut, { if (isSyncMigrating_) { hashKey = itemPut.hashKey; - } else if ((itemPut.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG) { + } else if ((itemPut.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG || + ((itemPut.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) == DataItem::REMOTE_DEVICE_DATA_MISS_QUERY)) { hashKey = itemPut.key; } else { int errCode = DBCommon::CalcValueHash(itemPut.key, hashKey); @@ -1472,6 +1684,34 @@ int SQLiteSingleVerStorageExecutor::DeleteLocalKvData(const Key &key, return DeleteLocalDataInner(committedData, key, value); } +int SQLiteSingleVerStorageExecutor::EraseSyncData(const Key &hashKey) +{ + sqlite3_stmt *stmt = nullptr; + std::string sql = (executorState_ == ExecutorState::CACHE_ATTACH_MAIN) ? + DELETE_SYNC_DATA_WITH_HASHKEY_FROM_CACHEHANDLE : DELETE_SYNC_DATA_WITH_HASHKEY; + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, stmt); + if (errCode != E_OK) { + LOGE("get erase statement failed:%d", errCode); + return errCode; + } + + errCode = SQLiteUtils::BindBlobToStatement(stmt, 1, hashKey, false); + if (errCode != E_OK) { + LOGE("bind hashKey failed:%d", errCode); + goto END; + } + + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } else { + LOGE("erase data failed:%d", errCode); + } +END: + SQLiteUtils::ResetStatement(stmt, true, errCode); + return CheckCorruptedStatus(errCode); +} + int SQLiteSingleVerStorageExecutor::RemoveDeviceData(const std::string &deviceName) { // Transfer the device name. @@ -1540,9 +1780,9 @@ int SQLiteSingleVerStorageExecutor::StepForResultEntries(sqlite3_stmt *statement } int SQLiteSingleVerStorageExecutor::BindDevForSavedSyncData(sqlite3_stmt *statement, const DataItem &dataItem, - const std::string &origDev, const std::string &deviceName, int beginIndex) + const std::string &origDev, const std::string &deviceName) { - int errCode = SQLiteUtils::BindInt64ToStatement(statement, beginIndex + BIND_SYNC_FLAG_INDEX, + int errCode = SQLiteUtils::BindInt64ToStatement(statement, BIND_SYNC_FLAG_INDEX, static_cast(dataItem.flag)); if (errCode != E_OK) { LOGE("Bind saved sync data flag failed:%d", errCode); @@ -1550,14 +1790,14 @@ int SQLiteSingleVerStorageExecutor::BindDevForSavedSyncData(sqlite3_stmt *statem } std::vector devVect(deviceName.begin(), deviceName.end()); - errCode = SQLiteUtils::BindBlobToStatement(statement, beginIndex + BIND_SYNC_DEV_INDEX, devVect, true); + errCode = SQLiteUtils::BindBlobToStatement(statement, BIND_SYNC_DEV_INDEX, devVect, true); if (errCode != E_OK) { LOGE("Bind dev for sync data failed:%d", errCode); return errCode; } std::vector origDevVect(origDev.begin(), origDev.end()); - errCode = SQLiteUtils::BindBlobToStatement(statement, beginIndex + BIND_SYNC_ORI_DEV_INDEX, origDevVect, true); + errCode = SQLiteUtils::BindBlobToStatement(statement, BIND_SYNC_ORI_DEV_INDEX, origDevVect, true); if (errCode != E_OK) { LOGE("Bind orig dev for sync data failed:%d", errCode); } @@ -1577,33 +1817,6 @@ size_t SQLiteSingleVerStorageExecutor::GetDataItemSerialSize(const DataItem &ite return dataSize; } -int SQLiteSingleVerStorageExecutor::GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem) -{ - dataItem.timeStamp = static_cast(sqlite3_column_int64(statement, SYNC_RES_TIME_INDEX)); - dataItem.writeTimeStamp = static_cast(sqlite3_column_int64(statement, SYNC_RES_W_TIME_INDEX)); - dataItem.flag = static_cast(sqlite3_column_int64(statement, SYNC_RES_FLAG_INDEX)); - dataItem.flag &= (~DataItem::LOCAL_FLAG); - std::vector devVect; - int errCode = SQLiteUtils::GetColumnBlobValue(statement, SYNC_RES_ORI_DEV_INDEX, devVect); - if (errCode != E_OK) { - return errCode; - } - - dataItem.origDev = std::string(devVect.begin(), devVect.end()); - int keyIndex = SYNC_RES_KEY_INDEX; - // If the data has been deleted, just use the hash key for sync. - if ((dataItem.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG) { - keyIndex = SYNC_RES_HASH_KEY_INDEX; - } - - errCode = SQLiteUtils::GetColumnBlobValue(statement, keyIndex, dataItem.key); - if (errCode != E_OK) { - return errCode; - } - - return SQLiteUtils::GetColumnBlobValue(statement, SYNC_RES_VAL_INDEX, dataItem.value); -} - int SQLiteSingleVerStorageExecutor::InitResultSet(const Key &keyPrefix, sqlite3_stmt *&countStmt) { if (dbHandle_ == nullptr) { @@ -1654,30 +1867,30 @@ int SQLiteSingleVerStorageExecutor::InitResultSetCount(QueryObject &queryObj, sq return -E_INVALID_DB; } - std::string countSql; - int errCode = queryObj.GetCountQuerySql(countSql); + int errCode = E_OK; + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); if (errCode != E_OK) { return errCode; } - errCode = queryObj.GetCountSqlStatement(dbHandle_, countSql, countStmt); + errCode = helper.GetCountSqlStatement(dbHandle_, countStmt); if (errCode != E_OK) { LOGE("Get count bind statement error:%d", errCode); SQLiteUtils::ResetStatement(countStmt, true, errCode); - return errCode; } return errCode; } int SQLiteSingleVerStorageExecutor::InitResultSetContent(QueryObject &queryObj) { - std::string selectSql; - int errCode = queryObj.GetQuerySql(selectSql, true); // only rowid sql + int errCode = E_OK; + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); if (errCode != E_OK) { return errCode; } + // bind statement for result set - errCode = queryObj.GetQuerySqlStatement(dbHandle_, selectSql, getResultRowIdStatement_); + errCode = helper.GetQuerySqlStatement(dbHandle_, true, getResultRowIdStatement_); if (errCode != E_OK) { LOGE("[SqlSinExe][InitResSetContent] Bind result set rowid statement of query error:%d", errCode); SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); @@ -1696,12 +1909,17 @@ int SQLiteSingleVerStorageExecutor::InitResultSet(QueryObject &queryObj, sqlite3 if (dbHandle_ == nullptr) { return -E_INVALID_DB; } + int errCode = E_OK; - bool isValid = queryObj.IsValid(errCode); - if (!isValid) { + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); + if (errCode != E_OK) { return errCode; } + if (!queryObj.IsValid()) { + return -E_INVALID_QUERY_FORMAT; + } + errCode = InitResultSetCount(queryObj, countStmt); if (errCode != E_OK) { return CheckCorruptedStatus(errCode); @@ -1765,11 +1983,9 @@ int SQLiteSingleVerStorageExecutor::GetOneRawDataItem(sqlite3_stmt *statement, D if (errCode != E_OK) { return errCode; } - if (!isCacheDb) { - return E_OK; + if (isCacheDb) { + verInCurCacheDb = static_cast(sqlite3_column_int64(statement, SYNC_RES_VERSION_INDEX)); } - verInCurCacheDb = static_cast(sqlite3_column_int64(statement, SYNC_RES_VERSION_INDEX)); - return E_OK; } @@ -1851,27 +2067,36 @@ int SQLiteSingleVerStorageExecutor::ResultSetLoadRowIdCache(std::vector return E_OK; } -void SQLiteSingleVerStorageExecutor::FinalizeAllStatements() +int SQLiteSingleVerStorageExecutor::SaveRecordStatements::ResetStatement() { int errCode = E_OK; - SQLiteUtils::ResetStatement(saveLocalStatements_.putStatement, true, errCode); + SQLiteUtils::ResetStatement(insertStatement, true, errCode); if (errCode != E_OK) { - LOGE("Finalize saveLocal put statements failed, error: %d", errCode); + LOGE("Finalize insert statements failed, error: %d", errCode); } - SQLiteUtils::ResetStatement(saveLocalStatements_.queryStatement, true, errCode); + SQLiteUtils::ResetStatement(updateStatement, true, errCode); if (errCode != E_OK) { - LOGE("Finalize saveLocal query statement failed, error: %d", errCode); + LOGE("Finalize update statements failed, error: %d", errCode); } - SQLiteUtils::ResetStatement(saveSyncStatements_.putStatement, true, errCode); + SQLiteUtils::ResetStatement(queryStatement, true, errCode); if (errCode != E_OK) { - LOGE("Finalize saveSync put statement failed, error: %d", errCode); + LOGE("Finalize query statement failed, error: %d", errCode); + } + return errCode; +} + +void SQLiteSingleVerStorageExecutor::FinalizeAllStatements() +{ + int errCode = saveLocalStatements_.ResetStatement(); + if (errCode != E_OK) { + LOGE("Finalize saveLocal statements failed, error: %d", errCode); } - SQLiteUtils::ResetStatement(saveSyncStatements_.queryStatement, true, errCode); + errCode = saveSyncStatements_.ResetStatement(); if (errCode != E_OK) { - LOGE("Finalize saveSync query statement failed, error: %d", errCode); + LOGE("Finalize saveSync statement failed, error: %d", errCode); } SQLiteUtils::ResetStatement(getResultRowIdStatement_, true, errCode); @@ -1884,14 +2109,9 @@ void SQLiteSingleVerStorageExecutor::FinalizeAllStatements() LOGE("Finalize getResultEntryStatement_ failed, error: %d", errCode); } - SQLiteUtils::ResetStatement(migrateSyncStatements_.putStatement, true, errCode); - if (errCode != E_OK) { - LOGE("Finalize migrateSync put statements failed, error: %d", errCode); - } - - SQLiteUtils::ResetStatement(migrateSyncStatements_.queryStatement, true, errCode); + errCode = migrateSyncStatements_.ResetStatement(); if (errCode != E_OK) { - LOGE("Finalize migrateSync query statement failed, error: %d", errCode); + LOGE("Finalize migrateSync statements failed, error: %d", errCode); } ReleaseContinueStatement(); @@ -1903,4 +2123,13 @@ void SQLiteSingleVerStorageExecutor::SetConflictResolvePolicy(int policy) conflictResolvePolicy_ = policy; } } + +int SQLiteSingleVerStorageExecutor::CheckIntegrity() const +{ + if (dbHandle_ == nullptr) { + return -E_INVALID_DB; + } + + return SQLiteUtils::CheckIntegrity(dbHandle_, CHECK_DB_INTEGRITY_SQL); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h index a1ea09068a357718d9370122fc2206c87038815f..021199aeda551972aa919cd6c535ca4e126152fe 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor.h @@ -129,6 +129,9 @@ public: int DeleteLocalKvData(const Key &key, SingleVerNaturalStoreCommitNotifyData *committedData, Value &value, TimeStamp &timeStamp); + // delete a row data by hashKey, with no tombstone left. + int EraseSyncData(const Key &hashKey); + int RemoveDeviceData(const std::string &deviceName); int RemoveDeviceDataInCacheMode(const std::string &deviceName, bool isNeedNotify, uint64_t recordVersion) const; @@ -137,7 +140,9 @@ public: void ReleaseContinueStatement(); - int GetSyncDataByTimestamp(std::vector &dataItems, size_t appenedLength, TimeStamp begin, + int GetSyncDataByTimestamp(std::vector &dataItems, size_t appendedLength, TimeStamp begin, + TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const; + int GetDeletedSyncDataByTimestamp(std::vector &dataItems, size_t appendedLength, TimeStamp begin, TimeStamp end, const DataSizeSpecInfo &dataSizeInfo) const; int GetDeviceIdentifier(PragmaEntryDeviceIdentifier *identifier); @@ -188,8 +193,8 @@ public: int PutLocalDataToCacheDB(const LocalDataItem &dataItem) const; - int SaveSyncDataItemInCacheMode(DataItem &dataItem, - const DeviceInfo &deviceInfo, TimeStamp &maxStamp, uint64_t recordVersion); + int SaveSyncDataItemInCacheMode(DataItem &dataItem, const DeviceInfo &deviceInfo, TimeStamp &maxStamp, + uint64_t recordVersion, const QueryObject &query); int PrepareForSavingCacheData(SingleVerDataType type); int ResetForSavingCacheData(SingleVerDataType type); @@ -212,24 +217,54 @@ public: void SetConflictResolvePolicy(int policy); + // Delete multiple meta data records in a transaction. + int DeleteMetaData(const std::vector &keys); + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix); + + int CheckIntegrity() const; + + int CheckQueryObjectLegal(QueryObject &queryObj) const; + + int CheckDataWithQuery(QueryObject query, std::vector &dataItems, const DeviceInfo &deviceInfo); + + static size_t GetDataItemSerialSize(const DataItem &item, size_t appendLen); + + int AddSubscribeTrigger(QueryObject &query, const std::string &subscribeId); + + int RemoveSubscribeTrigger(const std::vector &subscribeIds); + + int RemoveSubscribeTriggerWaterMark(const std::vector &subscribeIds); + + int GetTriggers(const std::string &namePreFix, std::vector &triggerNames); + + int RemoveTrigger(const std::vector &triggers); + + int GetSyncDataWithQuery(const QueryObject &query, size_t appendLength, const DataSizeSpecInfo &dataSizeInfo, + const std::pair &timeRange, std::vector &dataItems) const; + private: struct SaveRecordStatements { sqlite3_stmt *queryStatement = nullptr; - sqlite3_stmt *putStatement = nullptr; + sqlite3_stmt *insertStatement = nullptr; + sqlite3_stmt *updateStatement = nullptr; + + int ResetStatement(); + + inline sqlite3_stmt *GetDataSaveStatement(bool isUpdate) const + { + return isUpdate ? updateStatement : insertStatement; + } }; void PutIntoCommittedData(const DataItem &itemPut, const DataItem &itemGet, const DataOperStatus &status, const Key &hashKey, SingleVerNaturalStoreCommitNotifyData *committedData); static int BindSavedSyncData(sqlite3_stmt *statement, const DataItem &dataItem, const Key &hashKey, - const SyncDataDevices &devices, int beginIndex = 0); + const SyncDataDevices &devices, bool isUpdate); static int BindDevForSavedSyncData(sqlite3_stmt *statement, const DataItem &dataItem, const std::string &origDev, - const std::string &deviceName, int beginIndex = 0); - - static int GetDataItemForSync(sqlite3_stmt *statement, DataItem &dataItem); - - static size_t GetDataItemSerialSize(const DataItem &item, size_t appendLen); + const std::string &deviceName); static void PutConflictData(const DataItem &itemPut, const DataItem &itemGet, const DeviceInfo &deviceInfo, const DataOperStatus &dataStatus, SingleVerNaturalStoreCommitNotifyData *commitData); @@ -245,7 +280,8 @@ private: int GetSyncDataPreByHashKey(const Key &hashKey, DataItem &itemGet) const; - int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement) const; + int PrepareForSyncDataByTime(TimeStamp begin, TimeStamp end, sqlite3_stmt *&statement, bool getDeletedData = false) + const; int StepForResultEntries(sqlite3_stmt *statement, std::vector &entries) const; @@ -265,15 +301,15 @@ private: SingleVerDataType type); int SaveSyncDataToDatabase(const DataItem &dataItem, const Key &hashKey, const std::string &origDev, - const std::string &deviceName); + const std::string &deviceName, bool isUpdate); int SaveKvData(SingleVerDataType type, const Key &key, const Value &value, TimeStamp timestamp); int DeleteLocalDataInner(SingleVerNaturalStoreCommitNotifyData *committedData, const Key &key, const Value &value); - int PrepareForSavingData(const std::string &readSql, const std::string &writeSql, - SaveRecordStatements &statements) const; + int PrepareForSavingData(const std::string &readSql, const std::string &insertSql, + const std::string &updateSql, SaveRecordStatements &statements) const; int OpenResultSetForCacheRowIdModeCommon(std::vector &rowIdCache, uint32_t cacheLimit, int &count); @@ -304,47 +340,72 @@ private: // use for migrating data int BindLocalDataInCacheMode(sqlite3_stmt *statement, const LocalDataItem &dataItem) const; - int PutMigratingDataToMain(const std::vector &dataItems) const; // Process timestamp for syncdata in cacheDB when migrating. int ProcessTimeStampForSyncDataInCacheDB(std::vector &dataItems); + // Get migrateTimeOffset_. int InitMigrateTimeStampOffset(); + // Get min timestamp of local data in sync_data, cacheDB. int GetMinTimestampInCacheDB(TimeStamp &minStamp) const; // Prepare conflict notify and commit notify data. int PrepareForNotifyConflictAndObserver(DataItem &dataItem, const DeviceInfo &deviceInfo, NotifyConflictAndObserverData ¬ify); + // Put observer and conflict data into commit notify when migrating cacheDB. int PutIntoConflictAndCommitForMigrateCache(DataItem &dataItem, const DeviceInfo &deviceInfo, NotifyConflictAndObserverData ¬ify); - // Notify when migrating cacheDB. - int NotifyForMigrateCacheDB(std::vector &dataItems, NotifyMigrateSyncData &syncData); + + int MigrateDataItems(std::vector &dataItems, NotifyMigrateSyncData &syncData); + + int MigrateDataItem(DataItem &dataItem, NotifyMigrateSyncData &syncData); + int GetEntriesForNotifyRemoveDevData(const DataItem &item, std::vector &entries) const; // Reset migrateSyncStatements_. int ResetForMigrateCacheData(); + // Init migrating data. int InitMigrateData(); int MigrateRmDevData(const DataItem &dataItem) const; int VacuumLocalData() const; + int GetSyncDataItems(std::vector &dataItems, sqlite3_stmt *statement, + size_t appendLength, const DataSizeSpecInfo &dataSizeInfo) const; + + int GetSyncDataWithQuery(sqlite3_stmt *fullStmt, sqlite3_stmt *queryStmt, + size_t appendLength, const DataSizeSpecInfo &dataSizeInfo, std::vector &dataItems) const; + + int CheckMissQueryDataItems(sqlite3_stmt *&stmt, const SqliteQueryHelper &helper, const DeviceInfo &deviceInfo, + std::vector &dataItems); + + int CheckDataWithQuery(std::vector &dataItems); + + int GetExpandedCheckSql(QueryObject query, DataItem &dataItem); + + int CheckMissQueryDataItem(sqlite3_stmt *stmt, const std::string &deviceName, DataItem &item); + sqlite3_stmt *getSyncStatement_; sqlite3_stmt *getResultRowIdStatement_; sqlite3_stmt *getResultEntryStatement_; SaveRecordStatements saveSyncStatements_; SaveRecordStatements saveLocalStatements_; + // Used for migrating sync_data. SaveRecordStatements migrateSyncStatements_; bool isTransactionOpen_; bool attachMetaMode_; // true for attach meta mode ExecutorState executorState_; + // Max timestamp in mainDB. Used for migrating. TimeStamp maxTimeStampInMainDB_; + // The offset between min timestamp in cacheDB and max timestamp in mainDB. Used for migrating. TimeOffset migrateTimeOffset_; + // Migrating sync flag. When the flag is true, mainDB and cacheDB are attached, migrateSyncStatements_ is set, // maxTimeStampInMainDB_ and migrateTimeOffset_ is meaningful. bool isSyncMigrating_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp index ea9a1a1d3438750b9e29cc72cafcfac594d0cad1..872e873613f4d6e68993aaf08d497bee2732be84 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp @@ -32,13 +32,18 @@ int SQLiteSingleVerStorageExecutor::PrepareForSavingCacheData(SingleVerDataType if (type == SingleVerDataType::LOCAL_TYPE) { std::string insertLocalSql = ((executorState_ == ExecutorState::CACHE_ATTACH_MAIN) ? INSERT_LOCAL_SQL_FROM_CACHEHANDLE : INSERT_CACHE_LOCAL_SQL); - errCode = PrepareForSavingData(SELECT_CACHE_LOCAL_HASH_SQL, insertLocalSql, saveLocalStatements_); + std::string updateLocalSql = ((executorState_ == ExecutorState::CACHE_ATTACH_MAIN) ? + UPDATE_LOCAL_SQL_FROM_CACHEHANDLE : UPDATE_CACHE_LOCAL_SQL); + errCode = PrepareForSavingData(SELECT_CACHE_LOCAL_HASH_SQL, insertLocalSql, updateLocalSql, + saveLocalStatements_); } else if (type == SingleVerDataType::SYNC_TYPE) { std::string insertSyncSql = ((executorState_ == ExecutorState::MAIN_ATTACH_CACHE) ? INSERT_CACHE_SYNC_SQL_FROM_MAINHANDLE : INSERT_CACHE_SYNC_SQL); + std::string updateSyncSql = ((executorState_ == ExecutorState::MAIN_ATTACH_CACHE) ? + UPDATE_CACHE_SYNC_SQL_FROM_MAINHANDLE : UPDATE_CACHE_SYNC_SQL); std::string selectSyncHashSql = ((executorState_ == ExecutorState::MAIN_ATTACH_CACHE) ? SELECT_CACHE_SYNC_HASH_SQL_FROM_MAINHANDLE : SELECT_CACHE_SYNC_HASH_SQL); - errCode = PrepareForSavingData(selectSyncHashSql, insertSyncSql, saveSyncStatements_); + errCode = PrepareForSavingData(selectSyncHashSql, insertSyncSql, updateSyncSql, saveSyncStatements_); } if (errCode != E_OK) { LOGE("Prepare to save sync cache data failed:%d", errCode); @@ -50,10 +55,12 @@ int SQLiteSingleVerStorageExecutor::ResetForSavingCacheData(SingleVerDataType ty { int errCode = E_OK; if (type == SingleVerDataType::LOCAL_TYPE) { - SQLiteUtils::ResetStatement(saveLocalStatements_.putStatement, false, errCode); + SQLiteUtils::ResetStatement(saveLocalStatements_.insertStatement, false, errCode); + SQLiteUtils::ResetStatement(saveLocalStatements_.updateStatement, false, errCode); SQLiteUtils::ResetStatement(saveLocalStatements_.queryStatement, false, errCode); } else if (type == SingleVerDataType::SYNC_TYPE) { - SQLiteUtils::ResetStatement(saveSyncStatements_.putStatement, false, errCode); + SQLiteUtils::ResetStatement(saveSyncStatements_.insertStatement, false, errCode); + SQLiteUtils::ResetStatement(saveSyncStatements_.updateStatement, false, errCode); SQLiteUtils::ResetStatement(saveSyncStatements_.queryStatement, false, errCode); } @@ -63,7 +70,8 @@ int SQLiteSingleVerStorageExecutor::ResetForSavingCacheData(SingleVerDataType ty int SQLiteSingleVerStorageExecutor::ResetForMigrateCacheData() { int errCode = E_OK; - SQLiteUtils::ResetStatement(migrateSyncStatements_.putStatement, false, errCode); + SQLiteUtils::ResetStatement(migrateSyncStatements_.insertStatement, false, errCode); + SQLiteUtils::ResetStatement(migrateSyncStatements_.updateStatement, false, errCode); SQLiteUtils::ResetStatement(migrateSyncStatements_.queryStatement, false, errCode); return CheckCorruptedStatus(errCode); @@ -170,7 +178,7 @@ int SQLiteSingleVerStorageExecutor::MigrateRmDevData(const DataItem &dataItem) c errCode = SQLiteUtils::BindBlobToStatement(statement, 1, dataItem.value, true); if (errCode != E_OK) { - LOGE("[sinverExecutor][MiRmData] Bind dev for sync data failed:%d", errCode); + LOGE("[singerVerExecutor][MiRmData] Bind dev for sync data failed:%d", errCode); goto END; } @@ -183,54 +191,6 @@ END: return CheckCorruptedStatus(errCode); } -int SQLiteSingleVerStorageExecutor::PutMigratingDataToMain(const std::vector &dataItems) const -{ - std::string sql; - if (executorState_ == ExecutorState::MAIN_ATTACH_CACHE) { - sql = MIGRATE_PUT_DATA_TO_MAINDB_FROM_MAINHANDLE; - } else if (executorState_ == ExecutorState::CACHE_ATTACH_MAIN) { - sql = MIGRATE_PUT_DATA_TO_MAINDB_FROM_CACHEHANDLE; - } else { - return -E_INVALID_ARGS; - } - - sqlite3_stmt *statement = nullptr; - int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); - if (errCode != E_OK) { - LOGE("GetStatement fail when put migrating-data to main! errCode = [%d]", errCode); - return CheckCorruptedStatus(errCode); - } - - for (const auto &dataItem : dataItems) { - if ((dataItem.flag & DataItem::REMOVE_DEVICE_DATA_FLAG) == DataItem::REMOVE_DEVICE_DATA_FLAG || - (dataItem.flag & DataItem::REMOVE_DEVICE_DATA_NOTIFY_FLAG) == DataItem::REMOVE_DEVICE_DATA_NOTIFY_FLAG) { - errCode = MigrateRmDevData(dataItem); - LOGI("[PutMigratingDataToMain]Execute remove devices data! errCode = [%d]", errCode); - if (errCode != E_OK) { - break; - } - continue; - } - SyncDataDevices devices{ dataItem.origDev, dataItem.dev }; - errCode = BindSavedSyncData(statement, dataItem, dataItem.hashKey, devices); - if (errCode != E_OK) { - goto END; - } - - errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); - if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { - errCode = E_OK; - } else { - LOGD("StepWithRetry fail when put migrating-data to main!"); - break; - } - SQLiteUtils::ResetStatement(statement, false, errCode); - } -END: - SQLiteUtils::ResetStatement(statement, true, errCode); - return CheckCorruptedStatus(errCode); -} - int SQLiteSingleVerStorageExecutor::AttachMainDbAndCacheDb(CipherType type, const CipherPassword &passwd, const std::string &attachDbAbsPath, EngineState engineState) { @@ -300,6 +260,129 @@ END: return CheckCorruptedStatus(errCode); } +int SQLiteSingleVerStorageExecutor::MigrateDataItem(DataItem &dataItem, NotifyMigrateSyncData &syncData) +{ + // Put or delete. Prepare notify data here. + NotifyConflictAndObserverData notify; + notify.committedData = syncData.committedData; + int errCode = PutIntoConflictAndCommitForMigrateCache(dataItem, {dataItem.dev.empty(), dataItem.dev}, notify); + if (errCode != E_OK) { + ResetForMigrateCacheData(); + LOGE("PutIntoConflictAndCommitForMigrateCache failed, errCode = %d", errCode); + return errCode; + } + // after solving conflict, the item should not be saved into mainDB + if (notify.dataStatus.isDefeated) { + LOGD("Data status is defeated:%d", errCode); + return errCode; + } + bool isUpdate = notify.dataStatus.preStatus != DataStatus::NOEXISTED; + sqlite3_stmt *statement = migrateSyncStatements_.GetDataSaveStatement(isUpdate); + if (statement == nullptr) { + LOGE("GetStatement fail when put migrating-data to main! "); + return -E_INVALID_ARGS; + } + + if ((dataItem.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) != 0) { + errCode = EraseSyncData(dataItem.key); + goto END; + } + + errCode = BindSavedSyncData(statement, dataItem, dataItem.hashKey, { dataItem.origDev, dataItem.dev }, isUpdate); + if (errCode != E_OK) { + goto END; + } + + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } else { + LOGD("StepWithRetry fail when put migrating-data to main!"); + } +END: + ResetForMigrateCacheData(); + return errCode; +} + +int SQLiteSingleVerStorageExecutor::CheckDataWithQuery(std::vector &dataItems) +{ + int errCode = E_OK; + sqlite3_stmt *stmt = nullptr; + for (auto &item : dataItems) { + if ((item.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) == 0) { + continue; + } + std::string sql; + DBCommon::VectorToString(item.value, sql); + if (executorState_ == ExecutorState::CACHE_ATTACH_MAIN) { + static const std::string SYNC_DATA_TABLE = "sync_data"; + static const std::string SYNC_DATA_TABLE_MAIN = "maindb.sync_data"; + std::string::size_type startPos = sql.find(SYNC_DATA_TABLE); + if (startPos != std::string::npos) { + sql.replace(startPos, SYNC_DATA_TABLE.length(), SYNC_DATA_TABLE_MAIN); + } + } + errCode = SQLiteUtils::GetStatement(dbHandle_, sql, stmt); + if (errCode != E_OK) { + LOGE("Get Check miss query data statement failed. %d", errCode); + return errCode; + } + + errCode = CheckMissQueryDataItem(stmt, item.dev, item); + if (errCode != E_OK) { + LOGE("Check miss query data item failed. %d", errCode); + break; + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + return CheckCorruptedStatus(errCode); +} + +int SQLiteSingleVerStorageExecutor::MigrateDataItems(std::vector &dataItems, NotifyMigrateSyncData &syncData) +{ + syncData.isRemote = ((dataItems[0].flag & DataItem::LOCAL_FLAG) == 0); + syncData.isRemoveDeviceData = (dataItems[0].flag & DataItem::REMOVE_DEVICE_DATA_FLAG) != 0 || + (dataItems[0].flag & DataItem::REMOVE_DEVICE_DATA_NOTIFY_FLAG) != 0; + + int errCode = CheckDataWithQuery(dataItems); + if (errCode != E_OK) { + LOGE("Check migrate data with query failed! errCode = [%d]", errCode); + goto END; + } + + for (auto &item : dataItems) { + // Remove device data owns one version itself. + // Get entry here. Prepare notify data in storageEngine. + if (syncData.isRemoveDeviceData) { + errCode = GetEntriesForNotifyRemoveDevData(item, syncData.entries); + if (errCode != E_OK) { + LOGE("Failed to get remove devices data"); + return errCode; + } + errCode = MigrateRmDevData(item); + LOGI("[PutMigratingDataToMain]Execute remove devices data! errCode = [%d]", errCode); + if (errCode != E_OK) { + break; + } + continue; + } + + if (item.neglect) { // Do not save this record if it is neglected + continue; + } + + errCode = MigrateDataItem(item, syncData); + if (errCode != E_OK) { + LOGE("Migrate data item to main db failed! errCode = [%d]", errCode); + break; + } + } +END: + ResetForMigrateCacheData(); + return CheckCorruptedStatus(errCode); +} + int SQLiteSingleVerStorageExecutor::MigrateSyncDataByVersion(uint64_t recordVer, NotifyMigrateSyncData &syncData, std::vector &dataItems) { @@ -315,24 +398,15 @@ int SQLiteSingleVerStorageExecutor::MigrateSyncDataByVersion(uint64_t recordVer, goto END; } - // fix dataItem timestap for mingrate + // fix dataItem timestamp for migrate errCode = ProcessTimeStampForSyncDataInCacheDB(dataItems); if (errCode != E_OK) { LOGE("Chang the time stamp for migrate failed! errCode = [%d]", errCode); goto END; } - // observer and conflictdata - errCode = NotifyForMigrateCacheDB(dataItems, syncData); - if (errCode != E_OK) { - LOGE("Notify cacheDb to mainDb failed! errCode = [%d]", errCode); - goto END; - } - - // Put this version into main db - errCode = PutMigratingDataToMain(dataItems); + errCode = MigrateDataItems(dataItems, syncData); if (errCode != E_OK) { - LOGE("Put migrating data to main db failed! errCode = [%d]", errCode); goto END; } @@ -523,26 +597,69 @@ int SQLiteSingleVerStorageExecutor::BindDevSyncDataInCacheMode(sqlite3_stmt *sta return errCode; } -int SQLiteSingleVerStorageExecutor::SaveSyncDataItemInCacheMode(DataItem &dataItem, - const DeviceInfo &deviceInfo, TimeStamp &maxStamp, uint64_t recordVersion) +int SQLiteSingleVerStorageExecutor::GetExpandedCheckSql(QueryObject query, DataItem &dataItem) +{ + int errCode = E_OK; + SqliteQueryHelper helper = query.GetQueryHelper(errCode); + + std::string sql; + std::string expandedSql; + errCode = helper.GetSyncDataCheckSql(sql); + if (errCode != E_OK) { + LOGE("Get sync data check sql failed"); + return errCode; + } + sqlite3_stmt *stmt = nullptr; + errCode = SQLiteUtils::GetStatement(dbHandle_, sql, stmt); + if (errCode != E_OK) { + LOGE("Get statement fail. %d", errCode); + return -E_INVALID_QUERY_FORMAT; + } + + errCode = helper.BindSyncDataCheckStmt(stmt, dataItem.key); + if (errCode != E_OK) { + goto END; + } + + errCode = SQLiteUtils::ExpandedSql(stmt, expandedSql); + if (errCode != E_OK) { + LOGE("Get expand sql fail. %d", errCode); + } + DBCommon::StringToVector(expandedSql, dataItem.value); +END: + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} + +int SQLiteSingleVerStorageExecutor::SaveSyncDataItemInCacheMode(DataItem &dataItem, const DeviceInfo &deviceInfo, + TimeStamp &maxStamp, uint64_t recordVersion, const QueryObject &query) { Key hashKey; + int errCode = E_OK; if ((dataItem.flag & DataItem::DELETE_FLAG) == DataItem::DELETE_FLAG) { hashKey = dataItem.key; } else { - int errCode = DBCommon::CalcValueHash(dataItem.key, hashKey); + errCode = DBCommon::CalcValueHash(dataItem.key, hashKey); if (errCode != E_OK) { return errCode; } } + if ((dataItem.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) != 0) { + errCode = GetExpandedCheckSql(query, dataItem); // record check sql in value for miss query data + if (errCode != E_OK) { + LOGE("Get sync data check sql failed. %d", errCode); + return errCode; + } + } + std::string origDev = dataItem.origDev; if (((dataItem.flag & DataItem::LOCAL_FLAG) != 0) && dataItem.origDev.empty()) { origDev.clear(); } dataItem.dev = deviceInfo.deviceName; dataItem.origDev = origDev; - int errCode = SaveSyncDataToCacheDatabase(dataItem, hashKey, recordVersion); + errCode = SaveSyncDataToCacheDatabase(dataItem, hashKey, recordVersion); if (errCode == E_OK) { maxStamp = std::max(dataItem.timeStamp, maxStamp); } else { @@ -554,7 +671,7 @@ int SQLiteSingleVerStorageExecutor::SaveSyncDataItemInCacheMode(DataItem &dataIt int SQLiteSingleVerStorageExecutor::SaveSyncDataToCacheDatabase(const DataItem &dataItem, const Key &hashKey, uint64_t recordVersion) const { - auto statement = saveSyncStatements_.putStatement; + auto statement = saveSyncStatements_.GetDataSaveStatement(false); if (statement == nullptr) { return -E_INVALID_ARGS; } @@ -656,7 +773,7 @@ int SQLiteSingleVerStorageExecutor::PutIntoConflictAndCommitForMigrateCache(Data } PutConflictData(dataItem, notify.getData, deviceInfo, notify.dataStatus, notify.committedData); - if (notify.dataStatus.isDefeated == true) { + if (notify.dataStatus.isDefeated) { LOGD("Data status is defeated:%d", errCode); return ResetForMigrateCacheData(); } @@ -729,7 +846,7 @@ int SQLiteSingleVerStorageExecutor::InitMigrateTimeStampOffset() int SQLiteSingleVerStorageExecutor::ProcessTimeStampForSyncDataInCacheDB(std::vector &dataItems) { - if (dataItems.size() == 0) { + if (dataItems.empty()) { LOGE("[SQLiteSingleVerStorageExecutor::ProcessTimeStampForCacheDB] Invalid parameter : dataItems."); return -E_INVALID_ARGS; } @@ -770,50 +887,6 @@ int SQLiteSingleVerStorageExecutor::GetEntriesForNotifyRemoveDevData(const DataI return GetAllSyncedEntries(dev, entries); } -int SQLiteSingleVerStorageExecutor::NotifyForMigrateCacheDB(std::vector &dataItems, - NotifyMigrateSyncData &syncData) -{ - auto &isRemote = syncData.isRemote; - auto &isRemoveDeviceData = syncData.isRemoveDeviceData; - auto &committedData = syncData.committedData; - - isRemote = ((dataItems[0].flag & DataItem::LOCAL_FLAG) == 0); - isRemoveDeviceData = (dataItems[0].flag & DataItem::REMOVE_DEVICE_DATA_FLAG) != 0 || - (dataItems[0].flag & DataItem::REMOVE_DEVICE_DATA_NOTIFY_FLAG) != 0; - - for (auto iter = dataItems.begin(); iter != dataItems.end();) { - auto &item = *iter; - // Remove device data owns one version itself. - // Get entry here. Prepare notify data in storageEngine. - if (isRemoveDeviceData) { - return GetEntriesForNotifyRemoveDevData(item, syncData.entries); - } - - // Put or delete. Prepare notify data here. - DeviceInfo deviceInfo = {item.dev.empty(), item.dev}; - NotifyConflictAndObserverData notify; - notify.committedData = committedData; - int errCode = PutIntoConflictAndCommitForMigrateCache(item, deviceInfo, notify); - if (errCode != E_OK) { - ResetForMigrateCacheData(); - LOGE("PutIntoConflictAndCommitForMigrateCache failed, errCode = %d", errCode); - return errCode; - } - errCode = ResetForMigrateCacheData(); - if (errCode != E_OK) { - LOGE("ResetForMigrateCacheData failed, errCode = %d", errCode); - return errCode; - } - // after solving conflict, the item should not be saved into mainDB - if (notify.dataStatus.isDefeated == true) { - iter = dataItems.erase(iter); - continue; - } - iter++; - } - return E_OK; -} - int SQLiteSingleVerStorageExecutor::InitMigrateData() { // Sync_data already in migrating. Need not to init data. @@ -822,18 +895,21 @@ int SQLiteSingleVerStorageExecutor::InitMigrateData() } ClearMigrateData(); std::string querySQL; - std::string putSQL; + std::string insertSQL; + std::string updateSQL; if (executorState_ == ExecutorState::MAIN_ATTACH_CACHE) { querySQL = SELECT_SYNC_HASH_SQL; - putSQL = MIGRATE_PUT_DATA_TO_MAINDB_FROM_MAINHANDLE; + insertSQL = MIGRATE_INSERT_DATA_TO_MAINDB_FROM_MAINHANDLE; + updateSQL = MIGRATE_UPDATE_DATA_TO_MAINDB_FROM_MAINHANDLE; } else if (executorState_ == ExecutorState::CACHE_ATTACH_MAIN) { querySQL = SELECT_MAIN_SYNC_HASH_SQL_FROM_CACHEHANDLE; - putSQL = MIGRATE_PUT_DATA_TO_MAINDB_FROM_CACHEHANDLE; + insertSQL = MIGRATE_INSERT_DATA_TO_MAINDB_FROM_CACHEHANDLE; + updateSQL = MIGRATE_UPDATE_DATA_TO_MAINDB_FROM_CACHEHANDLE; } else { LOGE("[InitMigrateData] executor in an error state[%d]!", executorState_); return -E_INVALID_DB; } - int errCode = PrepareForSavingData(querySQL, putSQL, migrateSyncStatements_); + int errCode = PrepareForSavingData(querySQL, insertSQL, updateSQL, migrateSyncStatements_); if (errCode != E_OK) { LOGE("Prepare migrateSyncStatements_ fail, errCode = %d", errCode); return errCode; @@ -849,15 +925,9 @@ void SQLiteSingleVerStorageExecutor::ClearMigrateData() maxTimeStampInMainDB_ = 0; // Reset statement. - int errCode = E_OK; - SQLiteUtils::ResetStatement(migrateSyncStatements_.putStatement, true, errCode); + int errCode = migrateSyncStatements_.ResetStatement(); if (errCode != E_OK) { - LOGE("Reset migrateSyncStatements_.putStatement fail, errCode = %d", errCode); - } - errCode = E_OK; - SQLiteUtils::ResetStatement(migrateSyncStatements_.queryStatement, true, errCode); - if (errCode != E_OK) { - LOGE("Reset migrateSyncStatements_.queryStatement fail, errCode = %d", errCode); + LOGE("Reset migrateSync Statements failed, errCode = %d", errCode); } isSyncMigrating_ = false; @@ -871,4 +941,54 @@ int SQLiteSingleVerStorageExecutor::GetMaxTimeStampDuringMigrating(TimeStamp &ma maxTimeStamp = maxTimeStampInMainDB_; return E_OK; } + +int SQLiteSingleVerStorageExecutor::DeleteMetaData(const std::vector &keys) +{ + sqlite3_stmt *statement = nullptr; + const std::string sql = attachMetaMode_ ? REMOVE_ATTACH_META_VALUE_SQL : REMOVE_META_VALUE_SQL; + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); + if (errCode != E_OK) { + return errCode; + } + + for (const auto &key : keys) { + errCode = SQLiteUtils::BindBlobToStatement(statement, 1, key, false); // first arg. + if (errCode != E_OK) { + break; + } + + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + break; + } + errCode = E_OK; + SQLiteUtils::ResetStatement(statement, false, errCode); + } + + SQLiteUtils::ResetStatement(statement, true, errCode); + return CheckCorruptedStatus(errCode); +} + +int SQLiteSingleVerStorageExecutor::DeleteMetaDataByPrefixKey(const Key &keyPrefix) +{ + sqlite3_stmt *statement = nullptr; + const std::string sql = attachMetaMode_ ? + REMOVE_ATTACH_META_VALUE_BY_KEY_PREFIX_SQL : REMOVE_META_VALUE_BY_KEY_PREFIX_SQL; + + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); + if (errCode != E_OK) { + return errCode; + } + + errCode = SQLiteUtils::BindPrefixKey(statement, 1, keyPrefix); // 1 is first arg. + if (errCode == E_OK) { + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } + } + + SQLiteUtils::ResetStatement(statement, true, errCode); + return CheckCorruptedStatus(errCode); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_sql.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_sql.h index ff7b927a0b8ef0666bff4f83ce99cc94b6159218..a0a071e968b0415be24710dec482ae3a20430c4d 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_sql.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_sql.h @@ -30,6 +30,12 @@ namespace DistributedDB { const std::string INSERT_CACHE_LOCAL_SQL = "INSERT OR REPLACE INTO local_data VALUES(?,?,?,?,?);"; + const std::string UPDATE_LOCAL_SQL_FROM_CACHEHANDLE = + "UPDATE maindb.local_data SET key=?,value=?,timestamp=? where hash_key=?"; + + const std::string UPDATE_CACHE_LOCAL_SQL = + "UPDATE local_data SET key=?,value=?,timestamp=? where hash_key=?"; + const std::string INSERT_META_SQL = "INSERT OR REPLACE INTO meta_data VALUES(?,?);"; @@ -37,13 +43,23 @@ namespace DistributedDB { "INSERT OR REPLACE INTO meta.meta_data VALUES(?,?);"; const std::string INSERT_SYNC_SQL = - "INSERT OR REPLACE INTO sync_data VALUES(?,?,?,?,?,?,?,?);"; + "INSERT INTO sync_data VALUES(?,?,?,?,?,?,?,?);"; + + const std::string UPDATE_SYNC_SQL = + "UPDATE sync_data SET key=?,value=?,timestamp=?,flag=?,device=?,ori_device=?,w_timestamp=? WHERE hash_key=?;"; const std::string INSERT_CACHE_SYNC_SQL = "INSERT OR REPLACE INTO sync_data VALUES(?,?,?,?,?,?,?,?,?);"; const std::string INSERT_CACHE_SYNC_SQL_FROM_MAINHANDLE = "INSERT OR REPLACE INTO cache.sync_data VALUES(?,?,?,?,?,?,?,?,?);"; + const std::string UPDATE_CACHE_SYNC_SQL = + "UPDATE sync_data SET key=?,value=?,timestamp=?,flag=?,device=?,ori_device=?,w_timestamp=? WHERE hash_key=?;"; + + const std::string UPDATE_CACHE_SYNC_SQL_FROM_MAINHANDLE = + "UPDATE cache.sync_data SET key=?,value=?,timestamp=?,flag=?,device=?,ori_device=?,w_timestamp=? " + "WHERE hash_key=?;"; + const std::string DELETE_LOCAL_SQL = "DELETE FROM local_data WHERE key=?;"; const std::string DELETE_LOCAL_SQL_FROM_CACHEHANDLE = @@ -103,6 +119,12 @@ namespace DistributedDB { const std::string SELECT_SYNC_ENTRIES_SQL = "SELECT * FROM sync_data WHERE timestamp >= ? AND timestamp < ? AND (flag&0x02=0x02) ORDER BY timestamp ASC;"; + const std::string SELECT_SYNC_DELETED_ENTRIES_SQL = + "SELECT * FROM sync_data WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x03) ORDER BY timestamp ASC;"; + + const std::string SELECT_SYNC_MODIFY_SQL = + "SELECT * FROM sync_data WHERE timestamp >= ? AND timestamp < ? AND (flag&0x03=0x02) ORDER BY timestamp ASC;"; + const std::string SELECT_SYNC_PREFIX_SQL = "SELECT key, value FROM sync_data WHERE key>=? AND key<=? AND (flag&0x01=0) ORDER BY key ASC;"; @@ -148,10 +170,16 @@ namespace DistributedDB { const std::string GET_MAX_VER_CACHEDATA_FROM_MAINHANDLE = "select version from cache.sync_data order by version DESC limit 1;"; - const std::string MIGRATE_PUT_DATA_TO_MAINDB_FROM_CACHEHANDLE = - "INSERT OR REPLACE INTO maindb.sync_data VALUES(?,?,?,?,?,?,?,?);"; - const std::string MIGRATE_PUT_DATA_TO_MAINDB_FROM_MAINHANDLE = - "INSERT OR REPLACE INTO sync_data VALUES(?,?,?,?,?,?,?,?);"; + const std::string MIGRATE_INSERT_DATA_TO_MAINDB_FROM_CACHEHANDLE = + "INSERT INTO maindb.sync_data VALUES(?,?,?,?,?,?,?,?);"; + const std::string MIGRATE_UPDATE_DATA_TO_MAINDB_FROM_CACHEHANDLE = + "UPDATE maindb.sync_data SET key=?,value=?,timestamp=?,flag=?,device=?,ori_device=?,w_timestamp=? " + "WHERE hash_key=?;"; + + const std::string MIGRATE_INSERT_DATA_TO_MAINDB_FROM_MAINHANDLE = + "INSERT INTO sync_data VALUES(?,?,?,?,?,?,?,?);"; + const std::string MIGRATE_UPDATE_DATA_TO_MAINDB_FROM_MAINHANDLE = + "UPDATE sync_data SET key=?,value=?,timestamp=?,flag=?,device=?,ori_device=?,w_timestamp=? WHERE hash_key=?;"; const std::string MIGRATE_DEL_DATA_BY_VERSION_FROM_CACHEHANDLE = "DELETE FROM sync_data WHERE version=?;"; @@ -160,6 +188,26 @@ namespace DistributedDB { const std::string SELECT_MAIN_SYNC_HASH_SQL_FROM_CACHEHANDLE = "SELECT * FROM maindb.sync_data WHERE hash_key=?;"; + const std::string REMOVE_META_VALUE_SQL = + "DELETE FROM meta_data WHERE key=?;"; + const std::string REMOVE_ATTACH_META_VALUE_SQL = + "DELETE FROM meta.meta_data WHERE key=?;"; + + const std::string CHECK_DB_INTEGRITY_SQL = "PRAGMA integrity_check;"; + + const std::string REMOVE_META_VALUE_BY_KEY_PREFIX_SQL = + "DELETE FROM meta_data WHERE key>=? AND key<=?;"; + const std::string REMOVE_ATTACH_META_VALUE_BY_KEY_PREFIX_SQL = + "DELETE FROM meta.meta_data WHERE key>=? AND key<=?;"; + + const std::string DELETE_SYNC_DATA_WITH_HASHKEY = "DELETE FROM sync_data where hash_key = ?;"; + + const std::string DELETE_SYNC_DATA_WITH_HASHKEY_FROM_CACHEHANDLE = + "DELETE FROM maindb.sync_data where hash_key = ?;"; + + const std::string GET_SYNC_DATA_TIRGGER_SQL = + "SELECT name FROM SQLITE_MASTER WHERE TYPE = 'trigger' AND TBL_NAME = 'sync_data' AND name like ?;"; + const int BIND_KV_KEY_INDEX = 1; const int BIND_KV_VAL_INDEX = 2; const int BIND_LOCAL_TIMESTAMP_INDEX = 3; @@ -179,6 +227,9 @@ namespace DistributedDB { const int BIND_SYNC_HASH_KEY_INDEX = 7; const int BIND_SYNC_W_TIME_INDEX = 8; + const int BIND_SYNC_UPDATE_W_TIME_INDEX = 7; + const int BIND_SYNC_UPDATE_HASH_KEY_INDEX = 8; + // cacheDB const int BIND_CACHE_LOCAL_KEY_INDEX = 1; const int BIND_CACHE_LOCAL_VAL_INDEX = 2; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_subscribe.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_subscribe.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5f79dbecd1c76f04b8c192c7020fd3e9c1e85394 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_single_ver_storage_executor_subscribe.cpp @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sqlite_single_ver_storage_executor.h" + +#include "log_print.h" +#include "db_common.h" +#include "db_errno.h" +#include "sqlite_single_ver_storage_executor_sql.h" + +namespace DistributedDB { +using namespace TriggerMode; + +int SQLiteSingleVerStorageExecutor::CheckQueryObjectLegal(QueryObject &queryObj) const +{ + int errCode = E_OK; + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); + if (errCode != E_OK) { + LOGE("Get query helper failed [%d]!", errCode); + return errCode; + } + + sqlite3_stmt *statement = nullptr; + errCode = helper.GetQuerySyncStatement(dbHandle_, 0, INT64_MAX, statement); // (0, INT64_MAX):max range + int ret = E_OK; + SQLiteUtils::ResetStatement(statement, true, ret); + if (ret != E_OK) { + LOGW("Failed to reset statement. error:%d", ret); + } + return CheckCorruptedStatus(errCode); +} + +int SQLiteSingleVerStorageExecutor::CheckMissQueryDataItem(sqlite3_stmt *stmt, const std::string &deviceName, + DataItem &item) +{ + int errCode = SQLiteUtils::StepWithRetry(stmt, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + // the value with same hashKey in DB matched the query + std::vector dev; + errCode = SQLiteUtils::GetColumnBlobValue(stmt, SYNC_RES_DEVICE_INDEX, dev); + if (errCode != E_OK) { + LOGE("Get data device info failed. %d", errCode); + return errCode; + } + auto timeStamp = static_cast(sqlite3_column_int64(stmt, SYNC_RES_TIME_INDEX)); + std::string device = std::string(dev.begin(), dev.end()); + // this data item should be neglected when it's out of date of it's from same device + // otherwise, it should be erased after resolved the conflict + item.neglect = (timeStamp > item.timeStamp) || + (timeStamp == item.timeStamp && device == DBCommon::TransferHashString(deviceName)); + return E_OK; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + // the value with same hashKey in DB does not match the query, this data item should be neglected. + item.neglect = true; + return E_OK; + } + LOGE("Check sync data failed %d", errCode); + return errCode; +} + +// check the data with REMOTE_DEVICE_DATA_MISS_QUERY flag need neglect or not +int SQLiteSingleVerStorageExecutor::CheckMissQueryDataItems(sqlite3_stmt *&stmt, const SqliteQueryHelper &helper, + const DeviceInfo &deviceInfo, std::vector &dataItems) +{ + int errCode = E_OK; + for (auto &item : dataItems) { + if ((item.flag & DataItem::REMOTE_DEVICE_DATA_MISS_QUERY) != 0 && !item.key.empty()) { + errCode = helper.BindSyncDataCheckStmt(stmt, item.key); + if (errCode != E_OK) { + LOGE("Bind sync data check statement failed %d", errCode); + break; + } + errCode = CheckMissQueryDataItem(stmt, deviceInfo.deviceName, item); + if (errCode != E_OK) { + LOGE("Check miss query data item failed. %d", errCode); + return errCode; + } + SQLiteUtils::ResetStatement(stmt, false, errCode); + } + } + return errCode; +} + +int SQLiteSingleVerStorageExecutor::CheckDataWithQuery(QueryObject query, std::vector &dataItems, + const DeviceInfo &deviceInfo) +{ + int errCode = E_OK; + if (query.Empty()) { + LOGD("Query is empty, skip check."); + return E_OK; + } + SqliteQueryHelper helper = query.GetQueryHelper(errCode); + if (errCode != E_OK) { + LOGE("Get query helper failed [%d]!", errCode); + return errCode; + } + std::string sql; + errCode = helper.GetSyncDataCheckSql(sql); + if (errCode != E_OK) { + LOGE("Get sync data check sql failed"); + return errCode; + } + sqlite3_stmt *stmt = nullptr; + errCode = SQLiteUtils::GetStatement(dbHandle_, sql, stmt); + if (errCode != E_OK) { + LOGE("Get statement fail. %d", errCode); + return -E_INVALID_QUERY_FORMAT; + } + errCode = CheckMissQueryDataItems(stmt, helper, deviceInfo, dataItems); + if (errCode != E_OK) { + LOGE("check data with query failed. %d", errCode); + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + return CheckCorruptedStatus(errCode); +} + +namespace { +std::string FormatSubscribeTriggerSql(const std::string &subscribeId, const std::string &subscribeCondition, + TriggerModeEnum mode) +{ + std::string triggerModeString = GetTriggerModeString(mode); + std::string accessString = ((mode == TriggerModeEnum::DELETE) ? + DBConstant::TRIGGER_REFERENCES_OLD : DBConstant::TRIGGER_REFERENCES_NEW); + std::string keyStr = DBConstant::SUBSCRIBE_QUERY_PREFIX + DBCommon::TransferHashString(subscribeId); + Key key {keyStr.begin(), keyStr.end()}; + std::string hexKeyStr = DBCommon::VectorToHexString(key); + std::string triggerName = DBConstant::SUBSCRIBE_QUERY_PREFIX + subscribeId + "_ON_" + triggerModeString; + return "CREATE TRIGGER IF NOT EXISTS " + triggerName + " AFTER " + triggerModeString + " \n" + "ON sync_data\n" + "WHEN " + subscribeCondition + "\n" + "BEGIN\n" + " SELECT " + DBConstant::UPDATE_META_FUNC + "(x'" + hexKeyStr + "', NEW.TIMESTAMP);\n" + "END;"; +} +} + +int SQLiteSingleVerStorageExecutor::AddSubscribeTrigger(QueryObject &query, const std::string &subscribeId) +{ + if (executorState_ == ExecutorState::CACHEDB || executorState_ == ExecutorState::CACHE_ATTACH_MAIN) { + LOGE("Not support add subscribe in cache db."); + return -E_NOT_SUPPORT; + } + int errCode = E_OK; + SqliteQueryHelper helper = query.GetQueryHelper(errCode); + if (errCode != E_OK) { + LOGE("Get query helper failed. %d", errCode); + return errCode; + } + // check if sqlite function is registered or not + sqlite3_stmt *stmt = nullptr; + errCode = SQLiteUtils::GetStatement(dbHandle_, "SELECT " + DBConstant::UPDATE_META_FUNC + "('K', 0);", stmt); + if (errCode != E_OK) { + LOGE("sqlite function %s has not been created.", DBConstant::UPDATE_META_FUNC.c_str()); + return -E_NOT_SUPPORT; + } + SQLiteUtils::ResetStatement(stmt, true, errCode); + + // Delete data API is actually an update operation, there is no need for DELETE trigger + for (auto mode : {TriggerModeEnum::INSERT, TriggerModeEnum::UPDATE}) { + std::string subscribeCondition; + errCode = helper.GetSubscribeSql(subscribeId, mode, subscribeCondition); + if (errCode != E_OK) { + LOGE("Get subscribe trigger create sql failed. mode: %d, errCode: %d", mode, errCode); + return errCode; + } + std::string sql = FormatSubscribeTriggerSql(subscribeId, subscribeCondition, mode); + errCode = SQLiteUtils::ExecuteRawSQL(dbHandle_, sql); + if (errCode != E_OK) { + LOGE("Add subscribe trigger failed. mode: %d, errCode: %d", mode, errCode); + return errCode; + } + } + return E_OK; +} + +int SQLiteSingleVerStorageExecutor::RemoveSubscribeTrigger(const std::vector &subscribeIds) +{ + int errCode = E_OK; + for (const auto &id : subscribeIds) { + for (auto mode : {TriggerModeEnum::INSERT, TriggerModeEnum::UPDATE}) { + const std::string trigger = DBConstant::SUBSCRIBE_QUERY_PREFIX + id + "_ON_" + GetTriggerModeString(mode); + errCode = SQLiteUtils::DropTriggerByName(dbHandle_, trigger); + if (errCode != E_OK) { + LOGE("remove subscribe trigger failed. %d", errCode); + break; + } + } + if (errCode != E_OK) { + LOGE("remove subscribe trigger for id %s failed. %d", id.c_str(), errCode); + break; + } + } + return errCode; +} + +int SQLiteSingleVerStorageExecutor::RemoveTrigger(const std::vector &triggers) +{ + int errCode = E_OK; + for (const auto &trigger : triggers) { + errCode = SQLiteUtils::DropTriggerByName(dbHandle_, trigger); + if (errCode != E_OK) { + LOGE("remove trigger failed. %d", errCode); + break; + } + } + return errCode; +} + +int SQLiteSingleVerStorageExecutor::RemoveSubscribeTriggerWaterMark(const std::vector &subscribeIds) +{ + sqlite3_stmt *statement = nullptr; + const std::string sql = attachMetaMode_ ? REMOVE_ATTACH_META_VALUE_SQL : REMOVE_META_VALUE_SQL; + int errCode = SQLiteUtils::GetStatement(dbHandle_, sql, statement); + if (errCode != E_OK) { + LOGE("Get remove trigger water mark statement failed. %d", errCode); + return errCode; + } + for (const auto &id : subscribeIds) { + errCode = SQLiteUtils::BindTextToStatement(statement, 1, DBConstant::SUBSCRIBE_QUERY_PREFIX + id); + if (errCode != E_OK) { + LOGE("Bind mark key to statement failed. %d", errCode); + break; + } + errCode = SQLiteUtils::StepWithRetry(statement, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + } else { + LOGE("Remove trigger water mark failed. %d", errCode); + break; + } + SQLiteUtils::ResetStatement(statement, false, errCode); + } + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +int SQLiteSingleVerStorageExecutor::GetTriggers(const std::string &namePreFix, std::vector &triggerNames) +{ + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(dbHandle_, GET_SYNC_DATA_TIRGGER_SQL, stmt); + if (errCode != E_OK) { + LOGE("Get trigger query statement failed. %d", errCode); + return errCode; + } + + errCode = SQLiteUtils::BindTextToStatement(stmt, 1, namePreFix + "%"); + if (errCode != E_OK) { + SQLiteUtils::ResetStatement(stmt, true, errCode); + LOGE("Bind trigger name prefix to statement failed. %d", errCode); + return errCode; + } + + do { + errCode = SQLiteUtils::StepWithRetry(stmt, isMemDb_); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + break; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + std::string name; + SQLiteUtils::GetColumnTextValue(stmt, 0, name); + triggerNames.emplace_back(name); + } else { + LOGE("Get trigger by name prefix failed. %d", errCode); + break; + } + } while (true); + + SQLiteUtils::ResetStatement(stmt, true, errCode); + return errCode; +} +} // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp index b3ce9d9ff9a4f4ed8856d4674c705d5783aa61fe..009400728092ddbd0dadeeefd75b44166e90db0a 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_storage_engine.cpp @@ -102,7 +102,7 @@ int SQLiteStorageEngine::ReInit() bool SQLiteStorageEngine::IsNeedTobeReleased() const { EngineState engineState = GetEngineState(); - return ((engineState == MAINDB) || (engineState == INVALID)); + return ((engineState == EngineState::MAINDB) || (engineState == EngineState::INVALID)); } const std::string &SQLiteStorageEngine::GetIdentifier() const @@ -168,7 +168,7 @@ int SQLiteStorageEngine::CheckEngineOption(const KvDBProperties &kvDBProp) const securityOpt.securityLabel = kvDBProp.GetSecLabel(); securityOpt.securityFlag = kvDBProp.GetSecFlag(); } - std::string schemaStr = kvDBProp.GetSchema().ToSchemaString(); + int conflictReslovePolicy = kvDBProp.GetIntProp(KvDBProperties::CONFLICT_RESOLVE_POLICY, DEFAULT_LAST_WIN); bool createDirByStoreIdOnly = kvDBProp.GetBoolProp(KvDBProperties::CREATE_DIR_BY_STORE_ID_ONLY, false); diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp index e6a5aaf49dad467c4b41964f599bbc83ef3b5861..999d6f49fb57a40bd54ef70a8311f8c13cb75dbe 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.cpp @@ -61,7 +61,10 @@ namespace { const std::string BACK_KDF_ITER_CONFIG_SQL = "PRAGMA backup.codec_kdf_iter=5000;"; const std::string META_CIPHER_CONFIG_SQL = "PRAGMA meta.codec_cipher="; const std::string META_KDF_ITER_CONFIG_SQL = "PRAGMA meta.codec_kdf_iter=5000;"; + const std::string DETACH_BACKUP_SQL = "DETACH 'backup'"; + const std::string UPDATE_META_SQL = "INSERT OR REPLACE INTO meta_data VALUES (?, ?);"; + bool g_configLog = false; std::mutex g_logMutex; @@ -84,6 +87,21 @@ namespace { } } +namespace TriggerMode { +const std::map TRIGGER_MODE_MAP = { + {TriggerModeEnum::NONE, ""}, + {TriggerModeEnum::INSERT, "INSERT"}, + {TriggerModeEnum::UPDATE, "UPDATE"}, + {TriggerModeEnum::DELETE, "DELETE"}, +}; + +std::string GetTriggerModeString(TriggerModeEnum mode) +{ + auto it = TRIGGER_MODE_MAP.find(mode); + return (it == TRIGGER_MODE_MAP.end()) ? "" : it->second; +} +} + int SQLiteUtils::CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp) { uint64_t flag = SQLITE_OPEN_URI | SQLITE_OPEN_READWRITE; @@ -193,8 +211,8 @@ int SQLiteUtils::BindTextToStatement(sqlite3_stmt *statement, int index, const s // Check empty value. if (str.empty()) { - LOGI("[SQLiteUtil][Bind Text] Invalid value"); - return -E_INVALID_ARGS; + sqlite3_bind_null(statement, index); + return E_OK; } int errCode = sqlite3_bind_text(statement, index, str.c_str(), str.length(), SQLITE_TRANSIENT); @@ -344,7 +362,6 @@ int SQLiteUtils::BindPrefixKey(sqlite3_stmt *statement, int index, const Key &ke LOGE("Bind the prefix second error:%d", errCode); return SQLiteUtils::MapSQLiteErrno(errCode); } - return E_OK; } @@ -385,12 +402,17 @@ int SQLiteUtils::ExecuteRawSQL(sqlite3 *db, const std::string &sql) return SQLiteUtils::MapSQLiteErrno(errCode); } -int SQLiteUtils::SetKey(sqlite3 *db, CipherType type, const CipherPassword &passwd) +int SQLiteUtils::SetKey(sqlite3 *db, CipherType type, const CipherPassword &passwd, const bool &isMemDb) { if (db == nullptr) { return -E_INVALID_DB; } + // in memory mode no need cipher + if (isMemDb) { + return E_OK; + } + if (passwd.GetSize() != 0) { #ifndef OMIT_ENCRYPT int errCode = sqlite3_key(db, static_cast(passwd.GetData()), static_cast(passwd.GetSize())); @@ -416,6 +438,9 @@ int SQLiteUtils::SetKey(sqlite3 *db, CipherType type, const CipherPassword &pass if (errno == EKEYREVOKED) { return -E_EKEYREVOKED; } + if (errCode == -E_BUSY) { + return errCode; + } return -E_INVALID_PASSWD_OR_CORRUPTED_DB; } return E_OK; @@ -443,7 +468,17 @@ int SQLiteUtils::GetColumnBlobValue(sqlite3_stmt *statement, int index, std::vec return E_OK; } -int SQLiteUtils::AttachNewDatabase(sqlite3 *db, CipherType type, const CipherPassword &passwd, +int SQLiteUtils::GetColumnTextValue(sqlite3_stmt *statement, int index, std::string &value) +{ + if (statement == nullptr) { + return -E_INVALID_ARGS; + } + const unsigned char *val = sqlite3_column_text(statement, index); + value = (val != nullptr) ? std::string(reinterpret_cast(val)) : std::string(); + return E_OK; +} + +int SQLiteUtils::AttachNewDatabase(sqlite3 *db, CipherType type, const CipherPassword &password, const std::string &attachDbAbsPath, const std::string &attachAsName) { // example: "ATTACH '../new.db' AS backup KEY XXXX;" @@ -462,8 +497,8 @@ int SQLiteUtils::AttachNewDatabase(sqlite3 *db, CipherType type, const CipherPas goto END; } // Passwords do not allow vector operations, so we can not use function BindBlobToStatement here. - errCode = sqlite3_bind_blob(statement, 2, static_cast(passwd.GetData()), - passwd.GetSize(), SQLITE_TRANSIENT); // 2nd para is password. + errCode = sqlite3_bind_blob(statement, 2, static_cast(password.GetData()), + password.GetSize(), SQLITE_TRANSIENT); // 2nd para is password. if (errCode != SQLITE_OK) { LOGE("Bind the attached key failed:%d", errCode); errCode = SQLiteUtils::MapSQLiteErrno(errCode); @@ -500,6 +535,258 @@ int SQLiteUtils::CreateMetaDatabase(const std::string &metaDbPath) return errCode; } +int SQLiteUtils::CheckIntegrity(sqlite3 *db, const std::string &sql) +{ + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + LOGE("Prepare the integrity check statement error:%d", errCode); + return errCode; + } + int resultCnt = 0; + bool checkResultOK = false; + do { + errCode = SQLiteUtils::StepWithRetry(statement); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + break; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + auto result = reinterpret_cast(sqlite3_column_text(statement, 0)); + if (result == nullptr) { + continue; + } + resultCnt = (resultCnt > 1) ? resultCnt : (resultCnt + 1); + if (strcmp(result, "ok") == 0) { + checkResultOK = true; + } + } else { + checkResultOK = false; + LOGW("Step for the integrity check failed:%d", errCode); + break; + } + } while (true); + if (resultCnt == 1 && checkResultOK) { + errCode = E_OK; + } else { + errCode = -E_INVALID_PASSWD_OR_CORRUPTED_DB; + } + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} +#ifdef RELATIONAL_STORE +namespace { // anonymous namespace for schema analysis +int AnalysisSchemaSqlAndTrigger(sqlite3 *db, const std::string &tableName, TableInfo &table) +{ + std::string sql = "select type, name, tbl_name, rootpage, sql from sqlite_master where tbl_name = ?"; + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Prepare the analysis schema sql and trigger statement error:%d", errCode); + return errCode; + } + errCode = SQLiteUtils::BindTextToStatement(statement, 1, tableName); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Bind table name failed:%d", errCode); + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; + } + + errCode = -E_NOT_FOUND; + std::vector triggerList; + do { + int err = SQLiteUtils::StepWithRetry(statement); + if (err == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + break; + } else if (err == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + errCode = E_OK; + std::string type; + (void) SQLiteUtils::GetColumnTextValue(statement, 0, type); + if (type == "table") { + std::string createTableSql; + (void) SQLiteUtils::GetColumnTextValue(statement, 4, createTableSql); + table.SetCreateTableSql(createTableSql); + } else if (type == "trigger") { + std::string triggerName; + (void) SQLiteUtils::GetColumnTextValue(statement, 1, triggerName); + table.AddTrigger(triggerName); + } + } else { + LOGW("[AnalysisSchema] Step for the analysis create table sql and trigger failed:%d", err); + break; + } + } while (true); + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +int GetSchemaIndexList(sqlite3 *db, const std::string &tableName, std::vector &indexList, + std::vector &uniqueList) +{ + std::string sql = "pragma index_list(" + tableName + ")"; + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Prepare the get schema index list staement error:%d", errCode); + return errCode; + } + + do { + errCode = SQLiteUtils::StepWithRetry(statement); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + break; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + std::string indexName; + (void) SQLiteUtils::GetColumnTextValue(statement, 1, indexName); + std::string origin; + (void) SQLiteUtils::GetColumnTextValue(statement, 3, origin); + if (origin == "u") { + uniqueList.push_back(indexName); + } else if (origin == "c") { + indexList.push_back(indexName); + } + } else { + LOGW("[AnalysisSchema] Step for the get schema index list failed:%d", errCode); + break; + } + } while (true); + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +int AnalysisSchemaIndexDefine(sqlite3 *db, const std::string &indexName, CompositeFields &indexDefine) +{ + auto sql = "pragma index_info(" + indexName + ")"; + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Prepare the analysis schema index statement error:%d", errCode); + return errCode; + } + + do { + errCode = SQLiteUtils::StepWithRetry(statement); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + break; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + std::string indexField; + (void) SQLiteUtils::GetColumnTextValue(statement, 2, indexField); + indexDefine.push_back(indexField); + } else { + LOGW("[AnalysisSchema] Step for the analysis schema index failed:%d", errCode); + break; + } + } while (true); + + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} + +int AnalysisSchemaIndexAndUnique(sqlite3 *db, const std::string &tableName, TableInfo &table) +{ + std::vector indexList; + std::vector uniqueList; + int errCode = GetSchemaIndexList(db, tableName, indexList, uniqueList); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] get schema index list failed."); + return errCode; + } + + for (const auto &indexName : indexList) { + CompositeFields indexDefine; + errCode = AnalysisSchemaIndexDefine(db, indexName, indexDefine); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] analysis schema index columns failed."); + return errCode; + } + table.AddIndexDefine(indexName, indexDefine); + } + + for (const auto &uniqueName : uniqueList) { + CompositeFields uniqueDefine; + errCode = AnalysisSchemaIndexDefine(db, uniqueName, uniqueDefine); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] analysis schema unique columns failed."); + return errCode; + } + table.AddUniqueDefine(uniqueDefine); + } + + return E_OK; +} + +int AnalysisSchemaFieldDefine(sqlite3 *db, const std::string &tableName, TableInfo &table) +{ + std::string sql = "pragma table_info(" + tableName + ")"; + sqlite3_stmt *statement = nullptr; + int errCode = SQLiteUtils::GetStatement(db, sql, statement); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Prepare the analysis schema field statement error:%d", errCode); + return errCode; + } + do { + errCode = SQLiteUtils::StepWithRetry(statement); + if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + errCode = E_OK; + break; + } else if (errCode == SQLiteUtils::MapSQLiteErrno(SQLITE_ROW)) { + FieldInfo field; + std::string tmpString; + + field.SetColumnId(sqlite3_column_int(statement, 0)); + + (void) SQLiteUtils::GetColumnTextValue(statement, 1, tmpString); + field.SetFieldName(tmpString); + + (void) SQLiteUtils::GetColumnTextValue(statement, 2, tmpString); + field.SetDataType(tmpString); + + field.SetNotNull(static_cast(sqlite3_column_int64(statement, 3))); + + (void) SQLiteUtils::GetColumnTextValue(statement, 4, tmpString); + if (!tmpString.empty()) { + field.SetDefaultValue(tmpString); + } + + if (sqlite3_column_int64(statement, 5)) { + table.SetPrimaryKey(field.GetFieldName()); + } + table.AddField(field); + } else { + LOGW("[AnalysisSchema] Step for the analysis schema field failed:%d", errCode); + break; + } + } while (true); + + SQLiteUtils::ResetStatement(statement, true, errCode); + return errCode; +} +} // end of anonymous namespace for schema analysis + +int SQLiteUtils::AnalysisSchema(sqlite3 *db, const std::string &tableName, TableInfo &table) +{ + int errCode = AnalysisSchemaSqlAndTrigger(db, tableName, table); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Analysis sql and trigger failed. errCode = [%d]", errCode); + return errCode; + } + + errCode = AnalysisSchemaIndexAndUnique(db, tableName, table); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Analysis index failed."); + return errCode; + } + + errCode = AnalysisSchemaFieldDefine(db, tableName, table); + if (errCode != E_OK) { + LOGE("[AnalysisSchema] Analysis field failed."); + return errCode; + } + + table.SetTableName(tableName); + return E_OK; +} +#endif #ifndef OMIT_ENCRYPT int SQLiteUtils::ExportDatabase(sqlite3 *db, CipherType type, const CipherPassword &passwd, const std::string &newDbName) @@ -570,7 +857,8 @@ int SQLiteUtils::GetVersion(const OpenDbProperties &properties, int &version) LOGE("Open database failed: %d, sys:%d", errCode, errno); goto END; } - errCode = SQLiteUtils::SetKey(dbTemp, properties.cipherType, properties.passwd); + + errCode = SQLiteUtils::SetKey(dbTemp, properties.cipherType, properties.passwd, properties.isMemDb); if (errCode != E_OK) { LOGE("Set key failed: %d", errCode); goto END; @@ -936,10 +1224,157 @@ void SchemaObjectDestructor(SchemaObject *inObject) inObject = nullptr; } } +#ifdef RELATIONAL_STORE +int SQLiteUtils::RegisterCalcHash(sqlite3 *db) +{ + TransactFunc func; + func.xFunc = &CalcHashKey; + return SQLiteUtils::RegisterFunction(db, "calc_hash", 1, nullptr, func); +} + +void SQLiteUtils::GetSysTime(sqlite3_context *ctx, int argc, sqlite3_value **argv) +{ + if (ctx == nullptr || argc != 0) { + LOGE("Parameter does not meet restrictions."); + return; + } + + sqlite3_result_int64(ctx, (sqlite3_int64)TimeHelper::GetSysCurrentTime()); +} + +int SQLiteUtils::RegisterGetSysTime(sqlite3 *db) +{ + TransactFunc func; + func.xFunc = &GetSysTime; + return SQLiteUtils::RegisterFunction(db, "get_sys_time", 0, nullptr, func); +} + +int SQLiteUtils::CreateRelationalMetaTable(sqlite3 *db) +{ + std::string sql = + "CREATE TABLE IF NOT EXISTS distributeddatamgr_aux_metadata(" \ + "key BLOB PRIMARY KEY NOT NULL," \ + "value BLOB);"; + + int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + if (errCode != E_OK) { + LOGE("[SQLite] execute create table sql failed"); + return errCode; + } + return E_OK; +} + +int SQLiteUtils::CreateRelationalLogTable(sqlite3 *db, const std::string &oriTableName) +{ + std::string sql = + "CREATE TABLE IF NOT EXISTS distributeddatamgr_aux_" + oriTableName + "_log(" \ + "data_key INT NOT NULL," \ + "device BLOB," \ + "ori_device BLOB," \ + "timestamp INT NOT NULL," \ + "wtimestamp INT NOT NULL," \ + "flag INT NOT NULL," \ + "hash_key BLOB PRIMARY KEY NOT NULL);"; + + int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + if (errCode != E_OK) { + LOGE("[SQLite] execute create table sql failed"); + return errCode; + } + return E_OK; +} + +template +static std::string string_format(const std::string& format, Args ... args) +{ + auto size = static_cast(1024); // trigger string total len need < 1024 + auto buf = std::make_unique(size); + (void)sprintf_s(buf.get(), size, format.c_str(), args ...); + return std::string(buf.get(), buf.get() + size - 1); // We don't want the '\0' inside +} + +int SQLiteUtils::AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &table) +{ + std::string insertTrigger = + "CREATE TRIGGER IF NOT EXISTS distributeddatamgr_%s_ON_INSERT AFTER INSERT \n" \ + "ON %s\n" \ + "BEGIN\n" \ + "\t INSERT OR REPLACE INTO distributeddatamgr_aux_%s_log \ + (data_key, device, ori_device, timestamp, wtimestamp, flag, hash_key)" \ + "VALUES (new.rowid, '%s', '%s', get_sys_time(), get_sys_time(), 0x02, calc_hash(new.%s));\n" \ + "END;"; + insertTrigger = string_format(insertTrigger, table.GetTableName().c_str(), table.GetTableName().c_str(), + table.GetTableName().c_str(), table.GetDevId().c_str(), + table.GetDevId().c_str(), table.GetPrimaryKey().c_str()); + std::string updateTrigger = + "CREATE TRIGGER IF NOT EXISTS distributeddatamgr_%s_ON_UPDATE AFTER UPDATE \n" \ + "ON %s\n" \ + "BEGIN\n" \ + "\t UPDATE distributeddatamgr_aux_%s_log SET timestamp=get_sys_time(), device='%s' \ + where hash_key=calc_hash(old.%s) and flag&0x10=0;\n" \ + "END;"; + updateTrigger = string_format(updateTrigger, table.GetTableName().c_str(), table.GetTableName().c_str(), + table.GetTableName().c_str(), table.GetDevId().c_str(), table.GetPrimaryKey().c_str()); + std::string deleteTrigger = + "CREATE TRIGGER IF NOT EXISTS distributeddatamgr_%s_ON_DELETE BEFORE DELETE \n" \ + "ON %s\n" \ + "BEGIN\n" \ + "\t UPDATE distributeddatamgr_aux_%s_log set flag=0x03,timestamp=get_sys_time() \ + WHERE hash_key=calc_hash(old.%s);\n" \ + "END;"; + deleteTrigger = string_format(deleteTrigger, table.GetTableName().c_str(), + table.GetTableName().c_str(), table.GetTableName().c_str(), table.GetPrimaryKey().c_str()); + + std::vector sqls = {insertTrigger, updateTrigger, deleteTrigger}; + // add insert trigger + // add update trigger + // add delete trigger + for (const auto &sql : sqls) { + int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + if (errCode != E_OK) { + LOGE("[SQLite] execute create log trigger sql failed"); + return errCode; + } + } + return E_OK; +} + +int SQLiteUtils::CreateSameStuTable(sqlite3 *db, const std::string &oriTableName, const std::string &newTableName, + bool isCopyData) +{ + std::string sql = "create table IF NOT EXISTS " + newTableName + " as select * from " + oriTableName; + if (!isCopyData) { + sql += " where 1=0 "; + } + sql += ";"; + int errCode = SQLiteUtils::ExecuteRawSQL(db, sql); + if (errCode != E_OK) { + LOGE("[SQLite] execute create table sql failed"); + return errCode; + } + return E_OK; +} +int SQLiteUtils::RegisterFunction(sqlite3 *db, const std::string &funcName, int nArg, void *uData, TransactFunc &func) +{ + if (db == nullptr) { + LOGE("Sqlite DB not exists."); + return -E_INVALID_DB; + } + + int errCode = sqlite3_create_function_v2(db, funcName.c_str(), nArg, SQLITE_UTF8 | SQLITE_DETERMINISTIC, uData, + func.xFunc, func.xStep, func.xFinal, func.xDestroy); + if (errCode != SQLITE_OK) { + LOGE("sqlite3_create_function_v2 about [%s] returned %d", funcName.c_str(), errCode); + return MapSQLiteErrno(errCode); + } + return E_OK; +} +#endif int SQLiteUtils::RegisterFlatBufferFunction(sqlite3 *db, const std::string &inSchema) { if (db == nullptr) { + LOGE("Sqlite DB not exists."); return -E_INVALID_DB; } auto heapSchemaObj = new (std::nothrow) SchemaObject; @@ -972,6 +1407,70 @@ int SQLiteUtils::RegisterFlatBufferFunction(sqlite3 *db, const std::string &inSc return E_OK; } +void SQLiteUtils::UpdateMetaDataWithinTrigger(sqlite3_context *ctx, int argc, sqlite3_value **argv) +{ + if (ctx == nullptr || argc != 2 || argv == nullptr) { // 2 : Number of parameters for sqlite register function + LOGE("[UpdateMetaDataWithinTrigger] Invalid parameter, argc=%d.", argc); + return; + } + auto *handle = static_cast(sqlite3_user_data(ctx)); + if (handle == nullptr) { + sqlite3_result_error(ctx, "Sqlite context is invalid.", USING_STR_LEN); + LOGE("Sqlite context is invalid."); + return; + } + auto *keyPtr = static_cast(sqlite3_value_blob(argv[0])); // 0 : first argv for key + int keyLen = sqlite3_value_bytes(argv[0]); // 0 : first argv for key + if (keyPtr == nullptr || keyLen <= 0 || keyLen > static_cast(DBConstant::MAX_KEY_SIZE)) { + sqlite3_result_error(ctx, "key is invalid.", USING_STR_LEN); + LOGE("key is invalid."); + return; + } + auto val = sqlite3_value_int64(argv[1]); // 1 : second argv for value + + sqlite3_stmt *stmt = nullptr; + int errCode = SQLiteUtils::GetStatement(handle, UPDATE_META_SQL, stmt); + if (errCode != E_OK) { + sqlite3_result_error(ctx, "Get update meta_data statement failed.", USING_STR_LEN); + LOGE("Get update meta_data statement failed. %d", errCode); + return; + } + + Key key(keyPtr, keyPtr + keyLen); + errCode = SQLiteUtils::BindBlobToStatement(stmt, BIND_KEY_INDEX, key, false); + if (errCode != E_OK) { + sqlite3_result_error(ctx, "Bind key to statement failed.", USING_STR_LEN); + LOGE("Bind key to statement failed. %d", errCode); + goto END; + } + + errCode = SQLiteUtils::BindInt64ToStatement(stmt, BIND_VAL_INDEX, val); + if (errCode != E_OK) { + sqlite3_result_error(ctx, "Bind value to statement failed.", USING_STR_LEN); + LOGE("Bind value to statement failed. %d", errCode); + goto END; + } + + errCode = SQLiteUtils::StepWithRetry(stmt, false); + if (errCode != SQLiteUtils::MapSQLiteErrno(SQLITE_DONE)) { + sqlite3_result_error(ctx, "Execute the update meta_data attach failed.", USING_STR_LEN); + LOGE("Execute the update meta_data attach failed. %d", errCode); + } +END: + SQLiteUtils::ResetStatement(stmt, true, errCode); +} + +int SQLiteUtils::RegisterMetaDataUpdateFunction(sqlite3 *db) +{ + int errCode = sqlite3_create_function_v2(db, DBConstant::UPDATE_META_FUNC.c_str(), + 2, // 2: argc for register function + SQLITE_UTF8 | SQLITE_DETERMINISTIC, db, &SQLiteUtils::UpdateMetaDataWithinTrigger, nullptr, nullptr, nullptr); + if (errCode != SQLITE_OK) { + LOGE("sqlite3_create_function_v2 about %s returned %d", DBConstant::UPDATE_META_FUNC.c_str(), errCode); + } + return SQLiteUtils::MapSQLiteErrno(errCode); +} + struct ValueParseCache { ValueObject valueParsed; std::vector valueOriginal; @@ -1253,25 +1752,22 @@ int SQLiteUtils::GetDbSize(const std::string &dir, const std::string &dbName, ui std::string dataDir = dir + "/" + dbName + DBConstant::SQLITE_DB_EXTENSION; uint64_t localDbSize = 0; int errCode = OS::CalFileSize(dataDir, localDbSize); - if (errCode < 0) { - LOGE("Failed to get the db file size, errno:%d", errno); - if (errno == ENOENT) { - return -E_NOT_FOUND; - } - return -E_INVALID_DB; + if (errCode != E_OK) { + LOGD("Failed to get the db file size, errCode:%d", errCode); + return errCode; } std::string shmFileName = dataDir + "-shm"; uint64_t localshmFileSize = 0; errCode = OS::CalFileSize(shmFileName, localshmFileSize); - if (errCode < 0) { + if (errCode != E_OK) { localshmFileSize = 0; } std::string walFileName = dataDir + "-wal"; uint64_t localWalFileSize = 0; errCode = OS::CalFileSize(walFileName, localWalFileSize); - if (errCode < 0) { + if (errCode != E_OK) { localWalFileSize = 0; } @@ -1341,7 +1837,8 @@ int SQLiteUtils::SetDataBaseProperty(sqlite3 *db, const OpenDbProperties &proper if (errCode != E_OK) { return errCode; } - errCode = SQLiteUtils::SetKey(db, properties.cipherType, properties.passwd); + + errCode = SQLiteUtils::SetKey(db, properties.cipherType, properties.passwd, properties.isMemDb); if (errCode != E_OK) { LOGD("SQLiteUtils::SetKey fail!!![%d]", errCode); return errCode; @@ -1399,4 +1896,29 @@ std::string SQLiteUtils::GetCipherName(CipherType type) return ""; } #endif + +int SQLiteUtils::DropTriggerByName(sqlite3 *db, const std::string &name) +{ + const std::string dropTriggerSql = "DROP TRIGGER " + name + ";"; + int errCode = SQLiteUtils::ExecuteRawSQL(db, dropTriggerSql); + if (errCode != E_OK) { + LOGE("Remove trigger failed. %d", errCode); + } + return errCode; +} + +int SQLiteUtils::ExpandedSql(sqlite3_stmt *stmt, std::string &basicString) +{ + if (stmt == nullptr) { + return -E_INVALID_ARGS; + } + char *eSql = sqlite3_expanded_sql(stmt); + if (eSql == nullptr) { + LOGE("expand statement to sql failed."); + return -E_INVALID_DATA; + } + basicString = std::string(eSql); + sqlite3_free(eSql); + return E_OK; +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h index 0ec72484326f58b7d62ccecf25fc480103312c3a..94d7140010cec92e15b9bc74a1a5319ee507b2c4 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sqlite/sqlite_utils.h @@ -20,10 +20,13 @@ #include #include "sqlite_import.h" -#include "types.h" #include "db_types.h" -#include "schema_object.h" #include "iprocess_system_api_adapter.h" +#include "schema_object.h" +#include "types.h" +#ifdef RELATIONAL_STORE +#include "relational_schema_object.h" +#endif namespace DistributedDB { enum class TransactType { @@ -31,6 +34,24 @@ enum class TransactType { IMMEDIATE, }; +struct TransactFunc { + void (*xFunc)(sqlite3_context*, int, sqlite3_value**) = nullptr; + void (*xStep)(sqlite3_context*, int, sqlite3_value**) = nullptr; + void (*xFinal)(sqlite3_context*) = nullptr; + void(*xDestroy)(void*) = nullptr; +}; + +namespace TriggerMode { +enum class TriggerModeEnum { + NONE, + INSERT, + UPDATE, + DELETE +}; + +std::string GetTriggerModeString(TriggerModeEnum mode); +} + struct OpenDbProperties { std::string uri{}; bool createIfNecessary = true; @@ -38,8 +59,8 @@ struct OpenDbProperties { std::vector sqls{}; CipherType cipherType = CipherType::AES_256_GCM; CipherPassword passwd{}; - std::string schema = ""; - std::string subdir = ""; + std::string schema{}; + std::string subdir{}; SecurityOption securityOpt{}; int conflictReslovePolicy = DEFAULT_LAST_WIN; bool createDirByStoreIdOnly = false; @@ -79,10 +100,12 @@ public: static int ExecuteRawSQL(sqlite3 *db, const std::string &sql); - static int SetKey(sqlite3 *db, CipherType type, const CipherPassword &passwd); + static int SetKey(sqlite3 *db, CipherType type, const CipherPassword &passwd, const bool &isMemDb); static int GetColumnBlobValue(sqlite3_stmt *statement, int index, std::vector &value); + static int GetColumnTextValue(sqlite3_stmt *statement, int index, std::string &value); + static int ExportDatabase(sqlite3 *db, CipherType type, const CipherPassword &passwd, const std::string &newDbName); static int ExportDatabase(const std::string &srcFile, CipherType type, const CipherPassword &srcPasswd, @@ -120,14 +143,37 @@ public: // Register the flatBufferExtract function if the schema is of flatBuffer-type(To be refactor) static int RegisterFlatBufferFunction(sqlite3 *db, const std::string &inSchema); + static int RegisterMetaDataUpdateFunction(sqlite3 *db); + static int GetDbSize(const std::string &dir, const std::string &dbName, uint64_t &size); static int ExplainPlan(sqlite3 *db, const std::string &execSql, bool isQueryPlan); - static int AttachNewDatabase(sqlite3 *db, CipherType type, const CipherPassword &passwd, + static int AttachNewDatabase(sqlite3 *db, CipherType type, const CipherPassword &password, const std::string &attachDbAbsPath, const std::string &attachAsName = "backup"); static int CreateMetaDatabase(const std::string &metaDbPath); + + static int CheckIntegrity(sqlite3 *db, const std::string &sql); +#ifdef RELATIONAL_STORE + static int RegisterCalcHash(sqlite3 *db); + + static int RegisterGetSysTime(sqlite3 *db); + + static int CreateRelationalLogTable(sqlite3 *db, const std::string &oriTableName); + static int CreateRelationalMetaTable(sqlite3 *db); + + static int AddRelationalLogTableTrigger(sqlite3 *db, const TableInfo &table); + static int AnalysisSchema(sqlite3 *db, const std::string &tableName, TableInfo &table); + + static int CreateSameStuTable(sqlite3 *db, const std::string &oriTableName, const std::string &newTableName, + bool isCopyData); +#endif + + static int DropTriggerByName(sqlite3 *db, const std::string &name); + + static int ExpandedSql(sqlite3_stmt *stmt, std::string &basicString); + private: static int CreateDataBase(const OpenDbProperties &properties, sqlite3 *&dbTemp); @@ -147,17 +193,21 @@ private: static void CalcHashKey(sqlite3_context *ctx, int argc, sqlite3_value **argv); - static void GetSysCurrentTime(sqlite3_context *ctx, int argc, sqlite3_value **argv); + static void GetSysTime(sqlite3_context *ctx, int argc, sqlite3_value **argv); static int SetDataBaseProperty(sqlite3 *db, const OpenDbProperties &properties, const std::vector &sqls); + static int RegisterFunction(sqlite3 *db, const std::string &funcName, int nArg, void *uData, TransactFunc &func); + #ifndef OMIT_ENCRYPT static int SetCipherSettings(sqlite3 *db, CipherType type); static std::string GetCipherName(CipherType type); #endif + + static void UpdateMetaDataWithinTrigger(sqlite3_context *ctx, int argc, sqlite3_value **argv); }; -}; // namespace DistributedDB +} // namespace DistributedDB #endif // SQLITE_UTILS_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.cpp index 3939e4cedb34830236c2b7b4ee1f1b673dfda99b..674bb975fdb57d278e66be8fc8a780f098f7f9d6 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.cpp @@ -28,6 +28,7 @@ const int StorageEngine::MAX_READ_SIZE = 16; StorageEngine::StorageEngine() : isUpdated_(false), + isMigrating_(false), engineState_(EngineState::INVALID), commitNotifyFunc_(nullptr), isInitialized_(false), @@ -360,6 +361,7 @@ void StorageEngine::AddStorageExecutor(StorageExecutor *handle) } else { readIdleList_.push_back(handle); } + LOGD("add [%d] storage executor to handle pool.", handle->GetWritable()); } void StorageEngine::CloseExecutor() @@ -409,9 +411,8 @@ StorageExecutor *StorageEngine::FetchStorageExecutor(bool isWrite, std::list poolSize.maxReadNum || poolSize.minWriteNum > poolSize.maxWriteNum); } + +bool StorageEngine::IsMigrating() const +{ + return isMigrating_.load(); +} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.h index 7d3ca024134dd586931389bbd896c7ce4e867bbe..a948e236dc5ada3cfa962d02db668a521c870742 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine.h @@ -78,6 +78,8 @@ public: virtual int CheckEngineOption(const KvDBProperties &kvdbOption) const; + virtual bool IsMigrating() const; + protected: virtual int CreateNewExecutor(bool isWrite, StorageExecutor *&handle) = 0; @@ -89,10 +91,13 @@ protected: StorageEngineAttr engineAttr_; bool isUpdated_; + std::atomic isMigrating_; std::string identifier_; EngineState engineState_; + // Mutex for commitNotifyFunc_. mutable std::shared_mutex notifyMutex_; + // Callback function for commit notify. std::function commitNotifyFunc_; diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine_manager.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine_manager.cpp index c4116a16e3ad61bb9a66fa840dfdcc582360ebe9..369873ffcb5bcd9010db2c3e13ee156a548f921e 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/storage_engine_manager.cpp @@ -22,7 +22,7 @@ namespace DistributedDB { bool StorageEngineManager::isRegLockStatusListener_ = false; std::mutex StorageEngineManager::instanceLock_; -StorageEngineManager *StorageEngineManager::instance_ = nullptr; +std::atomic StorageEngineManager::instance_{nullptr}; std::mutex StorageEngineManager::storageEnginesLock_; namespace { @@ -67,7 +67,7 @@ StorageEngine *StorageEngineManager::GetStorageEngine(const KvDBProperties &prop errCode = storageEngine->CheckEngineOption(property); if (errCode != E_OK) { LOGE("kvdb property mismatch engine option! errCode = [%d]", errCode); - return nullptr; + storageEngine = nullptr; } } @@ -130,17 +130,20 @@ int StorageEngineManager::ExecuteMigration(StorageEngine *storageEngine) StorageEngineManager *StorageEngineManager::GetInstance() { - std::lock_guard lockGuard(instanceLock_); + // For Double-Checked Locking, we need check instance_ twice if (instance_ == nullptr) { - instance_ = new (std::nothrow) StorageEngineManager(); + std::lock_guard lockGuard(instanceLock_); if (instance_ == nullptr) { - LOGE("[StorageEngineManager] Failed to alloc the engine manager!"); - return nullptr; + instance_ = new (std::nothrow) StorageEngineManager(); + if (instance_ == nullptr) { + LOGE("[StorageEngineManager] Failed to alloc the engine manager!"); + return nullptr; + } } } if (!isRegLockStatusListener_) { - int errCode = instance_->RegisterLockStatusListener(); + int errCode = (instance_.load())->RegisterLockStatusListener(); if (errCode != E_OK) { LOGW("[StorageEngineManager] Failed to regitster lock status listener:%d", errCode); } else { diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5ae456f8f4c701ca2f362a24ac4455d0e5c90a0e --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.cpp @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "sync_able_engine.h" + +#include "db_errno.h" +#include "log_print.h" +#include "parcel.h" +#include "runtime_context.h" + +namespace DistributedDB { +SyncAbleEngine::SyncAbleEngine(ISyncInterface *store) + : started_(false), + store_(store) +{} + +SyncAbleEngine::~SyncAbleEngine() +{ +} + +// Start a sync action. +int SyncAbleEngine::Sync(const ISyncer::SyncParma &parm) +{ + if (!started_) { + StartSyncer(); + if (!started_) { + return -E_NOT_INIT; + } + } + return syncer_.Sync(parm); +} + +void SyncAbleEngine::WakeUpSyncer() +{ + StartSyncer(); +} + +void SyncAbleEngine::Close() +{ + StopSyncer(); +} + +void SyncAbleEngine::EnableAutoSync(bool enable) +{ + if (!started_) { + StartSyncer(); + } + return syncer_.EnableAutoSync(enable); +} + +int SyncAbleEngine::EnableManualSync(void) +{ + return syncer_.EnableManualSync(); +} + +int SyncAbleEngine::DisableManualSync(void) +{ + return syncer_.DisableManualSync(); +} + +// Stop a sync action in progress. +void SyncAbleEngine::StopSync(int syncId) +{ + if (started_) { + syncer_.RemoveSyncOperation(syncId); + } +} + +// Get The current virtual timestamp +uint64_t SyncAbleEngine::GetTimeStamp() +{ + if (!started_) { + StartSyncer(); + } + return syncer_.GetTimeStamp(); +} + +int SyncAbleEngine::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +{ + if (!started_) { + StartSyncer(); + } + return syncer_.EraseDeviceWaterMark(deviceId, isNeedHash); +} + +// Start syncer +void SyncAbleEngine::StartSyncer() +{ + if (store_ == nullptr) { + LOGF("KvDB got null sync interface."); + return; + } + + int errCode = syncer_.Initialize(store_); + if (errCode == E_OK) { + started_ = true; + } else { + LOGE("KvDB start syncer failed, err:'%d'.", errCode); + } +} + +// Stop syncer +void SyncAbleEngine::StopSyncer() +{ + if (started_) { + syncer_.Close(); + started_ = false; + } +} + +void SyncAbleEngine::TriggerSync(int notifyEvent) +{ + if (!started_) { + StartSyncer(); + } + if (started_) { + int errCode = RuntimeContext::GetInstance()->ScheduleTask([this, notifyEvent] { + syncer_.LocalDataChanged(notifyEvent); + }); + if (errCode != E_OK) { + LOGE("[TriggerSync] SyncAbleEngine TriggerSync LocalDataChanged retCode:%d", errCode); + } + } +} + +int SyncAbleEngine::GetLocalIdentity(std::string &outTarget) +{ + if (!started_) { + StartSyncer(); + } + return syncer_.GetLocalIdentity(outTarget); +} +} diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h new file mode 100644 index 0000000000000000000000000000000000000000..08e2120303c4f149f984c0cc4e5195ddb9f4be85 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_engine.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SYNC_ABLE_ENGINE_H +#define SYNC_ABLE_ENGINE_H + +#include + +#include "ref_object.h" +#include "syncer_proxy.h" + +namespace DistributedDB { +class SyncAbleEngine final { +public: + explicit SyncAbleEngine(ISyncInterface *store); + ~SyncAbleEngine(); + void TriggerSync(int notifyEvent); + + // Start a sync action. + int Sync(const ISyncer::SyncParma &parm); + + void WakeUpSyncer(); + void Close(); + + // Enable auto sync + void EnableAutoSync(bool enable); + + int EnableManualSync(void); + int DisableManualSync(void); + + // Stop a sync action in progress. + void StopSync(int syncId); + + // Get The current virtual timestamp + uint64_t GetTimeStamp(); + + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash); + + int GetLocalIdentity(std::string &outTarget); + +private: + // Start syncer + void StartSyncer(); + + // Stop syncer + void StopSyncer(); + + SyncerProxy syncer_; // use for sync Interactive + std::atomic started_; + ISyncInterface *store_; +}; +} // namespace DistributedDB +#endif // SYNC_ABLE_ENGINE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp index 3ae8676f7b220060282299ddfdbec1ab3749d056..e9ada439e305d1431a2e0c76605ccbc8821062e6 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.cpp @@ -14,24 +14,26 @@ */ #include "sync_able_kvdb.h" + #include "db_errno.h" #include "log_print.h" #include "parcel.h" +#include "runtime_context.h" namespace DistributedDB { const EventType SyncAbleKvDB::REMOTE_PUSH_FINISHED = 1; SyncAbleKvDB::SyncAbleKvDB() : started_(false), - remotePushNotifyChain_(nullptr) + notifyChain_(nullptr) {} SyncAbleKvDB::~SyncAbleKvDB() { - if (remotePushNotifyChain_ != nullptr) { - (void)remotePushNotifyChain_->UnRegisterEventType(REMOTE_PUSH_FINISHED); - KillAndDecObjRef(remotePushNotifyChain_); - remotePushNotifyChain_ = nullptr; + if (notifyChain_ != nullptr) { + (void)notifyChain_->UnRegisterEventType(REMOTE_PUSH_FINISHED); + KillAndDecObjRef(notifyChain_); + notifyChain_ = nullptr; } } @@ -67,9 +69,7 @@ void SyncAbleKvDB::Close() } // Start a sync action. -int SyncAbleKvDB::Sync(const std::vector &devices, int mode, - const std::function &)> &onComplete, - const std::function &onFinalize, bool wait = false) +int SyncAbleKvDB::Sync(const ISyncer::SyncParma &parma) { if (!started_) { StartSyncer(); @@ -77,7 +77,7 @@ int SyncAbleKvDB::Sync(const std::vector &devices, int mode, return -E_NOT_INIT; } } - return syncer_.Sync(devices, mode, onComplete, onFinalize, wait); + return syncer_.Sync(parma); } void SyncAbleKvDB::EnableAutoSync(bool enable) @@ -94,10 +94,10 @@ void SyncAbleKvDB::WakeUpSyncer() } // Stop a sync action in progress. -void SyncAbleKvDB::StopSync(int syncId) +void SyncAbleKvDB::StopSync() { if (started_) { - syncer_.RemoveSyncOperation(syncId); + syncer_.StopSync(); } } @@ -191,33 +191,44 @@ int SyncAbleKvDB::SetStaleDataWipePolicy(WipePolicy policy) return syncer_.SetStaleDataWipePolicy(policy); } -NotificationChain::Listener *SyncAbleKvDB::AddRemotePushFinishedNotify(const RemotePushFinishedNotifier ¬ifier, - int errCode) +int SyncAbleKvDB::RegisterEventType(EventType type) { - { - std::lock_guard lock(remotePushNotifyChainLock_); - if (remotePushNotifyChain_ == nullptr) { - remotePushNotifyChain_ = new (std::nothrow) NotificationChain; - if (remotePushNotifyChain_ == nullptr) { - errCode = -E_OUT_OF_MEMORY; - return nullptr; - } - - errCode = remotePushNotifyChain_->RegisterEventType(REMOTE_PUSH_FINISHED); - if (errCode != E_OK) { - LOGE("[SyncAbleKvDB] Register remote push finished event type failed! err %d", errCode); - KillAndDecObjRef(remotePushNotifyChain_); - remotePushNotifyChain_ = nullptr; - return nullptr; - } + if (notifyChain_ == nullptr) { + notifyChain_ = new (std::nothrow) NotificationChain; + if (notifyChain_ == nullptr) { + return -E_OUT_OF_MEMORY; } } - NotificationChain::Listener *listener = - remotePushNotifyChain_->RegisterListener(REMOTE_PUSH_FINISHED, - [notifier](void *arg) { - notifier(*static_cast(arg)); - }, nullptr, errCode); + int errCode = notifyChain_->RegisterEventType(type); + if (errCode == -E_ALREADY_REGISTER) { + return E_OK; + } + if (errCode != E_OK) { + LOGE("[SyncAbleKvDB] Register event type %u failed! err %d", type, errCode); + KillAndDecObjRef(notifyChain_); + notifyChain_ = nullptr; + } + return errCode; +} + +NotificationChain::Listener *SyncAbleKvDB::AddRemotePushFinishedNotify(const RemotePushFinishedNotifier ¬ifier, + int &errCode) +{ + std::unique_lock lock(notifyChainLock_); + errCode = RegisterEventType(REMOTE_PUSH_FINISHED); + if (errCode != E_OK) { + return nullptr; + } + + auto listener = notifyChain_->RegisterListener(REMOTE_PUSH_FINISHED, + [notifier](void *arg) { + if (arg == nullptr) { + LOGE("PragmaRemotePushNotify is null."); + return; + } + notifier(*static_cast(arg)); + }, nullptr, errCode); if (errCode != E_OK) { LOGE("[SyncAbleKvDB] Add remote push finished notifier failed! err %d", errCode); } @@ -226,10 +237,24 @@ NotificationChain::Listener *SyncAbleKvDB::AddRemotePushFinishedNotify(const Rem void SyncAbleKvDB::NotifyRemotePushFinishedInner(const std::string &targetId) const { - if (remotePushNotifyChain_ != nullptr) { - RemotePushNotifyInfo info; - info.deviceId = targetId; - remotePushNotifyChain_->NotifyEvent(REMOTE_PUSH_FINISHED, static_cast(&info)); + { + std::shared_lock lock(notifyChainLock_); + if (notifyChain_ == nullptr) { + return; + } } + RemotePushNotifyInfo info; + info.deviceId = targetId; + notifyChain_->NotifyEvent(REMOTE_PUSH_FINISHED, static_cast(&info)); +} + +int SyncAbleKvDB::SetSyncRetry(bool isRetry) +{ + return syncer_.SetSyncRetry(isRetry); +} + +int SyncAbleKvDB::SetEqualIdentifier(const std::string &identifier, const std::vector &targets) +{ + return syncer_.SetEqualIdentifier(identifier, targets); } } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h index 1b8116245faa46ef1a24d37cbd86528968c620ed..5c89a5e6a3b953f83621bb739756727d70b1991f 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb.h @@ -16,9 +16,12 @@ #ifndef SYNC_ABLE_KVDB_H #define SYNC_ABLE_KVDB_H +#include + #include "generic_kvdb.h" -#include "sync_able_kvdb_connection.h" #include "ikvdb_sync_interface.h" +#include "intercepted_data.h" +#include "sync_able_kvdb_connection.h" #include "syncer_proxy.h" namespace DistributedDB { @@ -38,15 +41,13 @@ public: void Close() override; // Start a sync action. - int Sync(const std::vector &devices, int mode, - const std::function &)> &onComplete, - const std::function &onFinalize, bool wait); + int Sync(const ISyncer::SyncParma &parma); // Enable auto sync void EnableAutoSync(bool enable); // Stop a sync action in progress. - void StopSync(int syncId); + void StopSync(); // Get The current virtual timestamp uint64_t GetTimeStamp(); @@ -70,12 +71,18 @@ public: int SetStaleDataWipePolicy(WipePolicy policy); - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash = true); + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash); - NotificationChain::Listener *AddRemotePushFinishedNotify(const RemotePushFinishedNotifier ¬ifier, int errCode); + NotificationChain::Listener *AddRemotePushFinishedNotify(const RemotePushFinishedNotifier ¬ifier, int &errCode); void NotifyRemotePushFinishedInner(const std::string &targetId) const; + int SetSyncRetry(bool isRetry); + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const std::string &identifier, const std::vector &targets); + + virtual void SetDataInterceptor(const PushDataInterceptor &interceptor) = 0; + protected: virtual IKvDBSyncInterface *GetSyncInterface() = 0; @@ -93,10 +100,13 @@ protected: void TriggerSync(int notifyEvent); private: + int RegisterEventType(EventType type); + SyncerProxy syncer_; std::atomic started_; - mutable std::mutex remotePushNotifyChainLock_; - NotificationChain *remotePushNotifyChain_; + mutable std::shared_mutex notifyChainLock_; + NotificationChain *notifyChain_; + static const EventType REMOTE_PUSH_FINISHED; }; } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp index 94ac9f8260dd1ef42f4586d5320ba2a8885aac6e..11a1e6217ac0beea378f12a7c5c9ed515f513877 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.cpp @@ -28,18 +28,14 @@ SyncAbleKvDBConnection::SyncAbleKvDBConnection(SyncAbleKvDB *kvDB) remotePushFinishedListener_(nullptr) { OnKill([this]() { - for (const auto &syncId : syncIdList_) { - SyncAbleKvDB *db = GetDB(); - if (syncId <= 0 || db == nullptr) { - continue; - } - - // Drop the lock before we call RemoveSyncOperation(). - UnlockObj(); - db->StopSync(syncId); - LockObj(); + auto *db = GetDB(); + if (db == nullptr) { + return; } - syncIdList_.clear(); + // Drop the lock before we call RemoveSyncOperation(). + UnlockObj(); + db->StopSync(); + LockObj(); }); } @@ -51,52 +47,61 @@ SyncAbleKvDBConnection::~SyncAbleKvDBConnection() remotePushFinishedListener_ = nullptr; } +void SyncAbleKvDBConnection::InitPragmaFunc() +{ + if (!pragmaFunc_.empty()) { + return; + } + pragmaFunc_ = { + {PRAGMA_SYNC_DEVICES, [this](void *parameter, int &errCode) { + errCode = PragmaSyncAction(static_cast(parameter)); }}, + {PRAGMA_AUTO_SYNC, [this](void *parameter, int &errCode) { + errCode = EnableAutoSync(*(static_cast(parameter))); }}, + {PRAGMA_PERFORMANCE_ANALYSIS_GET_REPORT, [](void *parameter, int &errCode) { + *(static_cast(parameter)) = PerformanceAnalysis::GetInstance()->GetStatistics(); }}, + {PRAGMA_PERFORMANCE_ANALYSIS_OPEN, [](void *parameter, int &errCode) { + PerformanceAnalysis::GetInstance()->OpenPerformanceAnalysis(); }}, + {PRAGMA_PERFORMANCE_ANALYSIS_CLOSE, [](void *parameter, int &errCode) { + PerformanceAnalysis::GetInstance()->ClosePerformanceAnalysis(); }}, + {PRAGMA_PERFORMANCE_ANALYSIS_SET_REPORTFILENAME, [](void *parameter, int &errCode) { + PerformanceAnalysis::GetInstance()->SetFileNumber(*(static_cast(parameter))); }}, + {PRAGMA_GET_QUEUED_SYNC_SIZE, [this](void *parameter, int &errCode) { + errCode = GetQueuedSyncSize(static_cast(parameter)); }}, + {PRAGMA_SET_QUEUED_SYNC_LIMIT, [this](void *parameter, int &errCode) { + errCode = SetQueuedSyncLimit(static_cast(parameter)); }}, + {PRAGMA_GET_QUEUED_SYNC_LIMIT, [this](void *parameter, int &errCode) { + errCode = GetQueuedSyncLimit(static_cast(parameter)); }}, + {PRAGMA_SET_WIPE_POLICY, [this](void *parameter, int &errCode) { + errCode = SetStaleDataWipePolicy(static_cast(parameter)); }}, + {PRAGMA_REMOTE_PUSH_FINISHED_NOTIFY, [this](void *parameter, int &errCode) { + errCode = SetRemotePushFinishedNotify(static_cast(parameter)); }}, + {PRAGMA_SET_SYNC_RETRY, [this](void *parameter, int &errCode) { + errCode = SetSyncRetry(*(static_cast(parameter))); }}, + {PRAGMA_ADD_EQUAL_IDENTIFIER, [this](void *parameter, int &errCode) { + errCode = SetEqualIdentifier(static_cast(parameter)); }}, + {PRAGMA_INTERCEPT_SYNC_DATA, [this](void *parameter, int &errCode) { + errCode = SetPushDataInterceptor(*static_cast(parameter)); }}, + {PRAGMA_SUBSCRIBE_QUERY, [this](void *parameter, int &errCode) { + errCode = PragmaSyncAction(static_cast(parameter)); }}, + }; +} + int SyncAbleKvDBConnection::Pragma(int cmd, void *parameter) { int errCode = PragmaParamCheck(cmd, parameter); if (errCode != E_OK) { return -E_INVALID_ARGS; } - switch (cmd) { - case PRAGMA_SYNC_DEVICES: - errCode = PragmaSyncAction(static_cast(parameter)); - break; - case PRAGMA_AUTO_SYNC: - errCode = EnableAutoSync(*(static_cast(parameter))); - break; - case PRAGMA_PERFORMANCE_ANALYSIS_GET_REPORT: - *(static_cast(parameter)) = PerformanceAnalysis::GetInstance()->GetStatistics(); - break; - case PRAGMA_PERFORMANCE_ANALYSIS_OPEN: - PerformanceAnalysis::GetInstance()->OpenPerformanceAnalysis(); - break; - case PRAGMA_PERFORMANCE_ANALYSIS_CLOSE: - PerformanceAnalysis::GetInstance()->ClosePerformanceAnalysis(); - break; - case PRAGMA_PERFORMANCE_ANALYSIS_SET_REPORTFILENAME: - PerformanceAnalysis::GetInstance()->SetFileNumber(*(static_cast(parameter))); - break; - case PRAGMA_GET_QUEUED_SYNC_SIZE: - errCode = GetQueuedSyncSize(static_cast(parameter)); - break; - case PRAGMA_SET_QUEUED_SYNC_LIMIT: - errCode = SetQueuedSyncLimit(static_cast(parameter)); - break; - case PRAGMA_GET_QUEUED_SYNC_LIMIT: - errCode = GetQueuedSyncLimit(static_cast(parameter)); - break; - case PRAGMA_SET_WIPE_POLICY: - errCode = SetStaleDataWipePolicy(static_cast(parameter)); - break; - case PRAGMA_REMOTE_PUSH_FINISHED_NOTIFY: - errCode = SetRemotePushFinishedNotify(static_cast(parameter)); - break; - default: - // Call Pragma() of super class. - errCode = GenericKvDBConnection::Pragma(cmd, parameter); - break; + + InitPragmaFunc(); + auto iter = pragmaFunc_.find(cmd); + if (iter != pragmaFunc_.end()) { + iter->second(parameter, errCode); + return errCode; } - return errCode; + + // Call Pragma() of super class. + return GenericKvDBConnection::Pragma(cmd, parameter); } int SyncAbleKvDBConnection::PragmaParamCheck(int cmd, const void *parameter) @@ -136,18 +141,19 @@ int SyncAbleKvDBConnection::PragmaSyncAction(const PragmaSync *syncParameter) } IncObjRef(this); } - int errCode = kvDB->Sync(syncParameter->devices_, syncParameter->mode_, - std::bind(&SyncAbleKvDBConnection::OnSyncComplete, this, std::placeholders::_1, syncParameter->onComplete_, - syncParameter->wait_), [this]() { - DecObjRef(this); - }, syncParameter->wait_); - if (errCode <= 0) { + + ISyncer::SyncParma syncParam; + syncParam.devices = syncParameter->devices_; + syncParam.mode = syncParameter->mode_; + syncParam.wait = syncParameter->wait_; + syncParam.isQuerySync = syncParameter->isQuerySync_; + syncParam.syncQuery = syncParameter->query_; + syncParam.onFinalize = [this]() { DecObjRef(this); }; + syncParam.onComplete = std::bind(&SyncAbleKvDBConnection::OnSyncComplete, this, std::placeholders::_1, + syncParameter->onComplete_, syncParameter->wait_); + int errCode = kvDB->Sync(syncParam); + if (errCode != E_OK) { DecObjRef(this); - } else if (!syncParameter->wait_) { - AutoLock lockGuard(this); - if (!IsKilled()) { - syncIdList_.push_back(errCode); - } } return errCode; } @@ -166,11 +172,6 @@ void SyncAbleKvDBConnection::OnSyncComplete(const std::map &st const std::function &devicesMap)> &onComplete, bool wait) { AutoLock lockGuard(this); - if (!wait && !IsKilled()) { - if (!syncIdList_.empty()) { - syncIdList_.pop_front(); - } - } if (!IsKilled() && onComplete) { // Drop the lock before invoking the callback. // Do pragma-sync again in the prev sync callback is supported. @@ -289,4 +290,36 @@ int SyncAbleKvDBConnection::SetRemotePushFinishedNotify(PragmaRemotePushNotify * remotePushFinishedListener_ = tmpListener; return errCode; } + +int SyncAbleKvDBConnection::SetSyncRetry(bool isRetry) +{ + SyncAbleKvDB *kvDB = GetDB(); + if (kvDB == nullptr) { + return -E_INVALID_CONNECTION; + } + return kvDB->SetSyncRetry(isRetry); +} + +int SyncAbleKvDBConnection::SetEqualIdentifier(const PragmaSetEqualIdentifier *param) +{ + if (param == nullptr) { + return -E_INVALID_ARGS; + } + + SyncAbleKvDB *kvDB = GetDB(); + if (kvDB == nullptr) { + return -E_INVALID_CONNECTION; + } + return kvDB->SetEqualIdentifier(param->identifier_, param->targets_); +} + +int SyncAbleKvDBConnection::SetPushDataInterceptor(const PushDataInterceptor &interceptor) +{ + SyncAbleKvDB *kvDB = GetDB(); + if (kvDB == nullptr) { + return -E_INVALID_CONNECTION; + } + kvDB->SetDataInterceptor(interceptor); + return E_OK; +} } diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h index c6c77128f86044267a5db14e92bef076eebe420e..a640bbe0fd92bd42c31015edcbe0712328287056 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/sync_able_kvdb_connection.h @@ -16,13 +16,15 @@ #ifndef SYNC_ABLE_KVDB_CONNECTION_H #define SYNC_ABLE_KVDB_CONNECTION_H -#include "ref_object.h" #include "generic_kvdb_connection.h" +#include "intercepted_data.h" +#include "ref_object.h" namespace DistributedDB { class SyncAbleKvDB; struct PragmaSync; struct PragmaRemotePushNotify; +struct PragmaSetEqualIdentifier; class SyncAbleKvDBConnection : public GenericKvDBConnection, public virtual RefObject { public: @@ -42,6 +44,7 @@ private: // Do pragma-sync action. int PragmaParamCheck(int cmd, const void *parameter); int PragmaSyncAction(const PragmaSync *syncParameter); + void InitPragmaFunc(); // If enable is true, it will enable auto sync int EnableAutoSync(bool enable); @@ -59,12 +62,15 @@ private: int SetRemotePushFinishedNotify(PragmaRemotePushNotify *notifyParma); - // For sync in progress. - std::list syncIdList_; + int SetSyncRetry(bool isRetry); + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const PragmaSetEqualIdentifier *param); + + int SetPushDataInterceptor(const PushDataInterceptor &interceptor); std::mutex remotePushFinishedListenerLock_; NotificationChain::Listener *remotePushFinishedListener_; + std::map> pragmaFunc_{}; }; } - #endif // SYNC_ABLE_KVDB_CONNECTION_H diff --git a/services/distributeddataservice/libs/distributeddb/storage/src/upgrader/single_ver_schema_database_upgrader.h b/services/distributeddataservice/libs/distributeddb/storage/src/upgrader/single_ver_schema_database_upgrader.h index 6f601655334179766c59d76ca6f958887a1b6168..2054671f441237f13ce311f335429438cedb13a2 100755 --- a/services/distributeddataservice/libs/distributeddb/storage/src/upgrader/single_ver_schema_database_upgrader.h +++ b/services/distributeddataservice/libs/distributeddb/storage/src/upgrader/single_ver_schema_database_upgrader.h @@ -27,11 +27,13 @@ public: ~SingleVerSchemaDatabaseUpgrader() override {}; protected: int ExecuteUpgrade() override; + // Database content related upgrade int ExecuteUpgradeSchema(); // Get an empty string with return_code E_OK indicate no schema but everything normally virtual int GetDatabaseSchema(std::string &dbSchema) const = 0; + // Set or update schema into database file virtual int SetDatabaseSchema(const std::string &dbSchema) = 0; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h index 6fdf05bd279d1d156f94130b349a9c30f60af3dd..fe177b5d36d6047a40b15b9bd7331180c2956ac5 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/isyncer.h @@ -20,16 +20,32 @@ #include #include -#include "ikvdb_sync_interface.h" -#include "types_export.h" +#include "isync_interface.h" +#include "types_export.h" +#include "query_sync_object.h" +#include "types.h" namespace DistributedDB { class ISyncer { public: + struct SyncParma { + std::vector devices; + std::function &devicesMap)> onComplete; + SyncStatusCallback relationOnComplete; + std::function onFinalize; + int mode = 0; + bool wait = false; + bool isQuerySync = false; + QuerySyncObject syncQuery; + }; + virtual ~ISyncer() {}; // Init the Syncer modules - virtual int Initialize(IKvDBSyncInterface *syncInterface) = 0; + virtual int Initialize(ISyncInterface *syncInterface) + { + return -E_NOT_SUPPORT; + } // Close virtual int Close() = 0; @@ -44,10 +60,15 @@ public: const std::function &)> &onComplete, const std::function &onFinalize, bool wait) = 0; + // Sync function. use SyncParma to reduce paramter. + virtual int Sync(const SyncParma ¶m) = 0; + // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. virtual int RemoveSyncOperation(int syncId) = 0; - // Get The current vitural timestamp + virtual int StopSync() = 0; + + // Get The current virtual timestamp virtual uint64_t GetTimeStamp() = 0; // Enable auto sync function @@ -79,6 +100,12 @@ public: // Set stale data wipe policy virtual int SetStaleDataWipePolicy(WipePolicy policy) = 0; + + // Set Manual Sync retry config + virtual int SetSyncRetry(bool isRetry) = 0; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + virtual int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/sync_config.h b/services/distributeddataservice/libs/distributeddb/syncer/include/sync_config.h new file mode 100644 index 0000000000000000000000000000000000000000..2d342a53b8693dcb28972e80dfc98861264912d4 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/sync_config.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SYNC_CONFIG_H +#define SYNC_CONFIG_H + +#include +#include +#include +#include "macro_utils.h" +#include "parcel.h" +#include "types_export.h" + +// db ability config +namespace DistributedDB { +// offset, used_bits_num, used_bits_num < 64 +using AbilityItem = std::pair; +// format: {offset, used_bits_num} +/* +if need to add new ability, just add append to the last ability +current ability format: +|first bit|second bit|third bit| +|DATABASE_COMPRESSION_ZLIB|ALLPREDICATEQUERY|SUBSCRIBEQUERY| +*/ +constexpr AbilityItem DATABASE_COMPRESSION_ZLIB = {0, 1}; +constexpr AbilityItem ALLPREDICATEQUERY = {1, 1}; // offset: 0 + 1 +constexpr AbilityItem SUBSCRIBEQUERY = {2, 1}; // // offset: 1 + 1 + +const std::vector ABILITYBITS = {DATABASE_COMPRESSION_ZLIB, ALLPREDICATEQUERY, SUBSCRIBEQUERY}; + +const std::map COMPRESSALGOMAP = { + {static_cast(CompressAlgorithm::ZLIB), DATABASE_COMPRESSION_ZLIB}, +}; +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h index 9e2ba913d5d176a5c79997f388901a9e0586799b..b2b698e6e24de034228a9958989509d61dbac82b 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_proxy.h @@ -22,7 +22,6 @@ #include #include "isyncer.h" -#include "ikvdb_sync_interface.h" namespace DistributedDB { class SyncerProxy : public ISyncer { @@ -31,7 +30,7 @@ public: ~SyncerProxy() {}; // Init the Syncer modules - int Initialize(IKvDBSyncInterface *syncInterface) override; + int Initialize(ISyncInterface *syncInterface) override; // Close the syncer int Close() override; @@ -46,9 +45,14 @@ public: const std::function &)> &onComplete, const std::function &onFinalize, bool wait) override; + // Sync function. use SyncParma to reduce paramter. + int Sync(const SyncParma ¶m) override; + // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. int RemoveSyncOperation(int syncId) override; + int StopSync() override; + // Get The current virtual timestamp uint64_t GetTimeStamp() override; @@ -82,6 +86,12 @@ public: // Set stale data wipe policy int SetStaleDataWipePolicy(WipePolicy policy) override; + // Set Manual Sync retry config + int SetSyncRetry(bool isRetry) override; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) override; + private: std::mutex syncerLock_; std::shared_ptr syncer_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp index 9d1cc7e070b39964a7a2b86bbb510e6e0aafe3ac..f7a591fe378c8d638c94a7991090f32841cb9535 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.cpp @@ -20,8 +20,12 @@ #include "db_errno.h" #include "log_print.h" #include "sync_types.h" +#include "db_common.h" #include "single_ver_kvdb_sync_interface.h" #include "single_ver_sync_task_context.h" +#ifdef RELATIONAL_STORE +#include "relational_db_sync_interface.h" +#endif namespace DistributedDB { AbilitySyncRequestPacket::AbilitySyncRequestPacket() @@ -30,7 +34,8 @@ AbilitySyncRequestPacket::AbilitySyncRequestPacket() softwareVersion_(SOFTWARE_VERSION_CURRENT), secLabel_(0), secFlag_(0), - schemaType_(0) + schemaType_(0), + dbCreateTime_(0) { } @@ -73,9 +78,9 @@ void AbilitySyncRequestPacket::SetSchema(const std::string &schema) schema_ = schema; } -void AbilitySyncRequestPacket::GetSchema(std::string &schema) const +std::string AbilitySyncRequestPacket::GetSchema() const { - schema = schema_; + return schema_; } void AbilitySyncRequestPacket::SetSchemaType(uint32_t schemaType) @@ -108,6 +113,16 @@ int32_t AbilitySyncRequestPacket::GetSecFlag() const return secFlag_; } +void AbilitySyncRequestPacket::SetDbCreateTime(uint64_t dbCreateTime) +{ + dbCreateTime_ = dbCreateTime; +} + +uint64_t AbilitySyncRequestPacket::GetDbCreateTime() const +{ + return dbCreateTime_; +} + uint32_t AbilitySyncRequestPacket::CalculateLen() const { uint64_t len = 0; @@ -123,6 +138,8 @@ uint32_t AbilitySyncRequestPacket::CalculateLen() const len += Parcel::GetIntLen(); // secLabel_ len += Parcel::GetIntLen(); // secFlag_ len += Parcel::GetUInt32Len(); // schemaType_ + len += Parcel::GetUInt64Len(); // dbCreateTime_ + len += DbAbility::CalculateLen(dbAbility_); // dbAbility_ // the reason why not 8-byte align is that old version is not 8-byte align // so it is not possible to set 8-byte align for high version. if (len > INT32_MAX) { @@ -132,6 +149,16 @@ uint32_t AbilitySyncRequestPacket::CalculateLen() const return len; } +DbAbility AbilitySyncRequestPacket::GetDbAbility() const +{ + return dbAbility_; +} + +void AbilitySyncRequestPacket::SetDbAbility(const DbAbility &dbAbility) +{ + dbAbility_ = dbAbility; +} + AbilitySyncAckPacket::AbilitySyncAckPacket() : protocolVersion_(ABILITY_SYNC_VERSION_V1), softwareVersion_(SOFTWARE_VERSION_CURRENT), @@ -140,7 +167,8 @@ AbilitySyncAckPacket::AbilitySyncAckPacket() secFlag_(0), schemaType_(0), permitSync_(0), - requirePeerConvert_(0) + requirePeerConvert_(0), + dbCreateTime_(0) { } @@ -183,9 +211,9 @@ void AbilitySyncAckPacket::SetSchema(const std::string &schema) schema_ = schema; } -void AbilitySyncAckPacket::GetSchema(std::string &schema) const +std::string AbilitySyncAckPacket::GetSchema() const { - schema = schema_; + return schema_; } void AbilitySyncAckPacket::SetSchemaType(uint32_t schemaType) @@ -238,6 +266,16 @@ uint32_t AbilitySyncAckPacket::GetRequirePeerConvert() const return requirePeerConvert_; } +void AbilitySyncAckPacket::SetDbCreateTime(uint64_t dbCreateTime) +{ + dbCreateTime_ = dbCreateTime; +} + +uint64_t AbilitySyncAckPacket::GetDbCreateTime() const +{ + return dbCreateTime_; +} + uint32_t AbilitySyncAckPacket::CalculateLen() const { uint64_t len = 0; @@ -255,6 +293,8 @@ uint32_t AbilitySyncAckPacket::CalculateLen() const len += Parcel::GetUInt32Len(); // schemaType_ len += Parcel::GetUInt32Len(); // permitSync_ len += Parcel::GetUInt32Len(); // requirePeerConvert_ + len += Parcel::GetUInt64Len(); // dbCreateTime_ + len += DbAbility::CalculateLen(dbAbility_); // dbAbility_ if (len > INT32_MAX) { LOGE("[AbilitySyncAckPacket][CalculateLen] err len:%llu", len); return 0; @@ -262,9 +302,20 @@ uint32_t AbilitySyncAckPacket::CalculateLen() const return len; } +DbAbility AbilitySyncAckPacket::GetDbAbility() const +{ + return dbAbility_; +} + +void AbilitySyncAckPacket::SetDbAbility(const DbAbility &dbAbility) +{ + dbAbility_ = dbAbility; +} + AbilitySync::AbilitySync() : communicator_(nullptr), storageInterface_(nullptr), + metadata_(nullptr), syncFinished_(false) { } @@ -273,15 +324,18 @@ AbilitySync::~AbilitySync() { communicator_ = nullptr; storageInterface_ = nullptr; + metadata_ = nullptr; } -int AbilitySync::Initialize(ICommunicator *inCommunicator, IKvDBSyncInterface *inStorage, const std::string &deviceId) +int AbilitySync::Initialize(ICommunicator *inCommunicator, ISyncInterface *inStorage, + std::shared_ptr &inMetadata, const std::string &deviceId) { - if (inCommunicator == nullptr || inStorage == nullptr || deviceId.empty()) { + if (inCommunicator == nullptr || inStorage == nullptr || deviceId.empty() || inMetadata == nullptr) { return -E_INVALID_ARGS; } communicator_ = inCommunicator; storageInterface_ = inStorage; + metadata_ = inMetadata; deviceId_ = deviceId; return E_OK; } @@ -289,30 +343,17 @@ int AbilitySync::Initialize(ICommunicator *inCommunicator, IKvDBSyncInterface *i int AbilitySync::SyncStart(uint32_t sessionId, uint32_t sequenceId, uint16_t remoteCommunicatorVersion, const CommErrHandler &handler) { + AbilitySyncRequestPacket packet; + int errCode = SetAbilityRequestBodyInfo(packet, remoteCommunicatorVersion); + if (errCode != E_OK) { + return errCode; + } Message *message = new (std::nothrow) Message(ABILITY_SYNC_MESSAGE); if (message == nullptr) { return -E_OUT_OF_MEMORY; } - AbilitySyncRequestPacket packet; - packet.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); - packet.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); - SchemaObject schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); - // 102 version is forbidden to sync with 103 json-schema or flatbuffer-schema - // so schema should put null string while remote is 102 version to avoid this bug. - if (remoteCommunicatorVersion == 1) { - packet.SetSchema(""); - packet.SetSchemaType(0); - } else { - packet.SetSchema(schemaObj.ToSchemaString()); - packet.SetSchemaType(static_cast(schemaObj.GetSchemaType())); - } - SecurityOption option; - GetPacketSecOption(option); - packet.SetSecLabel(option.securityLabel); - packet.SetSecFlag(option.securityFlag); - message->SetMessageType(TYPE_REQUEST); - int errCode = message->SetCopiedObject<>(packet); + errCode = message->SetCopiedObject<>(packet); if (errCode != E_OK) { LOGE("[AbilitySync][SyncStart] SetCopiedObject failed, err %d", errCode); delete message; @@ -322,8 +363,6 @@ int AbilitySync::SyncStart(uint32_t sessionId, uint32_t sequenceId, uint16_t rem message->SetVersion(MSG_VERSION_EXT); message->SetSessionId(sessionId); message->SetSequenceId(sequenceId); - LOGI("[AbilitySync][SyncStart] software version = %u, Label = %d, Flag = %d", SOFTWARE_VERSION_CURRENT, - option.securityLabel, option.securityFlag); errCode = communicator_->SendMessage(deviceId_, message, false, SEND_TIME_OUT, handler); if (errCode != E_OK) { LOGE("[AbilitySync][SyncStart] SendPacket failed, err %d", errCode); @@ -335,22 +374,17 @@ int AbilitySync::SyncStart(uint32_t sessionId, uint32_t sequenceId, uint16_t rem int AbilitySync::AckRecv(const Message *message, ISyncTaskContext *context) { - if (message == nullptr || context == nullptr) { - return -E_INVALID_ARGS; + int errCode = AckMsgCheck(message, context); + if (errCode != E_OK) { + return errCode; } const AbilitySyncAckPacket *packet = message->GetObject(); if (packet == nullptr) { return -E_INVALID_ARGS; } - int errCode = CheckAckCode(message, context, packet->GetAckCode()); - if (errCode != E_OK) { - return errCode; - } uint32_t remoteSoftwareVersion = packet->GetSoftwareVersion(); context->SetRemoteSoftwareVersion(remoteSoftwareVersion); - std::string schema; - packet->GetSchema(schema); - SingleVerKvDBSyncInterface *storage = static_cast(storageInterface_); + std::string schema = packet->GetSchema(); if (remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_2_0) { HandleVersionV3AckSecOptionParam(packet, context); SyncOpinion localSyncOpinion = HandleVersionV3AckSchemaParam(packet, schema, context); @@ -360,10 +394,22 @@ int AbilitySync::AckRecv(const Message *message, ISyncTaskContext *context) LOGE("[AbilitySync][AckRecv] scheme check failed"); return -E_SCHEMA_MISMATCH; } - (void)SendAck(message, storage->GetSchemaInfo(), AbilitySync::CHECK_SUCCESS, localSyncOpinion, true); + if (remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_3_0) { + errCode = metadata_->SetDbCreateTime(deviceId_, packet->GetDbCreateTime(), true); + if (errCode != E_OK) { + LOGE("[AbilitySync][AckRecv] set db create time failed,errCode=%d", errCode); + return errCode; + } + } + DbAbility remoteDbAbility = packet->GetDbAbility(); + (static_cast(context))->SetDbAbility(remoteDbAbility); + SendAck(message, localSyncOpinion, AbilitySync::CHECK_SUCCESS, true); (static_cast(context))->SetIsSchemaSync(true); } else { - bool isCompatible = storage->CheckCompatible(schema); + bool isCompatible = true; + if (IsSingleKvVer()) { + isCompatible = static_cast(storageInterface_)->CheckCompatible(schema); + } if (!isCompatible) { (static_cast(context))->SetTaskErrCode(-E_SCHEMA_MISMATCH); LOGE("[AbilitySync][AckRecv] scheme check failed"); @@ -385,16 +431,17 @@ int AbilitySync::RequestRecv(const Message *message, ISyncTaskContext *context) return -E_INVALID_ARGS; } SyncOpinion localSyncOpinion; - SingleVerKvDBSyncInterface *storage = static_cast(storageInterface_); if (packet->GetSendCode() == -E_VERSION_NOT_SUPPORT) { - (void)SendAck(message, storage->GetSchemaInfo(), -E_VERSION_NOT_SUPPORT, localSyncOpinion, false); + SendAck(message, localSyncOpinion, -E_VERSION_NOT_SUPPORT, false); LOGI("[AbilitySync][RequestRecv] version can not support, remote version is %u", packet->GetProtocolVersion()); return -E_VERSION_NOT_SUPPORT; } - std::string schema; - packet->GetSchema(schema); - bool isCompatible = storage->CheckCompatible(schema); + std::string schema = packet->GetSchema(); + bool isCompatible = true; + if (IsSingleKvVer()) { + isCompatible = static_cast(storageInterface_)->CheckCompatible(schema); + } if (!isCompatible) { (static_cast(context))->SetTaskErrCode(-E_SCHEMA_MISMATCH); } @@ -413,9 +460,24 @@ int AbilitySync::RequestRecv(const Message *message, ISyncTaskContext *context) remoteSoftwareVersion, isCompatible); return SendAck(message, SchemaObject(), E_OK, localSyncOpinion, false); } - LOGI("[AbilitySync][RequestRecv] remote dev = %s{private}, version = %u, CheckSchemaCompatible = %d", - deviceId_.c_str(), remoteSoftwareVersion, isCompatible); - return SendAck(message, storage->GetSchemaInfo(), ackCode, localSyncOpinion, false); + if (ackCode == E_OK && remoteSoftwareVersion > SOFTWARE_VERSION_RELEASE_3_0) { + ackCode = metadata_->SetDbCreateTime(deviceId_, packet->GetDbCreateTime(), true); + } + LOGI("[AbilitySync][RequestRecv] remote dev=%s,ver=%u,schemaCompatible=%d", STR_MASK(deviceId_), + remoteSoftwareVersion, isCompatible); + return SendAck(message, localSyncOpinion, ackCode, false); +} + +int AbilitySync::SendAck(const Message *message, SyncOpinion &localSyncOpinion, int ackCode, bool isAckNotify) +{ +#ifdef RELATIONAL_STORE + if (IsSingleRelationalVer()) { + SchemaObject schemaObject; + return SendAck(message, schemaObject, ackCode, localSyncOpinion, isAckNotify); + } +#endif + SchemaObject schemaObject = static_cast(storageInterface_)->GetSchemaInfo(); + return SendAck(message, schemaObject, ackCode, localSyncOpinion, isAckNotify); } int AbilitySync::AckNotifyRecv(const Message *message, ISyncTaskContext *context) @@ -423,7 +485,6 @@ int AbilitySync::AckNotifyRecv(const Message *message, ISyncTaskContext *context if (message == nullptr || context == nullptr) { return -E_INVALID_ARGS; } - if (message->GetErrorNo() == E_FEEDBACK_UNKNOWN_MESSAGE) { LOGE("[AbilitySync][AckNotifyRecv] Remote device dose not support this message id"); context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_EARLIEST); @@ -438,13 +499,12 @@ int AbilitySync::AckNotifyRecv(const Message *message, ISyncTaskContext *context LOGE("[AbilitySync][AckNotifyRecv] received a errCode %d", errCode); return errCode; } - std::string schema; - packet->GetSchema(schema); + std::string schema = packet->GetSchema(); uint32_t remoteSoftwareVersion = packet->GetSoftwareVersion(); context->SetRemoteSoftwareVersion(remoteSoftwareVersion); SyncOpinion localSyncOpinion = HandleVersionV3AckSchemaParam(packet, schema, context); - LOGI("[AckNotifyRecv] receive dev = %s{private} ack notify, remoteSoftwareVersion = %u, ackCode = %d", - deviceId_.c_str(), remoteSoftwareVersion, errCode); + LOGI("[AckNotifyRecv] receive dev = %s ack notify, remoteSoftwareVersion = %u, ackCode = %d", + STR_MASK(deviceId_), remoteSoftwareVersion, errCode); (static_cast(context))->SetIsSchemaSync(true); (void)SendAck(message, SchemaObject(), AbilitySync::LAST_NOTIFY, localSyncOpinion, true); return E_OK; @@ -468,7 +528,7 @@ bool AbilitySync::SecLabelCheck(const AbilitySyncRequestPacket *packet) const return true; } SecurityOption option; - int errCode = (static_cast(storageInterface_))->GetSecurityOption(option); + int errCode = (static_cast(storageInterface_))->GetSecurityOption(option); LOGI("[AbilitySync][RequestRecv] local l:%d, f:%d, errCode:%d", option.securityLabel, option.securityFlag, errCode); if (errCode == -E_NOT_SUPPORT || option.securityLabel == SecurityLabel::NOT_SET) { return true; @@ -492,8 +552,15 @@ SyncOpinion AbilitySync::HandleVersionV3RequestParam(const AbilitySyncRequestPac int32_t remoteSecLabel = packet->GetSecLabel(); int32_t remoteSecFlag = packet->GetSecFlag(); SecurityOption secOption = {remoteSecLabel, remoteSecFlag}; + DbAbility remoteDbAbility = packet->GetDbAbility(); + (static_cast(context))->SetDbAbility(remoteDbAbility); (static_cast(context))->SetRemoteSeccurityOption(secOption); (static_cast(context))->SetReceivcPermitCheck(false); +#ifdef RELATIONAL_STORE + if (IsSingleRelationalVer()) { + return SyncOpinion{true, false, false}; + } +#endif uint8_t remoteSchemaType = packet->GetSchemaType(); SchemaObject localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); SyncOpinion localSyncOpinion = SchemaObject::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); @@ -520,6 +587,14 @@ SyncOpinion AbilitySync::HandleVersionV3AckSchemaParam(const AbilitySyncAckPacke bool requirePeerConvert = static_cast(packet->GetRequirePeerConvert()); SyncOpinion remoteOpinion = {permitSync, requirePeerConvert, true}; uint8_t remoteSchemaType = packet->GetSchemaType(); +#ifdef RELATIONAL_STORE + if (IsSingleRelationalVer()) { + auto localOpinion = SyncOpinion{true, false, false}; + SyncStrategy localStrategy = SchemaObject::ConcludeSyncStrategy(localOpinion, remoteOpinion); + (static_cast(context))->SetSyncStrategy(localStrategy); + return localOpinion; + } +#endif SchemaObject localSchema = (static_cast(storageInterface_))->GetSchemaInfo(); SyncOpinion localOpinion = SchemaObject::MakeLocalSyncOpinion(localSchema, remoteSchema, remoteSchemaType); SyncStrategy localStrategy = SchemaObject::ConcludeSyncStrategy(localOpinion, remoteOpinion); @@ -527,9 +602,13 @@ SyncOpinion AbilitySync::HandleVersionV3AckSchemaParam(const AbilitySyncAckPacke return localOpinion; } -void AbilitySync::GetPacketSecOption(SecurityOption &option) +void AbilitySync::GetPacketSecOption(SecurityOption &option) const { - int errCode = (static_cast(storageInterface_))->GetSecurityOption(option); + int errCode = -E_NOT_SUPPORT; + if (IsSingleKvVer()) { + errCode = + (static_cast(storageInterface_))->GetSecurityOption(option); + } if (errCode == -E_NOT_SUPPORT) { LOGE("[AbilitySync][SyncStart] GetSecOpt not surpport sec classification"); option.securityLabel = NOT_SURPPORT_SEC_CLASSIFICATION; @@ -619,30 +698,25 @@ int AbilitySync::DeSerialization(const uint8_t *buffer, uint32_t length, Message } } +#ifdef RELATIONAL_STORE +int AbilitySync::SendAck(const Message *inMsg, const ISchema &schemaObj, int ackCode, SyncOpinion localOpinion, + bool isAckNotify) +#else int AbilitySync::SendAck(const Message *inMsg, const SchemaObject &schemaObj, int ackCode, SyncOpinion localOpinion, bool isAckNotify) +#endif { + AbilitySyncAckPacket ackPacket; + int errCode = SetAbilityAckBodyInfo(ackPacket, schemaObj, ackCode, localOpinion, isAckNotify); + if (errCode != E_OK) { + return errCode; + } Message *ackMessage = new (std::nothrow) Message(ABILITY_SYNC_MESSAGE); if (ackMessage == nullptr) { LOGE("[AbilitySync][SendAck] message create failed, may be memleak!"); return -E_OUT_OF_MEMORY; } - - AbilitySyncAckPacket ackPacket; - ackPacket.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); - ackPacket.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); - ackPacket.SetSchema(schemaObj.ToSchemaString()); - ackPacket.SetSchemaType(static_cast(schemaObj.GetSchemaType())); - ackPacket.SetAckCode(ackCode); - if (!isAckNotify) { - SecurityOption option; - GetPacketSecOption(option); - ackPacket.SetSecLabel(option.securityLabel); - ackPacket.SetSecFlag(option.securityFlag); - } - ackPacket.SetPermitSync(localOpinion.permitSync); - ackPacket.SetRequirePeerConvert(localOpinion.requirePeerConvert); - int errCode = ackMessage->SetCopiedObject<>(ackPacket); + errCode = ackMessage->SetCopiedObject<>(ackPacket); if (errCode != E_OK) { LOGE("[AbilitySync][SendAck] SetCopiedObject failed, err %d", errCode); delete ackMessage; @@ -653,7 +727,6 @@ int AbilitySync::SendAck(const Message *inMsg, const SchemaObject &schemaObj, in ackMessage->SetTarget(deviceId_); ackMessage->SetSessionId(inMsg->GetSessionId()); ackMessage->SetSequenceId(inMsg->GetSequenceId()); - errCode = communicator_->SendMessage(deviceId_, ackMessage, false, SEND_TIME_OUT); if (errCode != E_OK) { LOGE("[AbilitySync][SendAck] SendPacket failed, err %d", errCode); @@ -693,35 +766,17 @@ int AbilitySync::RequestPacketSerialization(uint8_t *buffer, uint32_t length, co } Parcel parcel(buffer, length); - int errCode = parcel.WriteUInt32(packet->GetProtocolVersion()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSendCode()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetSoftwareVersion()); - if (errCode != E_OK) { - return errCode; - } - std::string schema; - packet->GetSchema(schema); - errCode = parcel.WriteString(schema); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSecLabel()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSecFlag()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetSchemaType()); - if (errCode != E_OK) { - return errCode; + parcel.WriteUInt32(packet->GetProtocolVersion()); + parcel.WriteInt(packet->GetSendCode()); + parcel.WriteUInt32(packet->GetSoftwareVersion()); + parcel.WriteString(packet->GetSchema()); + parcel.WriteInt(packet->GetSecLabel()); + parcel.WriteInt(packet->GetSecFlag()); + parcel.WriteUInt32(packet->GetSchemaType()); + parcel.WriteUInt64(packet->GetDbCreateTime()); + int errCode = DbAbility::Serialize(parcel, packet->GetDbAbility()); + if (parcel.IsError() || errCode != E_OK) { + return -E_PARSE_FAIL; } return E_OK; } @@ -734,43 +789,19 @@ int AbilitySync::AckPacketSerialization(uint8_t *buffer, uint32_t length, const } Parcel parcel(buffer, length); - int errCode = parcel.WriteUInt32(ABILITY_SYNC_VERSION_V1); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(SOFTWARE_VERSION_CURRENT); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetAckCode()); - if (errCode != E_OK) { - return errCode; - } - std::string schema; - packet->GetSchema(schema); - errCode = parcel.WriteString(schema); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSecLabel()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteInt(packet->GetSecFlag()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetSchemaType()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetPermitSync()); - if (errCode != E_OK) { - return errCode; - } - errCode = parcel.WriteUInt32(packet->GetRequirePeerConvert()); - if (errCode != E_OK) { - return errCode; + parcel.WriteUInt32(ABILITY_SYNC_VERSION_V1); + parcel.WriteUInt32(SOFTWARE_VERSION_CURRENT); + parcel.WriteInt(packet->GetAckCode()); + parcel.WriteString(packet->GetSchema()); + parcel.WriteInt(packet->GetSecLabel()); + parcel.WriteInt(packet->GetSecFlag()); + parcel.WriteUInt32(packet->GetSchemaType()); + parcel.WriteUInt32(packet->GetPermitSync()); + parcel.WriteUInt32(packet->GetRequirePeerConvert()); + parcel.WriteUInt64(packet->GetDbCreateTime()); + int errCode = DbAbility::Serialize(parcel, packet->GetDbAbility()); + if (parcel.IsError() || errCode != E_OK) { + return -E_PARSE_FAIL; } return E_OK; } @@ -805,10 +836,8 @@ int AbilitySync::RequestPacketDeSerialization(const uint8_t *buffer, uint32_t le parcel.ReadInt(sendCode); parcel.ReadUInt32(softwareVersion); parcel.ReadString(schema); - if (!parcel.IsError() && softwareVersion > SOFTWARE_VERSION_RELEASE_2_0) { - RequestPacketDeSerializationTailPart(parcel, packet); - } - if (parcel.IsError()) { + errCode = RequestPacketDeSerializationTailPart(parcel, packet, softwareVersion); + if (parcel.IsError() || errCode != E_OK) { goto ERROR_OUT; } packet->SetSendCode(sendCode); @@ -826,36 +855,67 @@ ERROR_OUT: return errCode; } -void AbilitySync::RequestPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncRequestPacket *packet) +int AbilitySync::RequestPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncRequestPacket *packet, + uint32_t version) { - int32_t secLabel = 0; - int32_t secFlag = 0; - uint32_t schemaType = 0; - parcel.ReadInt(secLabel); - parcel.ReadInt(secFlag); - parcel.ReadUInt32(schemaType); - packet->SetSecLabel(secLabel); - packet->SetSecFlag(secFlag); - packet->SetSchemaType(schemaType); + if (!parcel.IsError() && version > SOFTWARE_VERSION_RELEASE_2_0) { + int32_t secLabel = 0; + int32_t secFlag = 0; + uint32_t schemaType = 0; + parcel.ReadInt(secLabel); + parcel.ReadInt(secFlag); + parcel.ReadUInt32(schemaType); + packet->SetSecLabel(secLabel); + packet->SetSecFlag(secFlag); + packet->SetSchemaType(schemaType); + } + if (!parcel.IsError() && version > SOFTWARE_VERSION_RELEASE_3_0) { + uint64_t dbCreateTime = 0; + parcel.ReadUInt64(dbCreateTime); + packet->SetDbCreateTime(dbCreateTime); + } + DbAbility remoteDbAbility; + int errCode = DbAbility::DeSerialize(parcel, remoteDbAbility); + if (errCode != E_OK) { + LOGE("[AbilitySync] request packet DeSerializ failed."); + return errCode; + } + packet->SetDbAbility(remoteDbAbility); + return E_OK; } -void AbilitySync::AckPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncAckPacket *packet) -{ - int32_t secLabel = 0; - int32_t secFlag = 0; - uint32_t schemaType = 0; - uint32_t permitSync = 0; - uint32_t requirePeerConvert = 0; - parcel.ReadInt(secLabel); - parcel.ReadInt(secFlag); - parcel.ReadUInt32(schemaType); - parcel.ReadUInt32(permitSync); - parcel.ReadUInt32(requirePeerConvert); - packet->SetSecLabel(secLabel); - packet->SetSecFlag(secFlag); - packet->SetSchemaType(schemaType); - packet->SetPermitSync(permitSync); - packet->SetRequirePeerConvert(requirePeerConvert); +int AbilitySync::AckPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncAckPacket *packet, uint32_t version) +{ + if (!parcel.IsError() && version > SOFTWARE_VERSION_RELEASE_2_0) { + int32_t secLabel = 0; + int32_t secFlag = 0; + uint32_t schemaType = 0; + uint32_t permitSync = 0; + uint32_t requirePeerConvert = 0; + parcel.ReadInt(secLabel); + parcel.ReadInt(secFlag); + parcel.ReadUInt32(schemaType); + parcel.ReadUInt32(permitSync); + parcel.ReadUInt32(requirePeerConvert); + packet->SetSecLabel(secLabel); + packet->SetSecFlag(secFlag); + packet->SetSchemaType(schemaType); + packet->SetPermitSync(permitSync); + packet->SetRequirePeerConvert(requirePeerConvert); + } + if (!parcel.IsError() && version > SOFTWARE_VERSION_RELEASE_3_0) { + uint64_t dbCreateTime = 0; + parcel.ReadUInt64(dbCreateTime); + packet->SetDbCreateTime(dbCreateTime); + } + DbAbility remoteDbAbility; + int errCode = DbAbility::DeSerialize(parcel, remoteDbAbility); + if (errCode != E_OK) { + LOGE("[AbilitySync] ack packet DeSerializ failed."); + return errCode; + } + packet->SetDbAbility(remoteDbAbility); + return E_OK; } int AbilitySync::AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) @@ -889,10 +949,8 @@ int AbilitySync::AckPacketDeSerialization(const uint8_t *buffer, uint32_t length parcel.ReadUInt32(softwareVersion); parcel.ReadInt(ackCode); parcel.ReadString(schema); - if (!parcel.IsError() && softwareVersion > SOFTWARE_VERSION_RELEASE_2_0) { - AckPacketDeSerializationTailPart(parcel, packet); - } - if (parcel.IsError()) { + errCode = AckPacketDeSerializationTailPart(parcel, packet, softwareVersion); + if (parcel.IsError() || errCode != E_OK) { LOGE("[AbilitySync][RequestDeSerialization] DeSerialization failed!"); errCode = -E_PARSE_FAIL; goto ERROR_OUT; @@ -911,21 +969,149 @@ ERROR_OUT: return errCode; } -int AbilitySync::CheckAckCode(const Message *message, ISyncTaskContext *context, int errCode) +int AbilitySync::SetAbilityRequestBodyInfo(AbilitySyncRequestPacket &packet, uint16_t remoteCommunicatorVersion) const { + uint64_t dbCreateTime; + int errCode = + (static_cast(storageInterface_))->GetDatabaseCreateTimeStamp(dbCreateTime); + if (errCode != E_OK) { + LOGE("[AbilitySync][FillAbilityRequest] GetDatabaseCreateTimeStamp failed, err %d", errCode); + return errCode; + } + SecurityOption option; + GetPacketSecOption(option); + std::string schemaStr; + uint32_t schemaType; + if (IsSingleKvVer()) { + SchemaObject schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); + schemaStr = schemaObj.ToSchemaString(); + schemaType = static_cast(schemaObj.GetSchemaType()); + } +#ifdef RELATIONAL_STORE + if (IsSingleRelationalVer()) { + // todo demo + auto schemaObj = (static_cast(storageInterface_))->GetSchemaInfo(); + schemaStr = schemaObj.ToSchemaString(); + schemaType = 0; + } +#endif + DbAbility dbAbility; + errCode = GetDbAbilityInfo(dbAbility); + if (errCode != E_OK) { + LOGE("[AbilitySync][FillAbilityRequest] GetDbAbility failed, err %d", errCode); + return errCode; + } + // 102 version is forbidden to sync with 103 json-schema or flatbuffer-schema + // so schema should put null string while remote is 102 version to avoid this bug. + if (remoteCommunicatorVersion == 1) { + packet.SetSchema(""); + packet.SetSchemaType(0); + } else { + packet.SetSchema(schemaStr); + packet.SetSchemaType(schemaType); + } + packet.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); + packet.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); + packet.SetSecLabel(option.securityLabel); + packet.SetSecFlag(option.securityFlag); + packet.SetDbCreateTime(dbCreateTime); + packet.SetDbAbility(dbAbility); + LOGI("[AbilitySync][FillRequest] ver=%u,Lab=%d,Flag=%d,dbCreateTime=%llu", SOFTWARE_VERSION_CURRENT, + option.securityLabel, option.securityFlag, dbCreateTime); + return E_OK; +} + +#ifdef RELATIONAL_STORE +int AbilitySync::SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const ISchema &schemaObj, int ackCode, + SyncOpinion localOpinion, bool isAckNotify) +#else +int AbilitySync::SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const SchemaObject &schemaObj, int ackCode, + SyncOpinion localOpinion, bool isAckNotify) +#endif +{ + int errCode = E_OK; + ackPacket.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); + ackPacket.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); + ackPacket.SetSchema(schemaObj.ToSchemaString()); + ackPacket.SetSchemaType(static_cast(schemaObj.GetSchemaType())); + if (!isAckNotify) { + SecurityOption option; + GetPacketSecOption(option); + ackPacket.SetSecLabel(option.securityLabel); + ackPacket.SetSecFlag(option.securityFlag); + uint64_t dbCreateTime = 0; + errCode = + (static_cast(storageInterface_))->GetDatabaseCreateTimeStamp(dbCreateTime); + if (errCode != E_OK) { + LOGE("[AbilitySync][SyncStart] GetDatabaseCreateTimeStamp failed, err %d", errCode); + ackCode = errCode; + } + DbAbility dbAbility; + errCode = GetDbAbilityInfo(dbAbility); + if (errCode != E_OK) { + LOGE("[AbilitySync][FillAbilityRequest] GetDbAbility failed, err %d", errCode); + return errCode; + } + ackPacket.SetDbCreateTime(dbCreateTime); + ackPacket.SetDbAbility(dbAbility); + } + ackPacket.SetAckCode(ackCode); + ackPacket.SetPermitSync(localOpinion.permitSync); + ackPacket.SetRequirePeerConvert(localOpinion.requirePeerConvert); + return E_OK; +} + +int AbilitySync::GetDbAbilityInfo(DbAbility &dbAbility) const +{ + int errCode = E_OK; + for (const auto &item : ABILITYBITS) { + errCode = dbAbility.SetAbilityItem(item, SUPPORT_MARK); + if (errCode != E_OK) { + return errCode; + } + } + return errCode; +} + +int AbilitySync::AckMsgCheck(const Message *message, ISyncTaskContext *context) const +{ + if (message == nullptr || context == nullptr) { + return -E_INVALID_ARGS; + } if (message->GetErrorNo() == E_FEEDBACK_UNKNOWN_MESSAGE) { - LOGE("[AbilitySync][AckRecv] Remote device dose not support this message id"); + LOGE("[AbilitySync][AckMsgCheck] Remote device dose not support this message id"); context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_EARLIEST); + context->SetTaskErrCode(-E_FEEDBACK_UNKNOWN_MESSAGE); return -E_FEEDBACK_UNKNOWN_MESSAGE; } - - if (errCode != E_OK) { - LOGE("[AbilitySync][AckRecv] received a errCode %d", errCode); - if (errCode == -E_SECURITY_OPTION_CHECK_ERROR) { + if (message->GetErrorNo() == E_FEEDBACK_COMMUNICATOR_NOT_FOUND) { + LOGE("[AbilitySync][AckMsgCheck] Remote db is closed"); + context->SetTaskErrCode(-E_FEEDBACK_COMMUNICATOR_NOT_FOUND); + return -E_FEEDBACK_COMMUNICATOR_NOT_FOUND; + } + const AbilitySyncAckPacket *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + int ackCode = packet->GetAckCode(); + if (ackCode != E_OK) { + LOGE("[AbilitySync][AckMsgCheck] received a errCode %d", ackCode); + if (ackCode == -E_SECURITY_OPTION_CHECK_ERROR) { context->SetTaskErrCode(-E_SECURITY_OPTION_CHECK_ERROR); } - return errCode; + return ackCode; } return E_OK; } + +bool AbilitySync::IsSingleKvVer() const +{ + return storageInterface_->GetInterfaceType() == ISyncInterface::SYNC_SVD; +} +#ifdef RELATIONAL_STORE +bool AbilitySync::IsSingleRelationalVer() const +{ + return storageInterface_->GetInterfaceType() == ISyncInterface::SYNC_RELATION; +} +#endif } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h index 4304e185605aa36f1b36809224b7b49d2dff8909..a76f8dc62727f9d4ec5db76a30f9fdbe5474a312 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/ability_sync.h @@ -19,10 +19,14 @@ #include #include +#include "db_ability.h" #include "icommunicator.h" #include "ikvdb_sync_interface.h" #include "isync_task_context.h" #include "parcel.h" +#ifdef RELATIONAL_STORE +#include "schema.h" +#endif namespace DistributedDB { class AbilitySyncRequestPacket { @@ -40,7 +44,7 @@ public: uint32_t GetSoftwareVersion() const; void SetSchema(const std::string &schema); - void GetSchema(std::string &schema) const; + std::string GetSchema() const; void SetSchemaType(uint32_t schemaType); uint32_t GetSchemaType() const; @@ -51,8 +55,15 @@ public: void SetSecFlag(int32_t secFlag); int32_t GetSecFlag() const; + void SetDbCreateTime(uint64_t dbCreateTime); + uint64_t GetDbCreateTime() const; + uint32_t CalculateLen() const; + DbAbility GetDbAbility() const; + + void SetDbAbility(const DbAbility &dbAbility); + private: uint32_t protocolVersion_; int32_t sendCode_; @@ -61,6 +72,8 @@ private: int32_t secLabel_; int32_t secFlag_; uint32_t schemaType_; + uint64_t dbCreateTime_; + DbAbility dbAbility_; }; class AbilitySyncAckPacket { @@ -78,7 +91,7 @@ public: int32_t GetAckCode() const; void SetSchema(const std::string &schema); - void GetSchema(std::string &schema) const; + std::string GetSchema() const; void SetSchemaType(uint32_t schemaType); uint32_t GetSchemaType() const; @@ -95,8 +108,15 @@ public: void SetRequirePeerConvert(uint32_t requirePeerConvert); uint32_t GetRequirePeerConvert() const; + void SetDbCreateTime(uint64_t dbCreateTime); + uint64_t GetDbCreateTime() const; + uint32_t CalculateLen() const; + DbAbility GetDbAbility() const; + + void SetDbAbility(const DbAbility &dbAbility); + private: uint32_t protocolVersion_; uint32_t softwareVersion_; @@ -107,6 +127,8 @@ private: uint32_t schemaType_; uint32_t permitSync_; uint32_t requirePeerConvert_; + uint64_t dbCreateTime_; + DbAbility dbAbility_; }; class AbilitySync { @@ -120,7 +142,8 @@ public: int SyncStart(uint32_t sessionId, uint32_t sequenceId, uint16_t remoteCommunicatorVersion, const CommErrHandler &handler = nullptr); - int Initialize(ICommunicator *inCommunicator, IKvDBSyncInterface *inStorage, const std::string &deviceId); + int Initialize(ICommunicator *inCommunicator, ISyncInterface *inStorage, std::shared_ptr &inMetadata, + const std::string &deviceId); int AckRecv(const Message *message, ISyncTaskContext *context); @@ -141,8 +164,13 @@ public: static int DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); // register to communicator private: - int SendAck(const Message *inMsg, const SchemaObject &schemaObj, int ackCode, SyncOpinion localOpinion, +#ifdef RELATIONAL_STORE + int SendAck(const Message *inMsg, const ISchema &schemaObj, int ackCode, SyncOpinion localOpinion, bool isAckNotify = false); +#else + int SendAck(const Message *inMsg, const SchemaObject &schemaObj, int ackCode, SyncOpinion localOpinion, + bool isAckNotify = false); +#endif static int RequestPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); @@ -156,9 +184,10 @@ private: static int AckPacketCalculateLen(const Message *inMsg, uint32_t &len); - static void RequestPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncRequestPacket *packet); + static int RequestPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncRequestPacket *packet, + uint32_t version); - static void AckPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncAckPacket *packet); + static int AckPacketDeSerializationTailPart(Parcel &parcel, AbilitySyncAckPacket *packet, uint32_t version); bool SecLabelCheck(const AbilitySyncRequestPacket *packet) const; @@ -171,12 +200,31 @@ private: SyncOpinion HandleVersionV3AckSchemaParam(const AbilitySyncAckPacket *packet, const std::string &remoteSchema, ISyncTaskContext *context) const; - void GetPacketSecOption(SecurityOption &option); + void GetPacketSecOption(SecurityOption &option) const; + + int SetAbilityRequestBodyInfo(AbilitySyncRequestPacket &packet, uint16_t remoteCommunicatorVersion) const; + +#ifdef RELATIONAL_STORE + int SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const ISchema &schemaObj, int ackCode, + SyncOpinion localOpinion, bool isAckNotify); +#else + int SetAbilityAckBodyInfo(AbilitySyncAckPacket &ackPacket, const SchemaObject &schemaObj, int ackCode, + SyncOpinion localOpinion, bool isAckNotify); +#endif + + int GetDbAbilityInfo(DbAbility &dbAbility) const; + + int AckMsgCheck(const Message *message, ISyncTaskContext *context) const; - int CheckAckCode(const Message *message, ISyncTaskContext *context, int errCode); + bool IsSingleKvVer() const; +#ifdef RELATIONAL_STORE + bool IsSingleRelationalVer() const; +#endif + int SendAck(const Message *message, SyncOpinion &localSyncOpinion, int ackCode, bool isAckNotify); ICommunicator *communicator_; - IKvDBSyncInterface *storageInterface_; + ISyncInterface *storageInterface_; + std::shared_ptr metadata_; std::string deviceId_; bool syncFinished_; static const int FAILED_GET_SEC_CLASSIFICATION = 0x55; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/commit_history_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/commit_history_sync.h old mode 100755 new mode 100644 similarity index 100% rename from services/distributeddataservice/libs/distributeddb/syncer/include/commit_history_sync.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/commit_history_sync.h index 881f85dab2b07f09590f239d99da02828f5973e9..80ff06d84b446906909d2128d10412b079375f09 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/commit_history_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/commit_history_sync.h @@ -20,8 +20,8 @@ #include #include -#include "multi_ver_kvdb_sync_interface.h" #include "icommunicator.h" +#include "multi_ver_kvdb_sync_interface.h" #include "multi_ver_sync_task_context.h" #include "sync_types.h" #include "version.h" diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp new file mode 100644 index 0000000000000000000000000000000000000000..49e02f345df2467ff580971343bacfa17191cdd8 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.cpp @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "communicator_proxy.h" +#include "db_constant.h" +#include "log_print.h" + +namespace DistributedDB { +CommunicatorProxy::CommunicatorProxy() : mainComm_(nullptr) +{ +} + +CommunicatorProxy::~CommunicatorProxy() +{ + if (mainComm_ != nullptr) { + RefObject::DecObjRef(mainComm_); + } + mainComm_ = nullptr; + + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + RefObject::DecObjRef(devCommMap_[iter.first]); + } + devCommMap_.clear(); +} + +int CommunicatorProxy::RegOnMessageCallback(const OnMessageCallback &onMessage, const Finalizer &inOper) +{ + if (mainComm_ != nullptr) { + (void) mainComm_->RegOnMessageCallback(onMessage, inOper); + } + + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + (void) devCommMap_[iter.first]->RegOnMessageCallback(onMessage, inOper); + } + return E_OK; +} + +int CommunicatorProxy::RegOnConnectCallback(const OnConnectCallback &onConnect, const Finalizer &inOper) +{ + if (mainComm_ != nullptr) { + (void) mainComm_->RegOnConnectCallback(onConnect, inOper); + } + + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + (void) devCommMap_[iter.first]->RegOnConnectCallback(onConnect, inOper); + } + + return E_OK; +} + +int CommunicatorProxy::RegOnSendableCallback(const std::function &onSendable, const Finalizer &inOper) +{ + if (mainComm_ != nullptr) { + (void) mainComm_->RegOnSendableCallback(onSendable, inOper); + } + + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + (void) devCommMap_[iter.first]->RegOnSendableCallback(onSendable, inOper); + } + + return E_OK; +} + +void CommunicatorProxy::Activate() +{ + if (mainComm_ != nullptr) { + mainComm_->Activate(); + } + + // use temp map to avoid active in lock + std::map tempMap; + { + std::lock_guard lock(devCommMapLock_); + for (const auto &iter : devCommMap_) { + tempMap[iter.first] = devCommMap_[iter.first]; + RefObject::IncObjRef(devCommMap_[iter.first]); + } + } + + for (const auto &iter : tempMap) { + tempMap[iter.first]->Activate(); + RefObject::DecObjRef(tempMap[iter.first]); + } +} + +uint32_t CommunicatorProxy::GetCommunicatorMtuSize() const +{ + if (mainComm_ == nullptr) { + return DBConstant::MIN_MTU_SIZE; + } + return mainComm_->GetCommunicatorMtuSize(); +} + +uint32_t CommunicatorProxy::GetCommunicatorMtuSize(const std::string &target) const +{ + ICommunicator *targetCommunicator = nullptr; + { + std::lock_guard lock(devCommMapLock_); + if (devCommMap_.count(target) != 0) { + targetCommunicator = devCommMap_.at(target); + RefObject::IncObjRef(targetCommunicator); + } + } + if (targetCommunicator != nullptr) { + uint32_t mtuSize = targetCommunicator->GetCommunicatorMtuSize(target); + RefObject::DecObjRef(targetCommunicator); + return mtuSize; + } + + if (mainComm_ != nullptr) { + return mainComm_->GetCommunicatorMtuSize(target); + } + + return DBConstant::MIN_MTU_SIZE; +} + +uint32_t CommunicatorProxy::GetTimeout() const +{ + if (mainComm_ == nullptr) { + return DBConstant::MIN_TIMEOUT; + } + return mainComm_->GetTimeout(); +} + +uint32_t CommunicatorProxy::GetTimeout(const std::string &target) const +{ + ICommunicator *targetCommunicator = nullptr; + { + std::lock_guard lock(devCommMapLock_); + if (devCommMap_.count(target) != 0) { + targetCommunicator = devCommMap_.at(target); + RefObject::IncObjRef(targetCommunicator); + } + } + if (targetCommunicator != nullptr) { + uint32_t timeout = targetCommunicator->GetTimeout(target); + RefObject::DecObjRef(targetCommunicator); + return timeout; + } + + if (mainComm_ != nullptr) { + return mainComm_->GetTimeout(target); + } + + return DBConstant::MIN_TIMEOUT; +} + +bool CommunicatorProxy::IsDeviceOnline(const std::string &device) const +{ + return mainComm_->IsDeviceOnline(device); +} + +int CommunicatorProxy::GetLocalIdentity(std::string &outTarget) const +{ + return mainComm_->GetLocalIdentity(outTarget); +} + +int CommunicatorProxy::GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const +{ + ICommunicator *targetCommunicator = nullptr; + { + std::lock_guard lock(devCommMapLock_); + if (devCommMap_.count(target) != 0) { + targetCommunicator = devCommMap_.at(target); + RefObject::IncObjRef(targetCommunicator); + } + } + if (targetCommunicator != nullptr) { + int errCode = targetCommunicator->GetRemoteCommunicatorVersion(target, outVersion); + RefObject::DecObjRef(targetCommunicator); + return errCode; + } + + if (mainComm_ != nullptr) { + return mainComm_->GetRemoteCommunicatorVersion(target, outVersion); + } + + return -E_NOT_INIT; +} + +int CommunicatorProxy::SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout) +{ + return SendMessage(dstTarget, inMsg, nonBlock, timeout, nullptr); +} + +int CommunicatorProxy::SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout, + const OnSendEnd &onEnd) +{ + ICommunicator *targetCommunicator = nullptr; + { + std::lock_guard lock(devCommMapLock_); + if (devCommMap_.count(dstTarget) != 0) { + targetCommunicator = devCommMap_[dstTarget]; + RefObject::IncObjRef(targetCommunicator); + } + } + if (targetCommunicator != nullptr) { + LOGI("[CommProxy] use equal label to send data"); + int errCode = targetCommunicator->SendMessage(dstTarget, inMsg, nonBlock, timeout, onEnd); + RefObject::DecObjRef(targetCommunicator); + return errCode; + } + + if (mainComm_ != nullptr) { + return mainComm_->SendMessage(dstTarget, inMsg, nonBlock, timeout, onEnd); + } + + return -E_NOT_INIT; +} + +void CommunicatorProxy::SetMainCommunicator(ICommunicator *communicator) +{ + mainComm_ = communicator; + RefObject::IncObjRef(mainComm_); +} + +void CommunicatorProxy::SetEqualCommunicator(ICommunicator *communicator, const std::vector &targets) +{ + std::lock_guard lock(devCommMapLock_); + // Clear offline target + for (auto dev = devCommMap_.begin(); dev != devCommMap_.end();) { + auto iter = std::find_if(targets.begin(), targets.end(), + [&dev](const std::string &target) { + return target == dev->first; + }); + if (iter == targets.end()) { + RefObject::DecObjRef(devCommMap_[dev->first]); + dev = devCommMap_.erase(dev); + continue; + } + dev++; + } + + // Add new online target + for (const auto &target : targets) { + if (devCommMap_.count(target) != 0) { + // change the identifier and dev relation + RefObject::DecObjRef(devCommMap_[target]); + } + RefObject::IncObjRef(communicator); + devCommMap_[target] = communicator; + } +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h new file mode 100644 index 0000000000000000000000000000000000000000..0f54e8e31649326480a42bb468abaa494b816a70 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/communicator_proxy.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COMMUNICATOR_PROXY_H +#define COMMUNICATOR_PROXY_H + +#include +#include +#include +#include +#include +#include +#include "icommunicator.h" +#include "message.h" + +namespace DistributedDB { +class CommunicatorProxy : public ICommunicator { +public: + CommunicatorProxy(); + ~CommunicatorProxy(); + + int RegOnMessageCallback(const OnMessageCallback &onMessage, const Finalizer &inOper) override; + int RegOnConnectCallback(const OnConnectCallback &onConnect, const Finalizer &inOper) override; + int RegOnSendableCallback(const std::function &onSendable, const Finalizer &inOper) override; + void Activate() override; + uint32_t GetCommunicatorMtuSize() const override; + uint32_t GetCommunicatorMtuSize(const std::string &target) const override; + uint32_t GetTimeout() const override; + uint32_t GetTimeout(const std::string &target) const override; + bool IsDeviceOnline(const std::string &device) const override; + int GetLocalIdentity(std::string &outTarget) const override; + int GetRemoteCommunicatorVersion(const std::string &target, uint16_t &outVersion) const override; + int SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout) override; + int SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout, + const OnSendEnd &onEnd) override; + + // Set an Main communicator for this database, used userid & appId & storeId + void SetMainCommunicator(ICommunicator *communicator); + + // Set an equal communicator for this database, After this called, send msg to the target will use this communicator + void SetEqualCommunicator(ICommunicator *communicator, const std::vector &targets); + +private: + ICommunicator *mainComm_; + mutable std::mutex devCommMapLock_; + std::map devCommMap_; +}; +} // namespace DistributedDB +#endif // COMMUNICATOR_PROXY_H diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/device_manager.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/device_manager.cpp index ea9be0c485746307a7a3d3ceca0c93391287e5e2..989b8177a77c23ccab3d4d167e6ae42abe1cf3bc 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/device_manager.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/device_manager.cpp @@ -54,15 +54,16 @@ int DeviceManager::RegisterTransformFunc() } // Initialize the DeviceManager -int DeviceManager::Initialize(ICommunicator *communicator, const std::function &callback) +int DeviceManager::Initialize(ICommunicator *communicator, const std::function &onlineCallback, + const std::function &offlineCallback) { if (communicator == nullptr) { return -E_INVALID_ARGS; } RefObject::IncObjRef(communicator); communicator_ = communicator; - RegDeviceOnLineCallBack(callback); - + RegDeviceOnLineCallBack(onlineCallback); + RegDeviceOffLineCallBack(offlineCallback); return E_OK; } @@ -151,7 +152,7 @@ int DeviceManager::SendLocalDataChanged() bool DeviceManager::IsDeviceOnline(const std::string &deviceId) const { std::lock_guard lock(devicesLock_); - auto iter = std::find(devices_.begin(), devices_.end(), deviceId); + auto iter = std::find(devices_.begin(), devices_.end(), deviceId); return (iter != devices_.end()); } } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/device_manager.h b/services/distributeddataservice/libs/distributeddb/syncer/src/device_manager.h similarity index 95% rename from services/distributeddataservice/libs/distributeddb/syncer/include/device_manager.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/device_manager.h index 82ad16481229a0f84a30276c1c2b97051667858d..5b638b1e20dbd1e333559ad8e0d180adb6cae33e 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/device_manager.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/device_manager.h @@ -35,7 +35,8 @@ public: static uint32_t CalculateLen(); // Initialize the DeviceManager. - int Initialize(ICommunicator *communicator, const std::function &callback); + int Initialize(ICommunicator *communicator, const std::function &onlineCallback, + const std::function &offlineCallback); // Set The Device online Callback. void RegDeviceOnLineCallBack(const std::function &callback); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp index 1c933f24dd89470a56446743c8920728f9a1860c..cf78835c376ad53eddef0eb358811f86ce99ecbe 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.cpp @@ -15,6 +15,7 @@ #include "generic_syncer.h" +#include "db_common.h" #include "db_errno.h" #include "log_print.h" #include "ref_object.h" @@ -29,6 +30,7 @@ #include "device_manager.h" #include "db_constant.h" #include "ability_sync.h" +#include "single_ver_serialize_manager.h" namespace DistributedDB { const int GenericSyncer::MIN_VALID_SYNC_ID = 1; @@ -45,7 +47,6 @@ GenericSyncer::GenericSyncer() queuedManualSyncLimit_(DBConstant::QUEUED_SYNC_LIMIT_DEFAULT), manualSyncEnable_(true), closing_(false) - { } @@ -53,6 +54,7 @@ GenericSyncer::~GenericSyncer() { LOGD("[GenericSyncer] ~GenericSyncer!"); if (syncEngine_ != nullptr) { + syncEngine_->OnKill([this]() { this->syncEngine_->Close(); }); RefObject::KillAndDecObjRef(syncEngine_); syncEngine_ = nullptr; } @@ -61,7 +63,7 @@ GenericSyncer::~GenericSyncer() syncInterface_ = nullptr; } -int GenericSyncer::Initialize(IKvDBSyncInterface *syncInterface) +int GenericSyncer::Initialize(ISyncInterface *syncInterface) { if (syncInterface == nullptr) { LOGE("[Syncer] Init failed, the syncInterface is null!"); @@ -130,11 +132,9 @@ int GenericSyncer::Close() } ClearSyncOperations(); if (syncEngine_ != nullptr) { - syncEngine_->OnKill([this]() { this->syncEngine_->Close(); }); + syncEngine_->Close(); LOGD("[Syncer] Close SyncEngine!"); - RefObject::KillAndDecObjRef(syncEngine_); std::lock_guard lock(syncerLock_); - syncEngine_ = nullptr; closing_ = false; } timeHelper_ = nullptr; @@ -146,51 +146,76 @@ int GenericSyncer::Sync(const std::vector &devices, int mode, const std::function &)> &onComplete, const std::function &onFinalize, bool wait = false) { - std::lock_guard lock(syncerLock_); - if (!initialized_) { - LOGE("[Syncer] Syncer is not initialized, return!"); - return -E_NOT_INIT; - } - if (closing_) { - LOGE("[Syncer] Syncer is closing, return!"); - return -E_BUSY; - } - if (!IsValidDevices(devices) || !IsValidMode(mode)) { - return -E_INVALID_ARGS; - } - if (IsQueuedManualSyncFull(mode, wait)) { - LOGE("[Syncer] -E_BUSY"); - return -E_BUSY; + SyncParma param; + param.devices = devices; + param.mode = mode; + param.onComplete = onComplete; + param.onFinalize = onFinalize; + param.wait = wait; + return Sync(param); +} + +int GenericSyncer::Sync(const InternalSyncParma ¶m) +{ + SyncParma syncParam; + syncParam.devices = param.devices; + syncParam.mode = param.mode; + syncParam.isQuerySync = param.isQuerySync; + syncParam.syncQuery = param.syncQuery; + return Sync(syncParam); +} + +int GenericSyncer::Sync(const SyncParma ¶m) +{ + int errCode = SyncParamCheck(param); + if (errCode != E_OK) { + return errCode; } - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); + errCode = AddQueuedManualSyncSize(param.mode, param.wait); + if (errCode != E_OK) { + return errCode; } + uint32_t syncId = GenerateSyncId(); - LOGI("[Syncer] GenerateSyncId %d, mode = %d, wait = %d , label = %s, devicesNum = %d", syncId, mode, wait, - label_.c_str(), devices.size()); - SyncOperation *operation = new (std::nothrow) SyncOperation(syncId, devices, mode, onComplete, wait); + errCode = PrepareSync(param, syncId); + if (errCode != E_OK) { + LOGE("[Syncer] PrepareSync failed when sync called, err %d", errCode); + return errCode; + } + PerformanceAnalysis::GetInstance()->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); + return E_OK; +} + +int GenericSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) +{ + auto *operation = + new (std::nothrow) SyncOperation(syncId, param.devices, param.mode, param.onComplete, param.wait); if (operation == nullptr) { - LOGE("[Syncer] SyncOperation alloc failed when sync called, may be out of memory"); + SubQueuedSyncSize(); return -E_OUT_OF_MEMORY; } - operation->Initialize(); - operation->OnKill(std::bind(&GenericSyncer::SyncOperationKillCallback, this, operation->GetSyncId())); - std::function onFinished = std::bind(&GenericSyncer::OnSyncFinished, this, std::placeholders::_1); - operation->SetOnSyncFinished(onFinished); - operation->SetOnSyncFinalize(onFinalize); - int errCode = AddSyncOperation(operation); - if (errCode != E_OK) { - LOGE("[Syncer] AddSyncOperation failed when sync called, err %d", errCode); - RefObject::KillAndDecObjRef(operation); - return errCode; + operation->SetIdentifier(syncInterface_->GetIdentifier()); + { + std::lock_guard autoLock(syncerLock_); + PerformanceAnalysis::GetInstance()->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); + InitSyncOperation(operation, param); + LOGI("[Syncer] GenerateSyncId %d, mode = %d, wait = %d , label = %s, devices = %s", syncId, param.mode, + param.wait, label_.c_str(), GetSyncDevicesStr(param.devices).c_str()); + AddSyncOperation(operation); + PerformanceAnalysis::GetInstance()->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); + } + if (!param.wait) { + std::lock_guard lockGuard(syncIdLock_); + syncIdList_.push_back((int)syncId); } - AddQueuedManualSyncSize(mode, wait); - LOGD("[Syncer] AddSyncOperation end"); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SYNC_TOTAL); + if (operation->CheckIsAllFinished()) { + operation->Finished(); + RefObject::KillAndDecObjRef(operation); + } else { + operation->WaitIfNeed(); + RefObject::DecObjRef(operation); } - return syncId; + return E_OK; } int GenericSyncer::RemoveSyncOperation(int syncId) @@ -203,17 +228,32 @@ int GenericSyncer::RemoveSyncOperation(int syncId) operation = iter->second; syncOperationMap_.erase(syncId); lock.unlock(); - if ((!operation->IsAutoSync()) && (!operation->IsBlockSync())) { + if ((!operation->IsAutoSync()) && (!operation->IsBlockSync()) && (!operation->IsAutoControlCmd())) { SubQueuedSyncSize(); } operation->NotifyIfNeed(); RefObject::KillAndDecObjRef(operation); operation = nullptr; + std::lock_guard lockGuard(syncIdLock_); + syncIdList_.remove(syncId); return E_OK; } return -E_INVALID_ARGS; } +int GenericSyncer::StopSync() +{ + std::list syncIdList; + { + std::lock_guard lockGuard(syncIdLock_); + syncIdList = syncIdList_; + } + for (const auto &syncId : syncIdList) { + RemoveSyncOperation(syncId); + } + return E_OK; +} + uint64_t GenericSyncer::GetTimeStamp() { if (timeHelper_ == nullptr) { @@ -222,56 +262,27 @@ uint64_t GenericSyncer::GetTimeStamp() return timeHelper_->GetTime(); } -int GenericSyncer::AddSyncOperation(SyncOperation *operation) +void GenericSyncer::QueryAutoSync(const InternalSyncParma ¶m) +{ +} + +void GenericSyncer::AddSyncOperation(SyncOperation *operation) { - LOGD("[Syncer] AddSyncOperation."); if (operation == nullptr) { - return -E_INVALID_ARGS; + return; } - int errCode = syncEngine_->AddSyncOperation(operation); - if (errCode != E_OK) { - return errCode; - } + LOGD("[Syncer] AddSyncOperation."); + syncEngine_->AddSyncOperation(operation); if (operation->CheckIsAllFinished()) { - if (operation->IsBlockSync()) { - operation->Finished(); - RefObject::KillAndDecObjRef(operation); - return errCode; - } - RefObject::IncObjRef(operation); - RefObject::IncObjRef(syncEngine_); - ISyncEngine *syncEngine = syncEngine_; - errCode = RuntimeContext::GetInstance()->ScheduleTask([operation, syncEngine, this] { - std::lock_guard lock(syncerLock_); - if (closing_) { - LOGI("[Syncer] Syncer is closing, return!"); - RefObject::DecObjRef(operation); - RefObject::DecObjRef(syncEngine); - return; - } - operation->Finished(); - RefObject::KillAndDecObjRef(operation); - RefObject::DecObjRef(operation); - RefObject::DecObjRef(syncEngine); - }); - if (errCode != E_OK) { - LOGE("[Syncer] AddSyncOperation start finish task errCode:%d", errCode); - RefObject::DecObjRef(operation); - RefObject::DecObjRef(syncEngine_); - } - return errCode; - } - { - std::lock_guard lock(operationMapLock_); - syncOperationMap_.insert(std::pair(operation->GetSyncId(), operation)); - // To make sure operation alive before WaitIfNeed out - RefObject::IncObjRef(operation); + return; } - operation->WaitIfNeed(); - RefObject::DecObjRef(operation); - return errCode; + + std::lock_guard lock(operationMapLock_); + syncOperationMap_.insert(std::pair(operation->GetSyncId(), operation)); + // To make sure operation alive before WaitIfNeed out + RefObject::IncObjRef(operation); } void GenericSyncer::SyncOperationKillCallbackInner(int syncId) @@ -287,7 +298,7 @@ void GenericSyncer::SyncOperationKillCallback(int syncId) SyncOperationKillCallbackInner(syncId); } -int GenericSyncer::InitMetaData(IKvDBSyncInterface *syncInterface) +int GenericSyncer::InitMetaData(ISyncInterface *syncInterface) { if (metadata_ != nullptr) { return E_OK; @@ -302,7 +313,7 @@ int GenericSyncer::InitMetaData(IKvDBSyncInterface *syncInterface) return errCode; } -int GenericSyncer::InitTimeHelper(IKvDBSyncInterface *syncInterface) +int GenericSyncer::InitTimeHelper(ISyncInterface *syncInterface) { if (timeHelper_ != nullptr) { return E_OK; @@ -317,17 +328,27 @@ int GenericSyncer::InitTimeHelper(IKvDBSyncInterface *syncInterface) return errCode; } -int GenericSyncer::InitSyncEngine(IKvDBSyncInterface *syncInterface) +int GenericSyncer::InitSyncEngine(ISyncInterface *syncInterface) { - syncEngine_ = CreateSyncEngine(); + if (syncEngine_ != nullptr && syncEngine_->IsEngineActive()) { + LOGI("[Syncer] syncEngine is active"); + return E_OK; + } if (syncEngine_ == nullptr) { - return -E_OUT_OF_MEMORY; + syncEngine_ = CreateSyncEngine(); + if (syncEngine_ == nullptr) { + return -E_OUT_OF_MEMORY; + } } syncEngine_->OnLastRef([]() { LOGD("[Syncer] SyncEngine finalized"); }); - const std::function func = std::bind(&GenericSyncer::RemoteDataChanged, + const std::function onlineFunc = std::bind(&GenericSyncer::RemoteDataChanged, + this, std::placeholders::_1); + const std::function offlineFunc = std::bind(&GenericSyncer::RemoteDeviceOffline, this, std::placeholders::_1); - int errCode = syncEngine_->Initialize(syncInterface, metadata_, func); + const std::function queryAutoSyncFunc = + std::bind(&GenericSyncer::QueryAutoSync, this, std::placeholders::_1); + int errCode = syncEngine_->Initialize(syncInterface, metadata_, onlineFunc, offlineFunc, queryAutoSyncFunc); if (errCode == E_OK) { syncInterface_ = syncInterface; syncInterface->IncRefCount(); @@ -356,13 +377,19 @@ uint32_t GenericSyncer::GenerateSyncId() bool GenericSyncer::IsValidMode(int mode) const { - if ((mode > SyncOperation::AUTO_PULL) || (mode < SyncOperation::PUSH)) { + if ((mode >= SyncModeType::INVALID_MODE) || (mode < SyncModeType::PUSH)) { LOGE("[Syncer] Sync mode is not valid!"); return false; } return true; } +int GenericSyncer::SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, + const std::vector &devices) const +{ + return E_OK; +} + bool GenericSyncer::IsValidDevices(const std::vector &devices) const { if (devices.empty()) { @@ -384,6 +411,10 @@ void GenericSyncer::ClearSyncOperations() void GenericSyncer::OnSyncFinished(int syncId) { + { + std::lock_guard lockGuard(syncIdLock_); + syncIdList_.remove(syncId); + } (void)(RemoveSyncOperation(syncId)); } @@ -409,7 +440,7 @@ int GenericSyncer::SyncResourceInit() LOGE("Register timesync message transform func ERR!"); return errCode; } - errCode = SingleVerDataSync::RegisterTransformFunc(); + errCode = SingleVerSerializeManager::RegisterTransformFunc(); if (errCode != E_OK) { LOGE("Register SingleVerDataSync message transform func ERR!"); return errCode; @@ -477,33 +508,47 @@ int GenericSyncer::GetQueuedSyncLimit(int *queuedSyncLimit) const return E_OK; } -void GenericSyncer::AddQueuedManualSyncSize(int mode, bool wait) +bool GenericSyncer::IsManualSync(int inMode) const +{ + int mode = SyncOperation::TransferSyncMode(inMode); + if ((mode == SyncModeType::PULL) || (mode == SyncModeType::PUSH) || (mode == SyncModeType::PUSH_AND_PULL) || + (mode == SyncModeType::SUBSCRIBE_QUERY) || (mode == SyncModeType::UNSUBSCRIBE_QUERY)) { + return true; + } + return false; +} + +int GenericSyncer::AddQueuedManualSyncSize(int mode, bool wait) { - if (((mode == SyncOperation::PULL) || (mode == SyncOperation::PUSH) || (mode == SyncOperation::PUSH_AND_PULL)) && - (wait == false)) { + if (IsManualSync(mode) && (!wait)) { std::lock_guard lock(queuedManualSyncLock_); + if (!manualSyncEnable_) { + LOGI("[GenericSyncer] manualSyncEnable is Disable"); + return -E_BUSY; + } queuedManualSyncSize_++; } + return E_OK; } bool GenericSyncer::IsQueuedManualSyncFull(int mode, bool wait) const { std::lock_guard lock(queuedManualSyncLock_); - if (((mode == SyncOperation::PULL) || (mode == SyncOperation::PUSH) || (mode == SyncOperation::PUSH_AND_PULL)) && - (manualSyncEnable_ == false)) { + if (IsManualSync(mode) && (!manualSyncEnable_)) { LOGI("[GenericSyncer] manualSyncEnable_:false"); return true; } - if (((mode == SyncOperation::PULL) || (mode == SyncOperation::PUSH) || (mode == SyncOperation::PUSH_AND_PULL)) && - (wait == false)) { + if (IsManualSync(mode) && (!wait)) { if (queuedManualSyncSize_ < queuedManualSyncLimit_) { return false; + } else { + LOGD("[GenericSyncer] queuedManualSyncSize_:%d < queuedManualSyncLimit_:%d", queuedManualSyncSize_, + queuedManualSyncLimit_); + return true; } - LOGD("[GenericSyncer] queuedManualSyncSize_:%d < queuedManualSyncLimit_:%d", queuedManualSyncSize_, - queuedManualSyncLimit_); - return true; + } else { + return false; } - return false; } void GenericSyncer::SubQueuedSyncSize(void) @@ -562,9 +607,6 @@ void GenericSyncer::GetOnlineDevices(std::vector &devices) const return; } std::string identifier = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); - // If this database is configured as autoLaunch, then this database on other devices is probably autoLaunch as well. - // Since this database on other devices might have not been opened, we have to use physical online devices as list. - // If this database is not configured as autoLaunch, the out devices will be empty. RuntimeContext::GetInstance()->GetAutoLaunchSyncDevices(identifier, devices); if (!devices.empty()) { return; @@ -578,4 +620,73 @@ void GenericSyncer::GetOnlineDevices(std::vector &devices) const syncEngine_->GetOnlineDevices(devices); } } + +int GenericSyncer::SetSyncRetry(bool isRetry) +{ + syncEngine_->SetSyncRetry(isRetry); + return E_OK; +} + +int GenericSyncer::SetEqualIdentifier(const std::string &identifier, const std::vector &targets) +{ + std::lock_guard lock(syncerLock_); + if (syncEngine_ == nullptr) { + return -E_NOT_INIT; + } + return syncEngine_->SetEqualIdentifier(identifier, targets); +} + +std::string GenericSyncer::GetSyncDevicesStr(const std::vector &devices) const +{ + std::string syncDevices; + for (const auto &dev:devices) { + syncDevices += STR_MASK(dev); + syncDevices += ","; + } + return syncDevices.substr(0, syncDevices.size() - 1); +} + +int GenericSyncer::StatusCheck() const +{ + if (!initialized_) { + LOGE("[Syncer] Syncer is not initialized, return!"); + return -E_NOT_INIT; + } + if (closing_) { + LOGE("[Syncer] Syncer is closing, return!"); + return -E_BUSY; + } + return E_OK; +} + +int GenericSyncer::SyncParamCheck(const SyncParma ¶m) const +{ + std::lock_guard lock(syncerLock_); + int errCode = StatusCheck(); + if (errCode != E_OK) { + return errCode; + } + if (!IsValidDevices(param.devices) || !IsValidMode(param.mode)) { + return -E_INVALID_ARGS; + } + if (IsQueuedManualSyncFull(param.mode, param.wait)) { + LOGE("[Syncer] -E_BUSY"); + return -E_BUSY; + } + QuerySyncObject syncQuery = param.syncQuery; + return SyncConditionCheck(syncQuery, param.mode, param.isQuerySync, param.devices); +} + +void GenericSyncer::InitSyncOperation(SyncOperation *operation, const SyncParma ¶m) +{ + operation->SetIdentifier(syncInterface_->GetIdentifier()); + operation->Initialize(); + operation->OnKill(std::bind(&GenericSyncer::SyncOperationKillCallback, this, operation->GetSyncId())); + std::function onFinished = std::bind(&GenericSyncer::OnSyncFinished, this, std::placeholders::_1); + operation->SetOnSyncFinished(onFinished); + operation->SetOnSyncFinalize(param.onFinalize); + if (param.isQuerySync) { + operation->SetQuery(param.syncQuery); + } +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h index c02195a5f7126ac02ab325491f216c4c71ce49af..47bd27ff538ef1e67e91663139311ac98ab2236d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/generic_syncer.h @@ -23,8 +23,8 @@ #include "isyncer.h" #include "isync_engine.h" #include "meta_data.h" -#include "time_helper.h" #include "sync_operation.h" +#include "time_helper.h" namespace DistributedDB { class GenericSyncer : public virtual ISyncer { @@ -32,10 +32,10 @@ using DataChangedFunc = std::function; public: GenericSyncer(); - virtual ~GenericSyncer(); + ~GenericSyncer() override; // Init the Syncer modules - int Initialize(IKvDBSyncInterface *syncInterface) override; + int Initialize(ISyncInterface *syncInterface) override; // Close int Close() override; @@ -50,9 +50,14 @@ public: const std::function &)> &onComplete, const std::function &onFinalize, bool wait) override; + // Sync function. use SyncParma to reduce paramter. + int Sync(const SyncParma ¶m) override; + // Remove the operation, with the given syncId, used to clean resource if sync finished or failed. int RemoveSyncOperation(int syncId) override; + int StopSync() override; + // Get The current virtual timestamp uint64_t GetTimeStamp() override; @@ -74,15 +79,33 @@ public: // Get local deviceId, is hashed int GetLocalIdentity(std::string &outTarget) const override; + // Set Manual Sync retry config + int SetSyncRetry(bool isRetry) override; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) override; + + // Inner function, Used for subscribe sync + int Sync(const InternalSyncParma ¶m); + protected: // Remote data changed callback virtual void RemoteDataChanged(const std::string &device) = 0; + virtual void RemoteDeviceOffline(const std::string &device) = 0; + + // trigger query auto sync or auto subscribe + // trigger auto subscribe only when subscribe task is failed triggered by remote db opened + // it won't be triggered again when subscribe task success + virtual void QueryAutoSync(const InternalSyncParma ¶m); + // Create a sync engine, if has memory error, will return nullptr. virtual ISyncEngine *CreateSyncEngine() = 0; + virtual int PrepareSync(const SyncParma ¶m, uint32_t syncId); + // Add a Sync Operation, after call this function, the operation will be start - virtual int AddSyncOperation(SyncOperation *operation); + virtual void AddSyncOperation(SyncOperation *operation); // Used to set to the SyncOperation Onkill virtual void SyncOperationKillCallbackInner(int syncId); @@ -91,13 +114,13 @@ protected: void SyncOperationKillCallback(int syncId); // Init the metadata - int InitMetaData(IKvDBSyncInterface *syncInterface); + int InitMetaData(ISyncInterface *syncInterface); // Init the TimeHelper - int InitTimeHelper(IKvDBSyncInterface *syncInterface); + int InitTimeHelper(ISyncInterface *syncInterface); // Init the Sync engine - int InitSyncEngine(IKvDBSyncInterface *syncInterface); + int InitSyncEngine(ISyncInterface *syncInterface); // Used to general a sync id, maybe it is currentSyncId++; // The return value is sync id. @@ -106,6 +129,9 @@ protected: // Check if the mode arg is valid bool IsValidMode(int mode) const; + virtual int SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, + const std::vector &devices) const; + // Check if the devices arg is valid bool IsValidDevices(const std::vector &devices) const; @@ -115,7 +141,9 @@ protected: // Callback when the special sync finished. void OnSyncFinished(int syncId); - void AddQueuedManualSyncSize(int mode, bool wait); + inline bool IsManualSync(int inMode) const; + + int AddQueuedManualSyncSize(int mode, bool wait); bool IsQueuedManualSyncFull(int mode, bool wait) const; @@ -123,6 +151,14 @@ protected: void GetOnlineDevices(std::vector &devices) const; + std::string GetSyncDevicesStr(const std::vector &devices) const; + + void InitSyncOperation(SyncOperation *operation, const SyncParma ¶m); + + int StatusCheck() const; + + int SyncParamCheck(const SyncParma ¶m) const; + static int SyncModuleInit(); static int SyncResourceInit(); @@ -133,9 +169,11 @@ protected: // Used to general the next sync id. static int currentSyncId_; static std::mutex syncIdLock_; + // For sync in progress. + std::list syncIdList_; ISyncEngine *syncEngine_; - IKvDBSyncInterface *syncInterface_; + ISyncInterface *syncInterface_; std::shared_ptr timeHelper_; std::shared_ptr metadata_; bool initialized_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_engine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_engine.h old mode 100755 new mode 100644 similarity index 60% rename from services/distributeddataservice/libs/distributeddb/syncer/include/isync_engine.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/isync_engine.h index fd66bb1544beb56897c3b0ce18e706b7bd4eaf41..f7db71ff924f94da86108709948df5aa5a01ba85 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_engine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_engine.h @@ -20,16 +20,18 @@ #include #include "ikvdb_sync_interface.h" -#include "sync_operation.h" #include "meta_data.h" #include "ref_object.h" +#include "sync_operation.h" namespace DistributedDB { class ISyncEngine : public virtual RefObject { public: // Do some init things - virtual int Initialize(IKvDBSyncInterface *syncInterface, std::shared_ptr &metadata, - const std::function &onRemoteDataChanged) = 0; + virtual int Initialize(ISyncInterface *syncInterface, std::shared_ptr &metadata, + const std::function &onRemoteDataChanged, + const std::function &offlineChanged, + const std::function &queryAutoSyncCallback) = 0; // Do some things, when db close. virtual int Close() = 0; @@ -47,14 +49,33 @@ public: // Get Online devices virtual void GetOnlineDevices(std::vector &devices) const = 0; - // Register the device connect callback, this function must be called after Engine inited + // Register the device connect callback, this function must be called after Engine initted virtual void RegConnectCallback() = 0; // Get local deviceId, is hashed virtual int GetLocalIdentity(std::string &outTarget) const = 0; + // Get the database identifier virtual std::string GetLabel() const = 0; + // Set Manual Sync retry config + virtual void SetSyncRetry(bool isRetry) = 0; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + virtual int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) = 0; + + // Add auto subscribe timer when start sync engine, used for auto subscribe failed subscribe task when db online + virtual int StartAutoSubscribeTimer() = 0; + + // Stop auto subscribe timer when start sync engine + virtual void StopAutoSubscribeTimer() = 0; + + // Check if number of subscriptions out of limit + virtual int SubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const = 0; + + // Check if the Sync Engine is active, some times synchronization is not allowed + virtual bool IsEngineActive() const = 0; + protected: virtual ~ISyncEngine() {}; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_state_machine.h old mode 100755 new mode 100644 similarity index 87% rename from services/distributeddataservice/libs/distributeddb/syncer/include/isync_state_machine.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/isync_state_machine.h index 26994f19cdf32de9eae6a08831d98e9d11f026f9..ac0ed90d6ea86812b00b533ea551aea3c2a57a6a --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_state_machine.h @@ -18,20 +18,19 @@ #include -#include "sync_target.h" -#include "sync_task_context.h" #include "icommunicator.h" #include "ikvdb_sync_interface.h" +#include "query_sync_object.h" +#include "sync_target.h" +#include "sync_task_context.h" namespace DistributedDB { class ISyncStateMachine { public: - const static int RETRY_TIME = 3; - virtual ~ISyncStateMachine() {}; // Init the SyncStateMachine, this function must be called before any other call. - virtual int Initialize(ISyncTaskContext *context, IKvDBSyncInterface *syncInterface, + virtual int Initialize(ISyncTaskContext *context, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) = 0; // start a sync step @@ -54,6 +53,9 @@ public: // stop timer to ResetWatchDog when sync data one (key,value) size bigger than mtu virtual void StopFeedDogForSync(SyncDirectionFlag flag) = 0; + + // check if need trigger query auto sync and get query from inMsg + virtual bool IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query) = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_target.h b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_target.h old mode 100755 new mode 100644 similarity index 95% rename from services/distributeddataservice/libs/distributeddb/syncer/include/isync_target.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/isync_target.h index fc05ebcc0df8611a7b5f3fd2c40dc99e213be0d4..36ca06578170a11c18415ce89a342a79d87a4e3e --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_target.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_target.h @@ -43,7 +43,7 @@ public: // Get the mode of this task request or response virtual int GetMode() const = 0; - // Set a Sync Status, it will increase the ref of operation + // Set a SyncOperation virtual void SetSyncOperation(SyncOperation *operation) = 0; // Get a SyncOperation @@ -51,6 +51,8 @@ public: // Is this target is a auto sync virtual bool IsAutoSync() const = 0; + + virtual uint32_t GetResponseSessionId() const = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_task_context.h old mode 100755 new mode 100644 similarity index 84% rename from services/distributeddataservice/libs/distributeddb/syncer/include/isync_task_context.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/isync_task_context.h index 95b5337a3412e1c8f131dfd3bed40fad7c9b85e4..a2477a14d0b82f45cef43e36df3afa47de7f7034 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/isync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/isync_task_context.h @@ -16,14 +16,15 @@ #ifndef I_SYNC_TASK_CONTEXT_H #define I_SYNC_TASK_CONTEXT_H -#include "sync_target.h" -#include "sync_operation.h" + #include "icommunicator.h" #include "ikvdb_sync_interface.h" #include "meta_data.h" -#include "time_helper.h" +#include "query_sync_object.h" #include "runtime_context.h" - +#include "sync_operation.h" +#include "sync_target.h" +#include "time_helper.h" namespace DistributedDB { using CommErrHandler = std::function; @@ -34,7 +35,7 @@ public: enum TASK_EXEC_STATUS { INIT, RUNNING, FAILED, FINISHED }; // Initialize the context - virtual int Initialize(const std::string &deviceId, IKvDBSyncInterface *syncInterface, + virtual int Initialize(const std::string &deviceId, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) = 0; // Add a sync task target with the operation to the queue @@ -79,6 +80,14 @@ public: virtual bool IsAutoSync() const = 0; + virtual bool IsSyncTaskNeedRetry() const = 0; + + virtual void SetSyncRetry(bool isRetry) = 0; + + virtual int GetSyncRetryTimes() const = 0; + + virtual int GetSyncRetryTimeout(int retryTime) const = 0; + // Set a Timer used for timeout virtual int StartTimer() = 0; @@ -149,6 +158,20 @@ public: virtual void SetTaskErrCode(int errCode) = 0; + virtual void ClearAllSyncTask() = 0; + + virtual bool IsAutoLiftWaterMark() const = 0; + + virtual void IncNegotiationCount() = 0; + + virtual bool IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query) = 0; + + virtual bool IsAutoSubscribe() const = 0; + + // some sync task can be skipped if there is no change in data base since last sync + virtual bool IsCurrentSyncTaskCanBeSkipped() const = 0; + + virtual void SetIsNeedResetAbilitySync(bool isNeedReset) = 0; protected: virtual ~ISyncTaskContext() {}; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp index 7d1da4efde181d8b7b9a49c3d2822148222e3783..41dd2e0cdcf74ab82b3d334e4b08f7dd0a5ab841 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.cpp @@ -16,18 +16,20 @@ #include "meta_data.h" #include -#include "securec.h" -#include "db_errno.h" #include "db_common.h" +#include "db_constant.h" +#include "db_errno.h" +#include "hash.h" #include "log_print.h" +#include "securec.h" +#include "sync_types.h" #include "time_helper.h" -#include "hash.h" namespace DistributedDB { namespace { - const int STR_TO_LL_BY_DEC = 10; + const int STR_TO_LL_BY_DEVALUE = 10; // store local timeoffset;this is a special key; - const std::string LOCALTIMEOFFSET_KEY = "localTimeOffset"; + const std::string LOCALTIME_OFFSET_KEY = "localTimeOffset"; const std::string DEVICEID_PREFIX_KEY = "deviceId"; } @@ -43,12 +45,12 @@ Metadata::~Metadata() metadataMap_.clear(); } -int Metadata::Initialize(IKvDBSyncInterface* storage) +int Metadata::Initialize(ISyncInterface* storage) { naturalStoragePtr_ = storage; std::vector key; std::vector timeOffset; - DBCommon::StringToVector(LOCALTIMEOFFSET_KEY, key); + DBCommon::StringToVector(LOCALTIME_OFFSET_KEY, key); int errCode = GetMetadataFromDb(key, timeOffset); if (errCode == -E_NOT_FOUND) { @@ -61,6 +63,7 @@ int Metadata::Initialize(IKvDBSyncInterface* storage) std::lock_guard lockGuard(metadataLock_); metadataMap_.clear(); } + (void)querySyncWaterMarkHelper_.Initialize(storage); return LoadAllMetadata(); } @@ -68,10 +71,10 @@ int Metadata::SaveTimeOffset(const DeviceID &deviceId, TimeOffset inValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); metadata.timeOffset = inValue; metadata.lastUpdateTime = TimeHelper::GetSysCurrentTime(); - LOGD("Metadata::SaveTimeOffset = %lld dev %s{private}", inValue, deviceId.c_str()); + LOGD("Metadata::SaveTimeOffset = %lld dev %s", inValue, STR_MASK(deviceId)); return SaveMetaDataValue(deviceId, metadata); } @@ -79,7 +82,7 @@ void Metadata::GetTimeOffset(const DeviceID &deviceId, TimeOffset &outValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); outValue = metadata.timeOffset; } @@ -87,7 +90,7 @@ void Metadata::GetLocalWaterMark(const DeviceID &deviceId, uint64_t &outValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); outValue = metadata.localWaterMark; } @@ -95,9 +98,9 @@ int Metadata::SaveLocalWaterMark(const DeviceID &deviceId, uint64_t inValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); metadata.localWaterMark = inValue; - LOGD("Metadata::SaveLocalWaterMark = %llu\n", inValue); + LOGD("Metadata::SaveLocalWaterMark = %llu", inValue); return SaveMetaDataValue(deviceId, metadata); } @@ -105,7 +108,7 @@ void Metadata::GetPeerWaterMark(const DeviceID &deviceId, uint64_t &outValue) { MetaDataValue metadata; std::lock_guard lockGuard(metadataLock_); - GetMetaDataValue(deviceId, metadata); + GetMetaDataValue(deviceId, metadata, true); outValue = metadata.peerWaterMark; } @@ -124,11 +127,11 @@ int Metadata::SaveLocalTimeOffset(TimeOffset timeOffset) std::string timeOffsetString = std::to_string(timeOffset); std::vector timeOffsetValue(timeOffsetString.begin(), timeOffsetString.end()); std::vector localTimeOffsetValue( - LOCALTIMEOFFSET_KEY.begin(), LOCALTIMEOFFSET_KEY.end()); + LOCALTIME_OFFSET_KEY.begin(), LOCALTIME_OFFSET_KEY.end()); std::lock_guard lockGuard(localTimeOffsetLock_); localTimeOffset_ = timeOffset; - LOGD("Metadata::SaveLocalTimeOffset offset = %lld\n", timeOffset); + LOGD("Metadata::SaveLocalTimeOffset offset = %lld", timeOffset); int errCode = SetMetadataToDb(localTimeOffsetValue, timeOffsetValue); if (errCode != E_OK) { LOGE("Metadata::SaveLocalTimeOffset SetMetadataToDb failed errCode:%d", errCode); @@ -138,12 +141,33 @@ int Metadata::SaveLocalTimeOffset(TimeOffset timeOffset) TimeOffset Metadata::GetLocalTimeOffset() const { - return localTimeOffset_.load(std::memory_order_seq_cst); + TimeOffset localTimeOffset = localTimeOffset_.load(std::memory_order_seq_cst); + return localTimeOffset; } int Metadata::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) { - return SavePeerWaterMark(deviceId, 0, isNeedHash); + // try to erase all the waterMark + // erase deleteSync recv waterMark + WaterMark waterMark = 0; + int errCodeDeleteSync = SetRecvDeleteSyncWaterMark(deviceId, waterMark); + // erase querySync recv waterMark + int errCodeQuerySync = ResetRecvQueryWaterMark(deviceId); + // peerWaterMark must be erased at last + int errCode = SavePeerWaterMark(deviceId, 0, isNeedHash); + if (errCode != E_OK) { + LOGE("[Metadata] erase peerWaterMark failed errCode:%d", errCode); + return errCode; + } + if (errCodeQuerySync != E_OK) { + LOGE("[Metadata] erase queryWaterMark failed errCode:%d", errCodeQuerySync); + return errCodeQuerySync; + } + if (errCodeDeleteSync != E_OK) { + LOGE("[Metadata] erase deleteWaterMark failed errCode:%d", errCodeDeleteSync); + return errCodeDeleteSync; + } + return E_OK; } void Metadata::SetLastLocalTime(TimeStamp lastLocalTime) @@ -169,12 +193,12 @@ int Metadata::SaveMetaDataValue(const DeviceID &deviceId, const MetaDataValue &i } DeviceID hashDeviceId; - GetHashDeviceId(deviceId, hashDeviceId); + GetHashDeviceId(deviceId, hashDeviceId, true); std::vector key; DBCommon::StringToVector(hashDeviceId, key); errCode = SetMetadataToDb(key, value); if (errCode != E_OK) { - LOGE("Metadata::SetMetadataToDb failed errCode:%d\n", errCode); + LOGE("Metadata::SetMetadataToDb failed errCode:%d", errCode); return errCode; } PutMetadataToMap(hashDeviceId, inValue); @@ -211,7 +235,7 @@ int Metadata::DeSerializeMetaData(const std::vector &inValue, MetaDataV return E_OK; } -int Metadata::GetMetadataFromDb(const std::vector &key, std::vector &outValue) +int Metadata::GetMetadataFromDb(const std::vector &key, std::vector &outValue) const { if (naturalStoragePtr_ == nullptr) { return -E_INVALID_DB; @@ -227,6 +251,14 @@ int Metadata::SetMetadataToDb(const std::vector &key, const std::vector return naturalStoragePtr_->PutMetaData(key, inValue); } +int Metadata::DeleteMetaDataFromDB(const std::vector &keys) const +{ + if (naturalStoragePtr_ == nullptr) { + return -E_INVALID_DB; + } + return naturalStoragePtr_->DeleteMetaData(keys); +} + void Metadata::PutMetadataToMap(const DeviceID &deviceId, const MetaDataValue &value) { metadataMap_[deviceId] = value; @@ -237,11 +269,11 @@ void Metadata::GetMetadataFromMap(const DeviceID &deviceId, MetaDataValue &outVa outValue = metadataMap_[deviceId]; } -int64_t Metadata::StringToLong(const std::vector &value) +int64_t Metadata::StringToLong(const std::vector &value) const { std::string valueString(value.begin(), value.end()); - int64_t longData = std::strtoll(valueString.c_str(), nullptr, STR_TO_LL_BY_DEC); - LOGD("Metadata::StringToLong longData = %lld\n", longData); + int64_t longData = std::strtoll(valueString.c_str(), nullptr, STR_TO_LL_BY_DEVALUE); + LOGD("Metadata::StringToLong longData = %lld", longData); return longData; } @@ -253,33 +285,59 @@ int Metadata::GetAllMetadataKey(std::vector> &keys) return naturalStoragePtr_->GetAllMetaKeys(keys); } -int Metadata::LoadAllMetadata() +namespace { +bool IsMetaDataKey(const Key &inKey, const std::string &expectPrefix) { - std::vector> deviceIds; - std::vector value; - int errCode = E_OK; - GetAllMetadataKey(deviceIds); + if (inKey.size() < expectPrefix.size()) { + return false; + } + std::string prefixInKey(inKey.begin(), inKey.begin() + expectPrefix.size()); + if (prefixInKey != expectPrefix) { + return false; + } + return true; +} +} - for (auto it = deviceIds.begin(); it != deviceIds.end(); ++it) { - if (it->size() < DEVICEID_PREFIX_KEY.size()) { - continue; - } - std::string prefixKey(it->begin(), it->begin() + DEVICEID_PREFIX_KEY.size()); - if (prefixKey != DEVICEID_PREFIX_KEY) { - continue; - } - errCode = GetMetadataFromDb(*it, value); - if (errCode != E_OK) { - return errCode; - } - MetaDataValue metadata; - std::string deviceId(it->begin(), it->end()); - errCode = DeSerializeMetaData(value, metadata); - { - std::lock_guard lockGuard(metadataLock_); - PutMetadataToMap(deviceId, metadata); +int Metadata::LoadAllMetadata() +{ + std::vector> metaDataKeys; + GetAllMetadataKey(metaDataKeys); + + std::vector> querySyncIds; + for (const auto &deviceId : metaDataKeys) { + if (IsMetaDataKey(deviceId, DEVICEID_PREFIX_KEY)) { + int errCode = LoadDeviceIdDataToMap(deviceId); + if (errCode != E_OK) { + return errCode; + } + } else if (IsMetaDataKey(deviceId, QuerySyncWaterMarkHelper::GetQuerySyncPrefixKey())) { + querySyncIds.push_back(deviceId); + } else if (IsMetaDataKey(deviceId, QuerySyncWaterMarkHelper::GetDeleteSyncPrefixKey())) { + int errCode = querySyncWaterMarkHelper_.LoadDeleteSyncDataToCache(deviceId); + if (errCode != E_OK) { + return errCode; + } } } + return querySyncWaterMarkHelper_.RemoveLeastUsedQuerySyncItems(querySyncIds); +} + +int Metadata::LoadDeviceIdDataToMap(const Key &key) +{ + std::vector value; + int errCode = GetMetadataFromDb(key, value); + if (errCode != E_OK) { + return errCode; + } + MetaDataValue metaValue; + std::string metaKey(key.begin(), key.end()); + errCode = DeSerializeMetaData(value, metaValue); + if (errCode != E_OK) { + return errCode; + } + std::lock_guard lockGuard(metadataLock_); + PutMetadataToMap(metaKey, metaValue); return errCode; } @@ -309,4 +367,207 @@ void Metadata::GetHashDeviceId(const DeviceID &deviceId, DeviceID &hashDeviceId, hashDeviceId = deviceIdToHashDeviceIdMap_[deviceId]; } } -} // namespace DistributedDB + +int Metadata::GetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, WaterMark &waterMark) +{ + QueryWaterMark queryWaterMark; + int errCode = querySyncWaterMarkHelper_.GetQueryWaterMark(queryIdentify, deviceId, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + WaterMark peerWaterMark; + GetPeerWaterMark(deviceId, peerWaterMark); + waterMark = std::max(queryWaterMark.recvWaterMark, peerWaterMark); + return E_OK; +} + +int Metadata::SetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark) +{ + return querySyncWaterMarkHelper_.SetRecvQueryWaterMark(queryIdentify, deviceId, waterMark); +} + +int Metadata::GetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, WaterMark &waterMark, bool isAutoLift) +{ + QueryWaterMark queryWaterMark; + int errCode = querySyncWaterMarkHelper_.GetQueryWaterMark(queryIdentify, deviceId, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + if (isAutoLift) { + WaterMark localWaterMark; + GetLocalWaterMark(deviceId, localWaterMark); + waterMark = std::max(queryWaterMark.sendWaterMark, localWaterMark); + } else { + waterMark = queryWaterMark.sendWaterMark; + } + return E_OK; +} + +int Metadata::SetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark) +{ + return querySyncWaterMarkHelper_.SetSendQueryWaterMark(queryIdentify, deviceId, waterMark); +} + +int Metadata::GetSendDeleteSyncWaterMark(const DeviceID &deviceId, WaterMark &waterMark, bool isAutoLift) +{ + DeleteWaterMark deleteWaterMark; + int errCode = querySyncWaterMarkHelper_.GetDeleteSyncWaterMark(deviceId, deleteWaterMark); + if (errCode != E_OK) { + return errCode; + } + if (isAutoLift) { + WaterMark localWaterMark; + GetLocalWaterMark(deviceId, localWaterMark); + waterMark = std::max(deleteWaterMark.sendWaterMark, localWaterMark); + } else { + waterMark = deleteWaterMark.sendWaterMark; + } + return E_OK; +} + +int Metadata::SetSendDeleteSyncWaterMark(const DeviceID &deviceId, const WaterMark &waterMark) +{ + return querySyncWaterMarkHelper_.SetSendDeleteSyncWaterMark(deviceId, waterMark); +} + +int Metadata::GetRecvDeleteSyncWaterMark(const DeviceID &deviceId, WaterMark &waterMark) +{ + DeleteWaterMark deleteWaterMark; + int errCode = querySyncWaterMarkHelper_.GetDeleteSyncWaterMark(deviceId, deleteWaterMark); + if (errCode != E_OK) { + return errCode; + } + WaterMark peerWaterMark; + GetPeerWaterMark(deviceId, peerWaterMark); + waterMark = std::max(deleteWaterMark.recvWaterMark, peerWaterMark); + return E_OK; +} + +int Metadata::SetRecvDeleteSyncWaterMark(const DeviceID &deviceId, const WaterMark &waterMark) +{ + return querySyncWaterMarkHelper_.SetRecvDeleteSyncWaterMark(deviceId, waterMark); +} + +int Metadata::ResetRecvQueryWaterMark(const DeviceID &deviceId) +{ + return querySyncWaterMarkHelper_.ResetRecvQueryWaterMark(deviceId); +} + +void Metadata::GetDbCreateTime(const DeviceID &deviceId, uint64_t &outValue) +{ + MetaDataValue metadata; + std::lock_guard lockGuard(metadataLock_); + DeviceID hashDeviceId; + GetHashDeviceId(deviceId, hashDeviceId, true); + if (metadataMap_.find(hashDeviceId) != metadataMap_.end()) { + metadata = metadataMap_[hashDeviceId]; + outValue = metadata.dbCreateTime; + return; + } + outValue = 0; + LOGI("Metadata::GetDbCreateTime, not found dev = %s dbCreateTime", STR_MASK(deviceId)); +} + +int Metadata::SetDbCreateTime(const DeviceID &deviceId, uint64_t inValue, bool isNeedHash) +{ + MetaDataValue metadata; + std::lock_guard lockGuard(metadataLock_); + DeviceID hashDeviceId; + GetHashDeviceId(deviceId, hashDeviceId, true); + if (metadataMap_.find(hashDeviceId) != metadataMap_.end()) { + metadata = metadataMap_[hashDeviceId]; + if (metadata.dbCreateTime != 0 && metadata.dbCreateTime != inValue) { + metadata.clearDeviceDataMark = REMOVE_DEVICE_DATA_MARK; + LOGI("Metadata::SetDbCreateTime,set cleardata mark,dev=%s,dbCreateTime=%llu", STR_MASK(deviceId), inValue); + } + if (metadata.dbCreateTime == 0) { + LOGI("Metadata::SetDbCreateTime,update dev=%s,dbCreateTime=%llu", STR_MASK(deviceId), inValue); + } + } + metadata.dbCreateTime = inValue; + return SaveMetaDataValue(deviceId, metadata); +} + +int Metadata::ResetMetaDataAfterRemoveData(const DeviceID &deviceId) +{ + MetaDataValue metadata; + std::lock_guard lockGuard(metadataLock_); + DeviceID hashDeviceId; + GetHashDeviceId(deviceId, hashDeviceId, true); + if (metadataMap_.find(hashDeviceId) != metadataMap_.end()) { + metadata = metadataMap_[hashDeviceId]; + metadata.clearDeviceDataMark = 0; + // when finished remove data, should reset send watermark between high version sync + // recv watermark has already reset in removedata func. + metadata.localWaterMark = 0; + return SaveMetaDataValue(deviceId, metadata); + } + return -E_NOT_FOUND; +} + +void Metadata::GetRemoveDataMark(const DeviceID &deviceId, uint64_t &outValue) +{ + MetaDataValue metadata; + std::lock_guard lockGuard(metadataLock_); + DeviceID hashDeviceId; + GetHashDeviceId(deviceId, hashDeviceId, true); + if (metadataMap_.find(hashDeviceId) != metadataMap_.end()) { + metadata = metadataMap_[hashDeviceId]; + outValue = metadata.clearDeviceDataMark; + return; + } + outValue = 0; +} + +int Metadata::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + if (naturalStoragePtr_ == nullptr) { + return -E_INVALID_DB; + } + return naturalStoragePtr_->DeleteMetaDataByPrefixKey(keyPrefix); +} + +uint64_t Metadata::GetQueryLastTimestamp(const DeviceID &deviceId, const std::string &queryId) const +{ + std::vector key; + std::vector value; + std::string hashqueryId = DBConstant::SUBSCRIBE_QUERY_PREFIX + DBCommon::TransferHashString(queryId); + DBCommon::StringToVector(hashqueryId, key); + int errCode = GetMetadataFromDb(key, value); + std::lock_guard lockGuard(metadataLock_); + if (errCode == -E_NOT_FOUND) { + auto iter = queryIdMap_.find(deviceId); + if (iter != queryIdMap_.end()) { + if (iter->second.find(hashqueryId) == iter->second.end()) { + iter->second.insert(hashqueryId); + return INT64_MAX; + } + return 0; + } else { + queryIdMap_[deviceId] = { hashqueryId }; + return INT64_MAX; + } + } + auto iter = queryIdMap_.find(deviceId); + // while value is found in db, it can be found in db later when db is not closed + // so no need to record the hashqueryId in map + if (errCode == E_OK && iter != queryIdMap_.end()) { + iter->second.erase(hashqueryId); + } + return StringToLong(value); +} + +void Metadata::RemoveQueryFromRecordSet(const DeviceID &deviceId, const std::string &queryId) +{ + std::lock_guard lockGuard(metadataLock_); + std::string hashqueryId = DBConstant::SUBSCRIBE_QUERY_PREFIX + DBCommon::TransferHashString(queryId); + auto iter = queryIdMap_.find(deviceId); + if (iter != queryIdMap_.end() && iter->second.find(hashqueryId) != iter->second.end()) { + iter->second.erase(hashqueryId); + } +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/meta_data.h b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.h old mode 100755 new mode 100644 similarity index 49% rename from services/distributeddataservice/libs/distributeddb/syncer/include/meta_data.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.h index 1cb6356fed4ca947b737c8dba8e38738bfb18d26..c0d280b1225007199e826da9c5fa1bbf49060b30 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/meta_data.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/meta_data.h @@ -16,13 +16,14 @@ #ifndef META_DATA_H #define META_DATA_H +#include +#include #include #include -#include #include "db_types.h" #include "ikvdb_sync_interface.h" -#include "ref_object.h" +#include "query_sync_water_mark_helper.h" namespace DistributedDB { struct MetaDataValue { @@ -30,6 +31,8 @@ struct MetaDataValue { uint64_t lastUpdateTime = 0; uint64_t localWaterMark = 0; uint64_t peerWaterMark = 0; + TimeStamp dbCreateTime = 0; + uint64_t clearDeviceDataMark = 0; // Default 0 for not remove device data. }; class Metadata { @@ -37,7 +40,7 @@ public: Metadata(); ~Metadata(); - int Initialize(IKvDBSyncInterface *storage); + int Initialize(ISyncInterface *storage); int SaveTimeOffset(const DeviceID &deviceId, TimeOffset inValue); @@ -49,37 +52,81 @@ public: void GetPeerWaterMark(const DeviceID &deviceId, uint64_t &outValue); - int SavePeerWaterMark(const DeviceID &deviceId, uint64_t inValue, bool isNeedHash = true); + int SavePeerWaterMark(const DeviceID &deviceId, uint64_t inValue, bool isNeedHash); int SaveLocalTimeOffset(TimeOffset timeOffset); TimeOffset GetLocalTimeOffset() const; - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash = true); + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash); void SetLastLocalTime(TimeStamp lastLocalTime); TimeStamp GetLastLocalTime() const; + int SetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark); + + // the querySync's sendWatermark will increase by the device watermark + // if the sendWatermark less than device watermark + int GetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, WaterMark &waterMark, bool isAutoLift = true); + + int SetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark); + + // the querySync's recvWatermark will increase by the device watermark + // if the watermark less than device watermark + int GetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, WaterMark &waterMark); + + int SetSendDeleteSyncWaterMark(const std::string &deviceId, const WaterMark &waterMark); + + // the deleteSync's sendWatermark will increase by the device watermark + // if the sendWatermark less than device watermark + int GetSendDeleteSyncWaterMark(const std::string &deviceId, WaterMark &waterMark, bool isAutoLift = true); + + int SetRecvDeleteSyncWaterMark(const std::string &deviceId, const WaterMark &waterMark); + + // the deleteSync's recvWatermark will increase by the device watermark + // if the recvWatermark less than device watermark + int GetRecvDeleteSyncWaterMark(const std::string &deviceId, WaterMark &waterMark); + + void GetDbCreateTime(const DeviceID &deviceId, uint64_t &outValue); + + int SetDbCreateTime(const DeviceID &deviceId, uint64_t inValue, bool isNeedHash); + + int ResetMetaDataAfterRemoveData(const DeviceID &deviceId); + + void GetRemoveDataMark(const DeviceID &deviceId, uint64_t &outValue); + + // always get value from db, value updated from storage trigger + uint64_t GetQueryLastTimestamp(const DeviceID &deviceId, const std::string &queryId) const; + + void RemoveQueryFromRecordSet(const DeviceID &deviceId, const std::string &queryId); private: int SaveMetaDataValue(const DeviceID &deviceId, const MetaDataValue &inValue); - void GetMetaDataValue(const DeviceID &deviceId, MetaDataValue &outValue, bool isNeedHash = true); + // sync module need hash devices id + void GetMetaDataValue(const DeviceID &deviceId, MetaDataValue &outValue, bool isNeedHash); int SerializeMetaData(const MetaDataValue &inValue, std::vector &outValue); int DeSerializeMetaData(const std::vector &inValue, MetaDataValue &outValue) const; - int GetMetadataFromDb(const std::vector &key, std::vector &outValue); + int GetMetadataFromDb(const std::vector &key, std::vector &outValue) const; int SetMetadataToDb(const std::vector &key, const std::vector &inValue); + int DeleteMetaDataFromDB(const std::vector &keys) const; + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const; + void PutMetadataToMap(const DeviceID &deviceId, const MetaDataValue &value); void GetMetadataFromMap(const DeviceID &deviceId, MetaDataValue &outValue); - int64_t StringToLong(const std::vector &value); + int64_t StringToLong(const std::vector &value) const; int GetAllMetadataKey(std::vector> &keys); @@ -87,23 +134,37 @@ private: uint64_t GetRandTimeOffset() const; - void GetHashDeviceId(const DeviceID &deviceId, DeviceID &hashDeviceId, bool isNeedHash = true); + void GetHashDeviceId(const DeviceID &deviceId, DeviceID &hashDeviceId, bool isNeedHash); + + // this function will read data from db by metaData's key + // and then serialize it and put to map + int LoadDeviceIdDataToMap(const Key &key); + + // reset the waterMark to zero + int ResetRecvQueryWaterMark(const DeviceID &deviceId); // store localTimeOffset in ram; if change, should add a lock first, change here and metadata, // then release lock std::atomic localTimeOffset_; std::mutex localTimeOffsetLock_; - IKvDBSyncInterface *naturalStoragePtr_; + ISyncInterface *naturalStoragePtr_; // if changed, it should be locked from save-to-db to change-in-memory.save to db must be first, // if save to db fail, it will not be changed in memory. std::map metadataMap_; - std::mutex metadataLock_; + mutable std::mutex metadataLock_; std::map deviceIdToHashDeviceIdMap_; // store localTimeOffset in ram, used to make timestamp increase mutable std::mutex lastLocalTimeLock_; TimeStamp lastLocalTime_; + + QuerySyncWaterMarkHelper querySyncWaterMarkHelper_; + + // set value: SUBSCRIBE_QUERY_PREFIX + DBCommon::TransferHashString(queryId) + // queryId is not in set while key is not found from db first time, and return lastTimestamp = INT64_MAX + // if query is in set return 0 while not found from db, means already sync before, don't trigger again + mutable std::map> queryIdMap_; }; } // namespace DistributedDB #endif diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/multi_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_data_sync.h old mode 100755 new mode 100644 similarity index 100% rename from services/distributeddataservice/libs/distributeddb/syncer/include/multi_ver_data_sync.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_data_sync.h index b8da3d0722e0f5c817c0656fdc02783f3a16b0d4..88de2845632682c4fd1b22fa0e00e6174b40a3bf --- a/services/distributeddataservice/libs/distributeddb/syncer/include/multi_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_data_sync.h @@ -19,10 +19,10 @@ #ifndef OMIT_MULTI_VER #include -#include "multi_ver_kvdb_sync_interface.h" #include "icommunicator.h" -#include "sync_task_context.h" +#include "multi_ver_kvdb_sync_interface.h" #include "multi_ver_sync_task_context.h" +#include "sync_task_context.h" namespace DistributedDB { class MultiVerRequestPacket { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.cpp index ba3231deb868e7020309c92b712249d4b2a2086f..e2bc23b141e65bc3a0cf87e3d3c22843f067bc58 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.cpp @@ -44,7 +44,7 @@ MultiVerSyncStateMachine::~MultiVerSyncStateMachine() Clear(); } -int MultiVerSyncStateMachine::Initialize(ISyncTaskContext *context, IKvDBSyncInterface *syncInterface, +int MultiVerSyncStateMachine::Initialize(ISyncTaskContext *context, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) { if (context == nullptr || syncInterface == nullptr || metadata == nullptr || communicator == nullptr) { @@ -102,7 +102,7 @@ void MultiVerSyncStateMachine::StepToIdle() StopWatchDog(); context_->Clear(); PerformanceAnalysis::GetInstance()->TimeRecordEnd(); - LOGD("[MultiVerSyncStateMachine][%s{private}] step to idle", context_->GetDeviceId().c_str()); + LOGD("[MultiVerSyncStateMachine][%s] step to idle", STR_MASK(context_->GetDeviceId())); } int MultiVerSyncStateMachine::MessageCallbackCheck(const Message *inMsg) @@ -126,7 +126,14 @@ int MultiVerSyncStateMachine::MessageCallbackCheck(const Message *inMsg) int MultiVerSyncStateMachine::ReceiveMessageCallback(Message *inMsg) { - if (inMsg != nullptr && inMsg->GetMessageId() == TIME_SYNC_MESSAGE) { + if (inMsg == nullptr) { + return -E_INVALID_ARGS; + } + if (inMsg->IsFeedbackError()) { + LOGE("[MultiVerSyncStateMachine] Feedback Message with errorNo=%u.", inMsg->GetErrorNo()); + return -(inMsg->GetErrorNo()); + } + if (inMsg->GetMessageId() == TIME_SYNC_MESSAGE) { return TimeSyncPacketRecvCallback(context_, inMsg); } std::lock_guard lock(stateMachineLock_); @@ -195,8 +202,7 @@ void MultiVerSyncStateMachine::SyncStepInnerLocked() goto SYNC_STEP_OUT; } - LOGD("[MultiVerSyncStateMachine] SyncStep dst=%s{private}, state = %d", - context_->GetDeviceId().c_str(), currentState_); + LOGD("[MultiVerSyncStateMachine] SyncStep dst=%s, state = %d", STR_MASK(context_->GetDeviceId()), currentState_); int errCode; { std::lock_guard lock(stateMachineLock_); @@ -279,7 +285,7 @@ int MultiVerSyncStateMachine::PrepareNextSyncTask() return StartSyncInner(); } -void MultiVerSyncStateMachine::SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) +void MultiVerSyncStateMachine::SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) { } @@ -402,20 +408,20 @@ void MultiVerSyncStateMachine::Finish() if (commitsSize > 0) { context_->GetCommit(commitsSize - 1, commit); context_->GetCommits(commits); - LOGD("MultiVerSyncStateMachine::Finish merge src=%s{private}", context_->GetDeviceId().c_str()); + LOGD("MultiVerSyncStateMachine::Finish merge src=%s", STR_MASK(context_->GetDeviceId())); PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); if (performance != nullptr) { performance->StepTimeRecordStart(MV_TEST_RECORDS::RECORD_MERGE); } int errCode = multiVerDataSync_->MergeSyncCommit(commit, commits); - LOGD("MultiVerSyncStateMachine::Finish merge src=%s{private}, MergeSyncCommit errCode:%d", - context_->GetDeviceId().c_str(), errCode); + LOGD("MultiVerSyncStateMachine::Finish merge src=%s, MergeSyncCommit errCode:%d", + STR_MASK(context_->GetDeviceId()), errCode); if (performance != nullptr) { performance->StepTimeRecordEnd(MV_TEST_RECORDS::RECORD_MERGE); } } RefObject::AutoLock lock(context_); - context_->SetOperationStatus(SyncOperation::FINISHED_ALL); + context_->SetOperationStatus(SyncOperation::OP_FINISHED_ALL); StepToIdle(); ExecNextTask(); } @@ -429,15 +435,15 @@ int MultiVerSyncStateMachine::OneCommitSyncFinish() int errCode = E_OK; int commitIndex = context_->GetCommitIndex(); - LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s{private}, commitIndex = %d,", - context_->GetDeviceId().c_str(), commitIndex); + LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s, commitIndex = %d,", STR_MASK(context_->GetDeviceId()), + commitIndex); if (commitIndex > 0) { context_->GetCommit(commitIndex - 1, commit); deviceName = context_->GetDeviceId(); context_->GetEntries(entries); - LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s{private}, entries size = %lu", - context_->GetDeviceId().c_str(), entries.size()); - errCode = timeSync_->GetTimeOffset(outOffset); + LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s, entries size = %lu", + STR_MASK(context_->GetDeviceId()), entries.size()); + errCode = timeSync_->GetTimeOffset(outOffset, TIME_SYNC_WAIT_TIME); if (errCode != E_OK) { LOGI("MultiVerSyncStateMachine::OneCommitSyncFinish GetTimeOffset fail errCode:%d", errCode); return errCode; @@ -447,8 +453,8 @@ int MultiVerSyncStateMachine::OneCommitSyncFinish() // Due to time sync error, commit timestamp may bigger than currentLocalTime, we need to fix the timestamp TimeOffset timefixOffset = (commit.timestamp < currentLocalTime) ? 0 : (commit.timestamp - currentLocalTime); - LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s{private}, timefixOffset = %lld", - context_->GetDeviceId().c_str(), timefixOffset); + LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish src=%s, timefixOffset = %lld", + STR_MASK(context_->GetDeviceId()), timefixOffset); commit.timestamp -= timefixOffset; for (MultiVerKvEntry *entry : entries) { TimeStamp timeStamp; @@ -461,8 +467,8 @@ int MultiVerSyncStateMachine::OneCommitSyncFinish() performance->StepTimeRecordStart(MV_TEST_RECORDS::RECORD_PUT_COMMIT_DATA); } errCode = multiVerDataSync_->PutCommitData(commit, entries, deviceName); - LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish PutCommitData src=%s{private}, errCode = %d", - context_->GetDeviceId().c_str(), errCode); + LOGD("MultiVerSyncStateMachine::OneCommitSyncFinish PutCommitData src=%s, errCode = %d", + STR_MASK(context_->GetDeviceId()), errCode); if (performance != nullptr) { performance->StepTimeRecordEnd(MV_TEST_RECORDS::RECORD_PUT_COMMIT_DATA); } @@ -529,10 +535,9 @@ void MultiVerSyncStateMachine::SyncResponseBegin(uint32_t sessionId) int errCode = RuntimeContext::GetInstance()->SetTimer( RESPONSE_TIME_OUT, timeOutCallback, [this]() { - int errCode = RuntimeContext::GetInstance()->ScheduleTask([this](){ RefObject::DecObjRef(context_); }); - if (errCode != E_OK) { - LOGE("[MultiVerSyncStateMachine][SyncResponseEnd] timer finalizer ScheduleTask, errCode %d", - errCode); + int ret = RuntimeContext::GetInstance()->ScheduleTask([this](){ RefObject::DecObjRef(context_); }); + if (ret != E_OK) { + LOGE("[MultiVerSyncStateMachine][SyncResponseEnd] timer finalizer ScheduleTask, errCode %d", ret); } }, timerId); @@ -583,5 +588,10 @@ int MultiVerSyncStateMachine::SyncResponseTimeout(TimerId timerId) SyncResponseEnd(sessionId); return E_OK; } + +bool MultiVerSyncStateMachine::IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query) +{ + return false; +} } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.h index 312e34730e4793401d71e6f272660a69d866d4d9..b4c6689f3d6a1f99d2410dbbef6009f7677664b8 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_state_machine.h @@ -19,14 +19,14 @@ #ifndef OMIT_MULTI_VER #include -#include "sync_state_machine.h" -#include "multi_ver_sync_task_context.h" -#include "meta_data.h" -#include "time_sync.h" #include "commit_history_sync.h" +#include "db_types.h" +#include "meta_data.h" #include "multi_ver_data_sync.h" +#include "multi_ver_sync_task_context.h" +#include "sync_state_machine.h" +#include "time_sync.h" #include "value_slice_sync.h" -#include "db_types.h" namespace DistributedDB { class MultiVerSyncStateMachine final : public SyncStateMachine { @@ -40,7 +40,7 @@ public: ~MultiVerSyncStateMachine() override; // Init the MultiVerSyncStateMachine - int Initialize(ISyncTaskContext *context, IKvDBSyncInterface *syncInterface, std::shared_ptr &metadata, + int Initialize(ISyncTaskContext *context, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) override; // send Message to the StateMachine @@ -72,7 +72,9 @@ protected: int PrepareNextSyncTask() override; // Called by StartSaveDataNotifyTimer, used to send a save data notify packet - void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) override; + void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) override; + + bool IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query) override; private: enum State { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_task_context.cpp index f4bd65a6aea04ab2a551147c7fcd3488e1ccad35..96599edac8e5473f4aba590dd25c095844b39006 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_task_context.cpp @@ -26,7 +26,7 @@ MultiVerSyncTaskContext::~MultiVerSyncTaskContext() { } -int MultiVerSyncTaskContext::Initialize(const std::string &deviceId, IKvDBSyncInterface *syncInterface, +int MultiVerSyncTaskContext::Initialize(const std::string &deviceId, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) { if (deviceId.empty() || (syncInterface == nullptr) || (communicator == nullptr)) { @@ -70,7 +70,7 @@ int MultiVerSyncTaskContext::AddSyncOperation(SyncOperation *operation) if (operation->IsAutoSync() && !IsTargetQueueEmpty()) { LOGI("[MultiVerSyncTaskContext] Exist operation in queue, skip it!"); - operation->SetStatus(deviceId_, SyncOperation::FINISHED_ALL); + operation->SetStatus(deviceId_, SyncOperation::OP_FINISHED_ALL); return E_OK; } @@ -226,9 +226,9 @@ void MultiVerSyncTaskContext::Clear() syncId_ = 0; } -void MultiVerSyncTaskContext::CopyTargetData(const ISyncTarget *target) +void MultiVerSyncTaskContext::CopyTargetData(const ISyncTarget *target, const TaskParam &taskParam) { - SyncTaskContext::CopyTargetData(target); + SyncTaskContext::CopyTargetData(target, taskParam); } } #endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_task_context.h index 6f03d165656550b46403e5a2205b37c1162bc269..054e9759db00065de931d5588a31e3d3b14b8308 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_sync_task_context.h @@ -17,8 +17,8 @@ #define MULTI_VER_SYNC_TASK_CONTEXT_H #ifndef OMIT_MULTI_VER -#include "sync_task_context.h" #include "multi_ver_kvdb_sync_interface.h" +#include "sync_task_context.h" namespace DistributedDB { class MultiVerSyncTaskContext final : public SyncTaskContext { @@ -28,7 +28,7 @@ public: DISABLE_COPY_ASSIGN_MOVE(MultiVerSyncTaskContext); // Init the MultiVerSyncTaskContext - int Initialize(const std::string &deviceId, IKvDBSyncInterface *syncInterface, std::shared_ptr &metadata, + int Initialize(const std::string &deviceId, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) override; // Add a sync task target with the operation to the queue @@ -85,7 +85,7 @@ public: protected: ~MultiVerSyncTaskContext() override; - void CopyTargetData(const ISyncTarget *target) override; + void CopyTargetData(const ISyncTarget *target, const TaskParam &taskParam) override; private: DECLARE_OBJECT_TAG(MultiVerSyncTaskContext); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp index 94b1a8d26e278293ae30aa9262296a4452c28745..669dff8434062b9301b6628edf5ec86fe7664f04 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.cpp @@ -52,9 +52,9 @@ void MultiVerSyncer::EnableAutoSync(bool enable) return; } - int syncId = Sync(devices, SyncOperation::AUTO_PULL, nullptr, nullptr, false); - if (syncId < MIN_VALID_SYNC_ID) { - LOGE("[Syncer] sync start by EnableAutoSync failed err %d", syncId); + int errCode = Sync(devices, SyncModeType::AUTO_PULL, nullptr, nullptr, false); + if (errCode != E_OK) { + LOGE("[Syncer] sync start by EnableAutoSync failed err %d", errCode); } } @@ -83,26 +83,27 @@ void MultiVerSyncer::RemoteDataChanged(const std::string &device) if (autoSyncEnable_) { std::vector devices; devices.push_back(device); - int syncId = Sync(devices, SyncOperation::AUTO_PULL, nullptr, nullptr, false); - if (syncId < MIN_VALID_SYNC_ID) { - LOGE("[MultiVerSyncer] sync start by RemoteDataChanged failed err %d", syncId); + int errCode = Sync(devices, SyncModeType::AUTO_PULL, nullptr, nullptr, false); + if (errCode != E_OK) { + LOGE("[MultiVerSyncer] sync start by RemoteDataChanged failed err %d", errCode); } } } +void MultiVerSyncer::RemoteDeviceOffline(const std::string &device) +{ +} + ISyncEngine *MultiVerSyncer::CreateSyncEngine() { return new (std::nothrow) MultiVerSyncEngine(); } -int MultiVerSyncer::AddSyncOperation(SyncOperation *operation) +void MultiVerSyncer::AddSyncOperation(SyncOperation *operation) { - if (operation == nullptr) { - return -E_INVALID_ARGS; - } MultiVerKvDBSyncInterface *syncInterface = static_cast(syncInterface_); syncInterface->NotifyStartSyncOperation(); - return GenericSyncer::AddSyncOperation(operation); + GenericSyncer::AddSyncOperation(operation); } void MultiVerSyncer::SyncOperationKillCallbackInner(int syncId) diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.h index 65f985bc464b4d4ba85ae2ff817b8147e540e485..97b9c6b027ce803f47cb6e2503b6af6f20bf5e67 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/multi_ver_syncer.h @@ -17,8 +17,8 @@ #define MULTI_VER_SYNCER_H #ifndef OMIT_MULTI_VER -#include "macro_utils.h" #include "generic_syncer.h" +#include "macro_utils.h" namespace DistributedDB { class MultiVerSyncer final : public GenericSyncer { @@ -38,6 +38,8 @@ public: // Remote data changed callback void RemoteDataChanged(const std::string &device) override; + void RemoteDeviceOffline(const std::string &device) override; + // Set stale data wipe policy int SetStaleDataWipePolicy(WipePolicy policy) override; @@ -46,7 +48,7 @@ protected: ISyncEngine *CreateSyncEngine() override; // Add a Sync Operation, after call this function, the operation will be start - int AddSyncOperation(SyncOperation *operation) override; + void AddSyncOperation(SyncOperation *operation) override; // Used to set to the SyncOperation Onkill void SyncOperationKillCallbackInner(int syncId) override; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7227596e0d7321586de4f95ba3fc020ca642f77a --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.cpp @@ -0,0 +1,569 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "query_sync_water_mark_helper.h" + +#include +#include +#include "platform_specific.h" +#include "parcel.h" +#include "db_errno.h" +#include "db_common.h" +#include "log_print.h" + +namespace DistributedDB { +namespace { + const int MAX_CACHE_ITEMS = 200; + const uint32_t MAX_STORE_ITEMS = 100000; + // WaterMark Version + constexpr uint32_t QUERY_WATERMARK_VERSION_CURRENT = SOFTWARE_VERSION_RELEASE_3_0; + constexpr uint32_t DELETE_WATERMARK_VERSION_CURRENT = SOFTWARE_VERSION_RELEASE_3_0; + // Prefix Key in db + const std::string QUERY_SYNC_PREFIX_KEY = "querySync"; + const std::string DELETE_SYNC_PREFIX_KEY = "deleteSync"; +} + +QuerySyncWaterMarkHelper::QuerySyncWaterMarkHelper() + : storage_(nullptr) +{} + +QuerySyncWaterMarkHelper::~QuerySyncWaterMarkHelper() +{ + storage_ = nullptr; + deviceIdToHashQuerySyncIdMap_.clear(); + deleteSyncCache_.clear(); + deviceIdToHashDeleteSyncIdMap_.clear(); +} + +int LruMap::Put(const std::string &key, const QueryWaterMark &inValue) +{ + std::lock_guard autoLock(lruLock_); + cache_[key] = inValue; + return Elimination(key, inValue); +} + +int LruMap::Get(const std::string &key, QueryWaterMark &outValue) +{ + std::lock_guard autoLock(lruLock_); + if (cache_.count(key) == 0) { + return -E_NOT_FOUND; + } + outValue = cache_[key]; + return Elimination(key, outValue); +} + +void LruMap::RemoveWithPrefixKey(const std::string &prefixKey) +{ + std::lock_guard autoLock(lruLock_); + auto iterator = eliminationChain_.begin(); + while (iterator != eliminationChain_.end()) { + const std::string &key = (*iterator).first; + if (key.find(prefixKey) == 0) { + (void)cache_.erase(key); + iterator = eliminationChain_.erase(iterator); + } else { + iterator++; + } + } +} + +// move the node to last and remove the first node until the size less than limit +int LruMap::Elimination(const std::string &key, const QueryWaterMark &inQueryWaterMark) +{ + auto iterator = eliminationChain_.begin(); + while (iterator != eliminationChain_.end()) { + if ((*iterator).first == key) { + eliminationChain_.erase(iterator); + break; + } + iterator++; + } + std::pair entry = {key, inQueryWaterMark}; + eliminationChain_.push_back(entry); + while (eliminationChain_.size() > MAX_CACHE_ITEMS) { + std::pair &pair = eliminationChain_.front(); + cache_.erase(pair.first); + eliminationChain_.pop_front(); + } + return E_OK; +} + +int QuerySyncWaterMarkHelper::GetMetadataFromDb(const std::vector &key, std::vector &outValue) +{ + if (storage_ == nullptr) { + return -E_INVALID_DB; + } + return storage_->GetMetaData(key, outValue); +} + +int QuerySyncWaterMarkHelper::SetMetadataToDb(const std::vector &key, const std::vector &inValue) +{ + if (storage_ == nullptr) { + return -E_INVALID_DB; + } + return storage_->PutMetaData(key, inValue); +} + +int QuerySyncWaterMarkHelper::DeleteMetaDataFromDB(const std::vector &keys) const +{ + if (storage_ == nullptr) { + return -E_INVALID_DB; + } + return storage_->DeleteMetaData(keys); +} + +int QuerySyncWaterMarkHelper::Initialize(ISyncInterface *storage) +{ + storage_ = storage; + return E_OK; +} + +int QuerySyncWaterMarkHelper::LoadDeleteSyncDataToCache(const Key &deleteWaterMarkKey) +{ + std::vector value; + int errCode = GetMetadataFromDb(deleteWaterMarkKey, value); + if (errCode != E_OK) { + return errCode; + } + DeleteWaterMark deleteWaterMark; + std::string dbKey(deleteWaterMarkKey.begin(), deleteWaterMarkKey.end()); + errCode = DeSerializeDeleteWaterMark(value, deleteWaterMark); + if (errCode != E_OK) { + return errCode; + } + std::lock_guard autoLock(deleteSyncLock_); + deleteSyncCache_[dbKey] = deleteWaterMark; + return errCode; +} + +int QuerySyncWaterMarkHelper::GetQueryWaterMarkInCacheAndDb(const std::string &cacheKey, + QueryWaterMark &queryWaterMark) +{ + // first get from cache_ + int errCode = querySyncCache_.Get(cacheKey, queryWaterMark); + bool addToCache = false; + if (errCode == -E_NOT_FOUND) { + // second get from db + errCode = GetQueryWaterMarkFromDB(cacheKey, queryWaterMark); + addToCache = true; + } + if (errCode == -E_NOT_FOUND) { + // third generate one and save to db + errCode = PutQueryWaterMarkToDB(cacheKey, queryWaterMark); + } + // something error return + if (errCode != E_OK) { + LOGE("[Meta]GetQueryWaterMark Fail code = %d", errCode); + return errCode; + } + // remember add to cache_ + if (addToCache) { + querySyncCache_.Put(cacheKey, queryWaterMark); + } + return errCode; +} + +int QuerySyncWaterMarkHelper::GetQueryWaterMark(const std::string &queryIdentify, const std::string &deviceId, + QueryWaterMark &queryWaterMark) +{ + std::string cacheKey; + GetHashQuerySyncDeviceId(deviceId, queryIdentify, cacheKey); + std::lock_guard autoLock(queryWaterMarkLock_); + return GetQueryWaterMarkInCacheAndDb(cacheKey, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::SetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark) +{ + std::string cacheKey; + GetHashQuerySyncDeviceId(deviceId, queryIdentify, cacheKey); + std::lock_guard autoLock(queryWaterMarkLock_); + return SetRecvQueryWaterMarkWithoutLock(deviceId, cacheKey, waterMark); +} + +int QuerySyncWaterMarkHelper::SetRecvQueryWaterMarkWithoutLock(const std::string &deviceId, + const std::string &cacheKey, const WaterMark &waterMark) +{ + QueryWaterMark queryWaterMark; + int errCode = GetQueryWaterMarkInCacheAndDb(cacheKey, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + queryWaterMark.recvWaterMark = waterMark; + return UpdateCacheAndSave(cacheKey, deviceId, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::SetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark) +{ + std::string cacheKey; + GetHashQuerySyncDeviceId(deviceId, queryIdentify, cacheKey); + QueryWaterMark queryWaterMark; + std::lock_guard autoLock(queryWaterMarkLock_); + int errCode = GetQueryWaterMarkInCacheAndDb(cacheKey, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + queryWaterMark.sendWaterMark = waterMark; + return UpdateCacheAndSave(cacheKey, deviceId, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::UpdateCacheAndSave(const std::string &cacheKey, + const std::string &deviceId, QueryWaterMark &queryWaterMark) +{ + // update lastUsedTime + int errCode = OS::GetCurrentSysTimeInMicrosecond(queryWaterMark.lastUsedTime); + if (errCode != E_OK) { + return errCode; + } + // save db first + errCode = SaveQueryWaterMarkToDB(cacheKey, queryWaterMark); + if (errCode != E_OK) { + return errCode; + } + querySyncCache_.Put(cacheKey, queryWaterMark); + return errCode; +} + +int QuerySyncWaterMarkHelper::PutQueryWaterMarkToDB(const DeviceID &dbKeyString, QueryWaterMark &queryWaterMark) +{ + int errCode = OS::GetCurrentSysTimeInMicrosecond(queryWaterMark.lastUsedTime); + if (errCode != E_OK) { + return errCode; + } + queryWaterMark.version = QUERY_WATERMARK_VERSION_CURRENT; + return SaveQueryWaterMarkToDB(dbKeyString, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::SaveQueryWaterMarkToDB(const DeviceID &dbKeyString, const QueryWaterMark &queryWaterMark) +{ + // serialize value + Value dbValue; + int errCode = SerializeQueryWaterMark(queryWaterMark, dbValue); + if (errCode != E_OK) { + return errCode; + } + // serialize key + Key dbKey; + DBCommon::StringToVector(dbKeyString, dbKey); + // save + errCode = SetMetadataToDb(dbKey, dbValue); + if (errCode != E_OK) { + LOGE("QuerySyncWaterMarkHelper::SaveQueryWaterMarkToDB failed errCode:%d", errCode); + } + return errCode; +} + +int QuerySyncWaterMarkHelper::GetQueryWaterMarkFromDB(const DeviceID &dbKeyString, QueryWaterMark &queryWaterMark) +{ + // serialize key + Key dbKey; + DBCommon::StringToVector(dbKeyString, dbKey); + // search in db + Value dbValue; + int errCode = GetMetadataFromDb(dbKey, dbValue); + if (errCode != E_OK) { + return errCode; + } + return DeSerializeQueryWaterMark(dbValue, queryWaterMark); +} + +int QuerySyncWaterMarkHelper::SerializeQueryWaterMark(const QueryWaterMark &queryWaterMark, Value &outValue) +{ + uint64_t length = CalculateQueryWaterMarkSize(queryWaterMark); + outValue.resize(length); + Parcel parcel(outValue.data(), outValue.size()); + parcel.WriteUInt32(queryWaterMark.version); + parcel.EightByteAlign(); + parcel.WriteUInt64(queryWaterMark.sendWaterMark); + parcel.WriteUInt64(queryWaterMark.recvWaterMark); + parcel.WriteUInt64(queryWaterMark.lastUsedTime); + parcel.WriteString(queryWaterMark.sql); + if (parcel.IsError()) { + LOGE("[Meta] Parcel error when serialize queryWaterMark"); + return -E_PARSE_FAIL; + } + return E_OK; +} + +int QuerySyncWaterMarkHelper::DeSerializeQueryWaterMark(const Value &dbQueryWaterMark, QueryWaterMark &queryWaterMark) +{ + Parcel parcel(const_cast(dbQueryWaterMark.data()), dbQueryWaterMark.size()); + parcel.ReadUInt32(queryWaterMark.version); + parcel.EightByteAlign(); + parcel.ReadUInt64(queryWaterMark.sendWaterMark); + parcel.ReadUInt64(queryWaterMark.recvWaterMark); + parcel.ReadUInt64(queryWaterMark.lastUsedTime); + parcel.ReadString(queryWaterMark.sql); + if (parcel.IsError()) { + LOGE("[Meta] Parcel error when deserialize queryWaterMark"); + return -E_PARSE_FAIL; + } + return E_OK; +} + +uint64_t QuerySyncWaterMarkHelper::CalculateQueryWaterMarkSize(const QueryWaterMark &queryWaterMark) +{ + uint64_t length = Parcel::GetUInt32Len(); // version + length = Parcel::GetEightByteAlign(length); + length += Parcel::GetUInt64Len(); // sendWaterMark + length += Parcel::GetUInt64Len(); // recvWaterMark + length += Parcel::GetUInt64Len(); // lastUsedTime + length += Parcel::GetStringLen(queryWaterMark.sql); + return length; +} + +void QuerySyncWaterMarkHelper::GetHashQuerySyncDeviceId(const DeviceID &deviceId, + const DeviceID &queryId, DeviceID &hashQuerySyncId) +{ + std::lock_guard autoLock(queryWaterMarkLock_); + if (deviceIdToHashQuerySyncIdMap_[deviceId].count(queryId) == 0) { + // do not modify this + hashQuerySyncId = QUERY_SYNC_PREFIX_KEY + DBCommon::TransferHashString(deviceId) + queryId; + deviceIdToHashQuerySyncIdMap_[deviceId][queryId] = hashQuerySyncId; + } else { + hashQuerySyncId = deviceIdToHashQuerySyncIdMap_[deviceId][queryId]; + } +} + +int QuerySyncWaterMarkHelper::GetDeleteSyncWaterMark(const std::string &deviceId, DeleteWaterMark &deleteWaterMark) +{ + std::string hashId; + GetHashDeleteSyncDeviceId(deviceId, hashId); + return GetDeleteWaterMarkFromCache(hashId, deleteWaterMark); +} + +int QuerySyncWaterMarkHelper::SetSendDeleteSyncWaterMark(const DeviceID &deviceId, const WaterMark &waterMark) +{ + std::string hashId; + GetHashDeleteSyncDeviceId(deviceId, hashId); + DeleteWaterMark deleteWaterMark; + GetDeleteWaterMarkFromCache(hashId, deleteWaterMark); + deleteWaterMark.sendWaterMark = waterMark; + std::lock_guard autoLock(deleteSyncLock_); + return UpdateDeleteSyncCacheAndSave(hashId, deleteWaterMark); +} + +int QuerySyncWaterMarkHelper::SetRecvDeleteSyncWaterMark(const DeviceID &deviceId, const WaterMark &waterMark) +{ + std::string hashId; + GetHashDeleteSyncDeviceId(deviceId, hashId); + DeleteWaterMark deleteWaterMark; + GetDeleteWaterMarkFromCache(hashId, deleteWaterMark); + deleteWaterMark.recvWaterMark = waterMark; + std::lock_guard autoLock(deleteSyncLock_); + return UpdateDeleteSyncCacheAndSave(hashId, deleteWaterMark); +} + +int QuerySyncWaterMarkHelper::UpdateDeleteSyncCacheAndSave(const std::string &dbKey, + const DeleteWaterMark &deleteWaterMark) +{ + // save db first + int errCode = SaveDeleteWaterMarkToDB(dbKey, deleteWaterMark); + if (errCode != E_OK) { + return errCode; + } + // modify cache + deleteSyncCache_[dbKey] = deleteWaterMark; + return errCode; +} + +int QuerySyncWaterMarkHelper::GetDeleteWaterMarkFromCache(const DeviceID &hashDeviceId, + DeleteWaterMark &deleteWaterMark) +{ + // lock prevent different thread visit deleteSyncCache_ + std::lock_guard autoLock(deleteSyncLock_); + // if not found + if (deleteSyncCache_.find(hashDeviceId) == deleteSyncCache_.end()) { + DeleteWaterMark waterMark; + waterMark.version = DELETE_WATERMARK_VERSION_CURRENT; + int errCode = GetDeleteWaterMarkFromDB(hashDeviceId, waterMark); + if (errCode == -E_NOT_FOUND) { + deleteWaterMark.sendWaterMark = 0; + deleteWaterMark.recvWaterMark = 0; + errCode = E_OK; + } + if (errCode != E_OK) { + LOGE("[Meta]GetDeleteWaterMark Fail code = %d", errCode); + return errCode; + } + deleteSyncCache_.insert(std::pair(hashDeviceId, waterMark)); + } + deleteWaterMark = deleteSyncCache_[hashDeviceId]; + return E_OK; +} + +int QuerySyncWaterMarkHelper::GetDeleteWaterMarkFromDB(const DeviceID &hashDeviceId, + DeleteWaterMark &deleteWaterMark) +{ + Key dbKey; + DBCommon::StringToVector(hashDeviceId, dbKey); + // search in db + Value dbValue; + int errCode = GetMetadataFromDb(dbKey, dbValue); + if (errCode != E_OK) { + return errCode; + } + // serialize value + return DeSerializeDeleteWaterMark(dbValue, deleteWaterMark); +} + +int QuerySyncWaterMarkHelper::SaveDeleteWaterMarkToDB(const DeviceID &hashDeviceId, + const DeleteWaterMark &deleteWaterMark) +{ + // serialize value + Value dbValue; + int errCode = SerializeDeleteWaterMark(deleteWaterMark, dbValue); + if (errCode != E_OK) { + return errCode; + } + Key dbKey; + DBCommon::StringToVector(hashDeviceId, dbKey); + // save + errCode = SetMetadataToDb(dbKey, dbValue); + if (errCode != E_OK) { + LOGE("QuerySyncWaterMarkHelper::SaveDeleteWaterMarkToDB failed errCode:%d", errCode); + } + return errCode; +} + +void QuerySyncWaterMarkHelper::GetHashDeleteSyncDeviceId(const DeviceID &deviceId, DeviceID &hashDeleteSyncId) +{ + std::lock_guard autoLock(deleteSyncLock_); + if (deviceIdToHashDeleteSyncIdMap_.count(deviceId) == 0) { + hashDeleteSyncId = DELETE_SYNC_PREFIX_KEY + DBCommon::TransferHashString(deviceId); + deviceIdToHashDeleteSyncIdMap_.insert(std::pair(deviceId, hashDeleteSyncId)); + } else { + hashDeleteSyncId = deviceIdToHashDeleteSyncIdMap_[deviceId]; + } +} + +int QuerySyncWaterMarkHelper::SerializeDeleteWaterMark(const DeleteWaterMark &deleteWaterMark, + std::vector &outValue) +{ + uint64_t length = CalculateDeleteWaterMarkSize(deleteWaterMark); + outValue.resize(length); + Parcel parcel(outValue.data(), outValue.size()); + parcel.WriteUInt32(deleteWaterMark.version); + parcel.EightByteAlign(); + parcel.WriteUInt64(deleteWaterMark.sendWaterMark); + parcel.WriteUInt64(deleteWaterMark.recvWaterMark); + if (parcel.IsError()) { + LOGE("[Meta] Parcel error when serialize deleteWaterMark."); + return -E_PARSE_FAIL; + } + return E_OK; +} + +int QuerySyncWaterMarkHelper::DeSerializeDeleteWaterMark(const std::vector &inValue, + DeleteWaterMark &deleteWaterMark) +{ + Parcel parcel(const_cast(inValue.data()), inValue.size()); + parcel.ReadUInt32(deleteWaterMark.version); + parcel.EightByteAlign(); + parcel.ReadUInt64(deleteWaterMark.sendWaterMark); + parcel.ReadUInt64(deleteWaterMark.recvWaterMark); + if (parcel.IsError()) { + LOGE("[Meta] Parcel error when deserialize deleteWaterMark."); + return -E_PARSE_FAIL; + } + return E_OK; +} + +uint64_t QuerySyncWaterMarkHelper::CalculateDeleteWaterMarkSize(const DeleteWaterMark &deleteWaterMark) +{ + uint64_t length = Parcel::GetUInt32Len(); // version + length = Parcel::GetEightByteAlign(length); + length += Parcel::GetUInt64Len(); // sendWaterMark + length += Parcel::GetUInt64Len(); // recvWaterMark + return length; +} + +std::string QuerySyncWaterMarkHelper::GetQuerySyncPrefixKey() +{ + return QUERY_SYNC_PREFIX_KEY; +} + +std::string QuerySyncWaterMarkHelper::GetDeleteSyncPrefixKey() +{ + return DELETE_SYNC_PREFIX_KEY; +} + +int QuerySyncWaterMarkHelper::RemoveLeastUsedQuerySyncItems(const std::vector &querySyncIds) +{ + if (querySyncIds.size() < MAX_STORE_ITEMS) { + return E_OK; + } + std::vector> allItems; + std::map> idMap; + std::vector> waitToRemove; + for (const auto &id : querySyncIds) { + Value value; + int errCode = GetMetadataFromDb(id, value); + if (errCode != E_OK) { + waitToRemove.push_back(id); + continue; // may be this failure cause by wrong data + } + QueryWaterMark queryWaterMark; + std::string queryKey(id.begin(), id.end()); + errCode = DeSerializeQueryWaterMark(value, queryWaterMark); + if (errCode != E_OK) { + waitToRemove.push_back(id); + continue; // may be this failure cause by wrong data + } + idMap.insert({queryKey, id}); + allItems.emplace_back(queryKey, queryWaterMark.lastUsedTime); + } + // we only remove broken data below + // 1. common data size less then 10w + // 2. allItems.size() - MAX_STORE_ITEMS - waitToRemove.size() < 0 + // so we only let allItems.size() < MAX_STORE_ITEMS + waitToRemove.size() + if (allItems.size() < MAX_STORE_ITEMS + waitToRemove.size()) { + // remove in db + return DeleteMetaDataFromDB(waitToRemove); + } + uint32_t removeCount = allItems.size() - MAX_STORE_ITEMS - waitToRemove.size(); + // quick select the k_th least used + std::nth_element(allItems.begin(), allItems.begin() + removeCount, allItems.end(), + [](std::pair &w1, std::pair &w2) { + return w1.second < w2.second; + }); + for (uint32_t i = 0; i < removeCount; ++i) { + waitToRemove.push_back(idMap[allItems[i].first]); + } + // remove in db + return DeleteMetaDataFromDB(waitToRemove); +} + +int QuerySyncWaterMarkHelper::ResetRecvQueryWaterMark(const DeviceID &deviceId) +{ + // lock prevent other thread modify queryWaterMark at this moment + { + std::lock_guard autoLock(queryWaterMarkLock_); + std::string prefixKeyStr = QUERY_SYNC_PREFIX_KEY + DBCommon::TransferHashString(deviceId); + // remove in db + Key prefixKey; + DBCommon::StringToVector(prefixKeyStr, prefixKey); + int errCode = storage_->DeleteMetaDataByPrefixKey(prefixKey); + if (errCode != E_OK) { + LOGE("[META]ResetRecvQueryWaterMark fail errCode:%d", errCode); + return errCode; + } + // clean cache + querySyncCache_.RemoveWithPrefixKey(prefixKeyStr); + } + return E_OK; +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.h b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.h new file mode 100644 index 0000000000000000000000000000000000000000..e9f82e9a5f5ec0ed265779dc1c3d9dad6a993bb1 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/query_sync_water_mark_helper.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef QUERY_SYNC_WATER_MARK_HELPER_H +#define QUERY_SYNC_WATER_MARK_HELPER_H + +#include +#include +#include +#include +#include +#include "db_types.h" +#include "ikvdb_sync_interface.h" + +namespace DistributedDB { +struct QueryWaterMark { + uint32_t version = 0; + WaterMark sendWaterMark = 0; + WaterMark recvWaterMark = 0; + TimeStamp lastUsedTime = 0; // use for delete data + std::string sql; // for analyze sql from logs +}; + +struct DeleteWaterMark { + uint32_t version = 0; + WaterMark sendWaterMark = 0; + WaterMark recvWaterMark = 0; +}; + +// LRU map +class LruMap final { +public: + LruMap() = default; + ~LruMap() = default; + + DISABLE_COPY_ASSIGN_MOVE(LruMap); + + int Get(const std::string &key, QueryWaterMark &outValue); + int Put(const std::string &key, const QueryWaterMark &inValue); + void RemoveWithPrefixKey(const std::string &prefixKey); +private: + int Elimination(const std::string &key, const QueryWaterMark &inQueryWaterMark); + + std::mutex lruLock_; + std::map cache_; + std::deque> eliminationChain_; +}; + +class QuerySyncWaterMarkHelper { +public: + QuerySyncWaterMarkHelper(); + ~QuerySyncWaterMarkHelper(); + + DISABLE_COPY_ASSIGN_MOVE(QuerySyncWaterMarkHelper); + + int Initialize(ISyncInterface *storage); + + int GetQueryWaterMark(const std::string &queryIdentify, const std::string &deviceId, + QueryWaterMark &queryWaterMark); + + int SetSendQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark); + + int SetRecvQueryWaterMark(const std::string &queryIdentify, + const std::string &deviceId, const WaterMark &waterMark); + + int GetDeleteSyncWaterMark(const std::string &deviceId, DeleteWaterMark &deleteWaterMark); + + int SetSendDeleteSyncWaterMark(const std::string &deviceId, const WaterMark &waterMark); + + int SetRecvDeleteSyncWaterMark(const std::string &deviceId, const WaterMark &waterMark); + + // this function will read deleteWaterMark from db by it's deleteWaterMarkKey + // and then serialize it and put to cache + int LoadDeleteSyncDataToCache(const Key &deleteWaterMarkKey); + + // this function will remove data in db + int RemoveLeastUsedQuerySyncItems(const std::vector &querySyncIds); + + // reset the waterMark to zero + int ResetRecvQueryWaterMark(const DeviceID &deviceId); + + static std::string GetQuerySyncPrefixKey(); + + static std::string GetDeleteSyncPrefixKey(); + +private: + + int GetMetadataFromDb(const std::vector &key, std::vector &outValue); + + int SetMetadataToDb(const std::vector &key, const std::vector &inValue); + + int DeleteMetaDataFromDB(const std::vector &keys) const; + + int SaveQueryWaterMarkToDB(const DeviceID &dbKeyString, const QueryWaterMark &queryWaterMark); + + int GetQueryWaterMarkFromDB(const DeviceID &dbKeyString, QueryWaterMark &queryWaterMark); + + int SetRecvQueryWaterMarkWithoutLock(const std::string &deviceId, const std::string &cacheKey, + const WaterMark &waterMark); + + // search the queryWaterMark from db or cache_ + // and ensure it exit in cache_ + int GetQueryWaterMarkInCacheAndDb(const std::string &cacheKey, QueryWaterMark &queryWaterMark); + + // only first create queryWaterMark will call this function + // it will create a queryWaterMark and save to db + int PutQueryWaterMarkToDB(const DeviceID &dbKeyString, QueryWaterMark &queryWaterMark); + + // get the querySync hashId in cache_ or generate one and then put it in to cache_ + // the hashId is made up of "QUERY_SYNC_PREFIX_KEY" + hash(deviceId) + queryId + void GetHashQuerySyncDeviceId(const DeviceID &deviceId, + const DeviceID &queryId, DeviceID &hashQuerySyncId); + + // put queryWaterMark to lru cache_ and then save to db + int UpdateCacheAndSave(const std::string &cacheKey, const std::string &deviceId, QueryWaterMark &queryWaterMark); + + // search the deleteWaterMark from db or cache_ + // and ensure it exit in cache_ + int GetDeleteWaterMarkFromCache(const DeviceID &hashDeviceId, DeleteWaterMark &deleteWaterMark); + + // get the deleteSync hashId in cache_ or generate one and then put it in to cache_ + // the hashId is made up of "DELETE_SYNC_PREFIX_KEY" + hash(deviceId) + void GetHashDeleteSyncDeviceId(const DeviceID &deviceId, DeviceID &hashDeleteSyncId); + + int SaveDeleteWaterMarkToDB(const DeviceID &hashDeviceId, const DeleteWaterMark &deleteWaterMark); + + int GetDeleteWaterMarkFromDB(const DeviceID &hashDeviceId, DeleteWaterMark &deleteWaterMark); + + // put queryWaterMark to lru cache_ and then save to db + int UpdateDeleteSyncCacheAndSave(const std::string &dbKey, const DeleteWaterMark &deleteWaterMark); + + static int SerializeQueryWaterMark(const QueryWaterMark &queryWaterMark, std::vector &outValue); + + static int DeSerializeQueryWaterMark(const std::vector &dbQueryWaterMark, QueryWaterMark &queryWaterMark); + + static uint64_t CalculateQueryWaterMarkSize(const QueryWaterMark &queryWaterMark); + + static int SerializeDeleteWaterMark(const DeleteWaterMark &deleteWaterMark, std::vector &outValue); + + static int DeSerializeDeleteWaterMark(const std::vector &inValue, DeleteWaterMark &deleteWaterMark); + + static uint64_t CalculateDeleteWaterMarkSize(const DeleteWaterMark &deleteWaterMark); + + // store or visit queryWaterMark should add a lock + // because it will change the eliminationChain + // and the queryWaterMark use a LRU Map to store in ram + std::mutex queryWaterMarkLock_; + LruMap querySyncCache_; + std::map> deviceIdToHashQuerySyncIdMap_; + + // also store deleteKeyWaterMark should add a lock + std::mutex deleteSyncLock_; + std::map deleteSyncCache_; + std::map deviceIdToHashDeleteSyncIdMap_; + + ISyncInterface *storage_; +}; +} // namespace DistributedDB +#endif diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2942d8587917fc6a7370c5bdd7d7d8ed47ebd6ff --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.cpp @@ -0,0 +1,472 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "single_ver_data_packet.h" +#include "icommunicator.h" +#include "single_ver_kvdb_sync_interface.h" +#include "query_sync_object.h" +#include "generic_single_ver_kv_entry.h" +#include "sync_types.h" +#include "version.h" +#include "parcel.h" + + +namespace DistributedDB { +DataRequestPacket::~DataRequestPacket() +{ + for (auto &entry : data_) { + delete entry; + entry = nullptr; + } +} + +void DataRequestPacket::SetData(std::vector &data) +{ + data_ = std::move(data); +} + +const std::vector &DataRequestPacket::GetData() const +{ + return data_; +} + +void DataRequestPacket::SetCompressData(std::vector &compressData) +{ + compressData_ = std::move(compressData); +} + +const std::vector &DataRequestPacket::GetCompressData() const +{ + return compressData_; +} + +void DataRequestPacket::SetEndWaterMark(WaterMark waterMark) +{ + endWaterMark_ = waterMark; +} + +WaterMark DataRequestPacket::GetEndWaterMark() const +{ + return endWaterMark_; +} + +void DataRequestPacket::SetLocalWaterMark(WaterMark waterMark) +{ + localWaterMark_ = waterMark; +} + +WaterMark DataRequestPacket::GetLocalWaterMark() const +{ + return localWaterMark_; +} + +void DataRequestPacket::SetPeerWaterMark(WaterMark waterMark) +{ + peerWaterMark_ = waterMark; +} + +WaterMark DataRequestPacket::GetPeerWaterMark() const +{ + return peerWaterMark_; +} + +void DataRequestPacket::SetSendCode(int32_t errCode) +{ + sendCode_ = errCode; +} + +int32_t DataRequestPacket::GetSendCode() const +{ + return sendCode_; +} + +void DataRequestPacket::SetMode(int32_t mode) +{ + mode_ = mode; +} + +int32_t DataRequestPacket::GetMode() const +{ + return mode_; +} + +void DataRequestPacket::SetSessionId(uint32_t sessionId) +{ + sessionId_ = sessionId; +} + +uint32_t DataRequestPacket::GetSessionId() const +{ + return sessionId_; +} + +void DataRequestPacket::SetVersion(uint32_t version) +{ + version_ = version; +} + +uint32_t DataRequestPacket::GetVersion() const +{ + return version_; +} + +void DataRequestPacket::SetReserved(std::vector &reserved) +{ + reserved_ = std::move(reserved); +} + +void DataRequestPacket::SetReserved(std::vector &&reserved) +{ + reserved_ = reserved; +} + +std::vector DataRequestPacket::GetReserved() const +{ + return reserved_; +} + +uint64_t DataRequestPacket::GetPacketId() const +{ + uint64_t packetId = 0; + std::vector DataRequestReserve = GetReserved(); + if (DataRequestReserve.size() > REQUEST_PACKET_RESERVED_INDEX_PACKETID) { + return DataRequestReserve[REQUEST_PACKET_RESERVED_INDEX_PACKETID]; + } else { + return packetId; + } +} + +uint32_t DataRequestPacket::CalculateLen(uint32_t messageId) const +{ + uint64_t totalLen = GenericSingleVerKvEntry::CalculateLens( + IsCompressData() ? std::vector {} : data_, version_); // for data + totalLen += Parcel::GetUInt64Len(); // endWaterMark + totalLen += Parcel::GetUInt64Len(); // localWaterMark + totalLen += Parcel::GetUInt64Len(); // peerWaterMark + totalLen += Parcel::GetIntLen(); // sendCode + totalLen += Parcel::GetIntLen(); // mode + totalLen += Parcel::GetUInt32Len(); // sessionId + totalLen += Parcel::GetUInt32Len(); // version + totalLen += Parcel::GetVectorLen(reserved_); // reserved + + if (version_ > SOFTWARE_VERSION_RELEASE_2_0) { + totalLen += Parcel::GetUInt32Len(); // flag bit0 used for isLastSequence + } + totalLen = Parcel::GetEightByteAlign(totalLen); // 8-byte align + if (totalLen > INT32_MAX) { + return 0; + } + if (messageId == QUERY_SYNC_MESSAGE) { + // deleted watermark + totalLen += Parcel::GetUInt64Len(); + // query id + totalLen += Parcel::GetStringLen(queryId_); + // add for queryObject + totalLen += query_.CalculateParcelLen(SOFTWARE_VERSION_CURRENT); + } + if (IsCompressData()) { + totalLen += GenericSingleVerKvEntry::CalculateCompressedLens(compressData_); // add forcompressData_ + } + if (totalLen > INT32_MAX) { + return 0; + } + return totalLen; +} + +void DataRequestPacket::SetFlag(uint32_t flag) +{ + flag_ = flag; +} + +uint32_t DataRequestPacket::GetFlag() const +{ + return flag_; +} + +bool DataRequestPacket::IsLastSequence() const +{ + return ((flag_ & IS_LAST_SEQUENCE) == IS_LAST_SEQUENCE); +} + +void DataRequestPacket::SetLastSequence() +{ + flag_ = flag_ | IS_LAST_SEQUENCE; +} + +bool DataRequestPacket::IsNeedUpdateWaterMark() const +{ + return !((flag_ & IS_UPDATE_WATER) == IS_UPDATE_WATER); +} + +void DataRequestPacket::SetUpdateWaterMark() +{ + flag_ = flag_ | IS_UPDATE_WATER; +} + +void DataRequestPacket::SetCompressDataMark() +{ + flag_ = flag_ | IS_COMPRESS_DATA; +} + +bool DataRequestPacket::IsCompressData() const +{ + return ((flag_ & IS_COMPRESS_DATA) == IS_COMPRESS_DATA); +} + +void DataRequestPacket::SetCompressAlgo(CompressAlgorithm algo) +{ + algo_ = algo; +} + +CompressAlgorithm DataRequestPacket::GetCompressAlgo() const +{ + return algo_; +} + +void DataRequestPacket::SetBasicInfo(int sendCode, uint32_t version, int32_t mode) +{ + SetSendCode(sendCode); + SetVersion(version); + SetMode(mode); +} + +void DataRequestPacket::SetWaterMark(WaterMark localMark, WaterMark peerMark, WaterMark deletedWatermark) +{ + localWaterMark_ = localMark; + peerWaterMark_ = peerMark; + deletedWatermark_ = deletedWatermark; +} + +void DataRequestPacket::SetQuery(const QuerySyncObject &query) +{ + query_ = query; +} + +QuerySyncObject DataRequestPacket::GetQuery() const +{ + return query_; +} + +void DataRequestPacket::SetQueryId(const std::string &queryId) +{ + queryId_ = queryId; +} + +std::string DataRequestPacket::GetQueryId() const +{ + return queryId_; +} + +void DataRequestPacket::SetDeletedWaterMark(WaterMark watermark) +{ + deletedWatermark_ = watermark; +} + +WaterMark DataRequestPacket::GetDeletedWaterMark() const +{ + return deletedWatermark_; +} + +void DataAckPacket::SetData(uint64_t data) +{ + data_ = data; +} + +uint64_t DataAckPacket::GetData() const +{ + return data_; +} + +void DataAckPacket::SetRecvCode(int32_t errorCode) +{ + recvCode_ = errorCode; +} + +int32_t DataAckPacket::GetRecvCode() const +{ + return recvCode_; +} + +void DataAckPacket::SetVersion(uint32_t version) +{ + version_ = version; +} + +uint32_t DataAckPacket::GetVersion() const +{ + return version_; +} + +void DataAckPacket::SetReserved(std::vector &reserved) +{ + reserved_ = std::move(reserved); +} + +std::vector DataAckPacket::GetReserved() const +{ + return reserved_; +} + +uint64_t DataAckPacket::GetPacketId() const +{ + uint64_t packetId = 0; + std::vector DataAckReserve = GetReserved(); + if (DataAckReserve.size() > ACK_PACKET_RESERVED_INDEX_PACKETID) { + packetId = DataAckReserve[ACK_PACKET_RESERVED_INDEX_PACKETID]; + } + // while remote db is close and open again, it may not carry packetId + // so the second index is deletewatermark if it is the query Sync, should drop the deletewatermark here + if (packetId > MAX_PACKETID) { + return 0; + } + return packetId; +} + +bool DataAckPacket::IsPacketIdValid(uint64_t packetId) +{ + return (packetId > 0); +} + +uint32_t DataAckPacket::CalculateLen() const +{ + uint64_t len = Parcel::GetUInt64Len(); // ackWaterMark + len += Parcel::GetIntLen(); // recvCode + len += Parcel::GetUInt32Len(); // version + len += Parcel::GetVectorLen(reserved_); // reserved + + len = Parcel::GetEightByteAlign(len); + if (len > INT32_MAX) { + return 0; + } + return len; +} + +void ControlRequestPacket::SetPacketHead(int sendCode, uint32_t version, int32_t controlCmd, uint32_t flag) +{ + sendCode_ = sendCode; + version_ = version; + controlCmdType_ = controlCmd; + flag_ = flag; +} + +int32_t ControlRequestPacket::GetSendCode() const +{ + return sendCode_; +} + +uint32_t ControlRequestPacket::GetVersion() const +{ + return version_; +} + +uint32_t ControlRequestPacket::GetcontrolCmdType() const +{ + return controlCmdType_; +} + +uint32_t ControlRequestPacket::GetFlag() const +{ + return flag_; +} + +void ControlRequestPacket::SetQuery(const QuerySyncObject &query) +{ +} + +uint32_t ControlRequestPacket::CalculateLen() const +{ + uint64_t len = Parcel::GetUInt32Len(); // version_ + len += Parcel::GetIntLen(); // sendCode_ + len += Parcel::GetUInt32Len(); // controlCmdType_ + len += Parcel::GetUInt32Len(); // flag + + len = Parcel::GetEightByteAlign(len); + if (len > INT32_MAX) { + return 0; + } + return len; +} + +void SubscribeRequest::SetQuery(const QuerySyncObject &query) +{ + query_ = query; +} + +QuerySyncObject SubscribeRequest::GetQuery() const +{ + return query_; +} + +uint32_t SubscribeRequest::CalculateLen() const +{ + uint64_t totalLen = ControlRequestPacket::CalculateLen(); + if (totalLen == 0) { + LOGE("[SubscribeRequest] cal packet len failed"); + return 0; + } + // add for queryObject + totalLen += query_.CalculateParcelLen(SOFTWARE_VERSION_CURRENT); + if (totalLen > INT32_MAX) { + return 0; + } + return totalLen; +} + +bool SubscribeRequest::IsAutoSubscribe() const +{ + return ((GetFlag() & IS_AUTO_SUBSCRIBE) == IS_AUTO_SUBSCRIBE); +} + +void ControlAckPacket::SetPacketHead(int recvCode, uint32_t version, int32_t controlCmd, uint32_t flag) +{ + recvCode_ = recvCode; + version_ = version; + controlCmdType_ = controlCmd; + flag_ = flag; +} + +int32_t ControlAckPacket::GetRecvCode() const +{ + return recvCode_; +} + +uint32_t ControlAckPacket::GetVersion() const +{ + return version_; +} + +uint32_t ControlAckPacket::GetcontrolCmdType() const +{ + return controlCmdType_; +} + +uint32_t ControlAckPacket::GetFlag() const +{ + return flag_; +} + +uint32_t ControlAckPacket::CalculateLen() const +{ + uint64_t len = Parcel::GetUInt32Len(); // version_ + len += Parcel::GetIntLen(); // recvCode_ + len += Parcel::GetUInt32Len(); // controlCmdType_ + len += Parcel::GetUInt32Len(); // flag + len = Parcel::GetEightByteAlign(len); + if (len > INT32_MAX) { + return 0; + } + return len; +} +} // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.h new file mode 100644 index 0000000000000000000000000000000000000000..7731a71a45597371eaebd3596aa53485390f14c4 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_packet.h @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SINGLE_VER_DATA_PACKET_NEW_H +#define SINGLE_VER_DATA_PACKET_NEW_H + +#include "icommunicator.h" +#include "parcel.h" +#include "query_sync_object.h" +#include "single_ver_kvdb_sync_interface.h" +#include "sync_types.h" +#include "version.h" + +namespace DistributedDB { +using SendDataItem = SingleVerKvEntry *; + +class DataRequestPacket { +public: + DataRequestPacket() {}; + virtual ~DataRequestPacket(); + + void SetData(std::vector &data); + + const std::vector &GetData() const; + const std::vector &GetCompressedData() const; + + void SetCompressData(std::vector &compressData); + + const std::vector &GetCompressData() const; + + void SetEndWaterMark(WaterMark waterMark); + + WaterMark GetEndWaterMark() const; + + void SetLocalWaterMark(WaterMark waterMark); + + WaterMark GetLocalWaterMark() const; + + void SetPeerWaterMark(WaterMark waterMark); + + WaterMark GetPeerWaterMark() const; + + void SetSendCode(int32_t errCode); + + int32_t GetSendCode() const; + + void SetMode(int32_t mode); + + int32_t GetMode() const; + + void SetSessionId(uint32_t sessionId); + + uint32_t GetSessionId() const; + + void SetVersion(uint32_t version); + + uint32_t GetVersion() const; + + uint32_t CalculateLen(uint32_t messageId) const; + + void SetReserved(std::vector &reserved); + void SetReserved(std::vector &&reserved); + + std::vector GetReserved() const; + + uint64_t GetPacketId() const; + + void SetFlag(uint32_t flag); + + uint32_t GetFlag() const; + + bool IsLastSequence() const; + + void SetLastSequence(); + + bool IsNeedUpdateWaterMark() const; + + void SetUpdateWaterMark(); + + void SetBasicInfo(int sendCode, uint32_t version, int32_t mode); + + void SetWaterMark(WaterMark localMark, WaterMark peerMark, WaterMark deletedWatermark); + + void SetQuery(const QuerySyncObject &query); + QuerySyncObject GetQuery() const; + + void SetQueryId(const std::string &queryId); + std::string GetQueryId() const; + + void SetDeletedWaterMark(WaterMark watermark); + WaterMark GetDeletedWaterMark() const; + + void SetCompressDataMark(); + bool IsCompressData() const; + + void SetCompressAlgo(CompressAlgorithm algo); + CompressAlgorithm GetCompressAlgo() const; + +protected: + std::vector data_; + WaterMark endWaterMark_ = 0; + WaterMark localWaterMark_ = 0; + WaterMark peerWaterMark_ = 0; + int32_t sendCode_ = 0; + int32_t mode_ = SyncModeType::INVALID_MODE; + uint32_t sessionId_ = 0; + uint32_t version_ = SOFTWARE_VERSION_CURRENT; + std::vector reserved_; + uint32_t flag_ = 0; // bit 0 used for isLastSequence + // add for query sync mode + QuerySyncObject query_; + std::string queryId_; + WaterMark deletedWatermark_ = 0; + std::vector compressData_; // if compressData size is above 0, means use compressData and ignore data_ + CompressAlgorithm algo_ = CompressAlgorithm::NONE; // used for param while serialize compress data + static const uint32_t IS_LAST_SEQUENCE = 0x1; // bit 0 used for isLastSequence, 1: is last, 0: not last + static const uint32_t IS_UPDATE_WATER = 0x2; // bit 1 used for update watermark, 0: update, 1: not update + static const uint32_t IS_COMPRESS_DATA = 0x4; // bit 3 used for compress data, 0: raw data, 1: compress data +}; + +class DataAckPacket { +public: + DataAckPacket() {}; + virtual ~DataAckPacket() {}; + + void SetData(uint64_t data); + + uint64_t GetData() const; + + void SetRecvCode(int32_t errorCode); + + int32_t GetRecvCode() const; + + void SetVersion(uint32_t version); + + uint32_t GetVersion() const; + + void SetReserved(std::vector &reserved); + + std::vector GetReserved() const; + + uint64_t GetPacketId() const; + + static bool IsPacketIdValid(uint64_t packetId); + + uint32_t CalculateLen() const; + +private: + /* + * data_ is waterMark when revCode_ == LOCAL_WATER_MARK_NOT_INIT || revCode_ == E_OK; + * data_ is timer in milliSeconds when revCode_ == -E_SAVE_DATA_NOTIFY && data_ != 0. + */ + uint64_t data_ = 0; + int32_t recvCode_ = 0; + uint32_t version_ = SOFTWARE_VERSION_CURRENT; + std::vector reserved_; +}; + +class ControlRequestPacket { +public: + ControlRequestPacket() {}; + virtual ~ControlRequestPacket() {}; + void SetPacketHead(int sendCode, uint32_t version, int32_t controlCmd, uint32_t flag); + + int32_t GetSendCode() const; + uint32_t GetVersion() const; + uint32_t GetcontrolCmdType() const; + uint32_t GetFlag() const; + virtual void SetQuery(const QuerySyncObject &query); + virtual uint32_t CalculateLen() const; +private: + uint32_t version_ = SOFTWARE_VERSION_CURRENT; + int32_t sendCode_ = 0; + uint32_t controlCmdType_ = 0; + uint32_t flag_ = 0; +}; + +class SubscribeRequest : public ControlRequestPacket { +public: + SubscribeRequest() {}; + ~SubscribeRequest() override {}; + QuerySyncObject GetQuery() const; + bool IsAutoSubscribe() const; + void SetQuery(const QuerySyncObject &query) override; + uint32_t CalculateLen() const override; + static const uint32_t IS_AUTO_SUBSCRIBE = 0x1; +private: + QuerySyncObject query_; +}; + +class ControlAckPacket { +public: + ControlAckPacket() {}; + virtual ~ControlAckPacket() {}; + void SetPacketHead(int recvCode, uint32_t version, int32_t controlCmd, uint32_t flag); + int32_t GetRecvCode() const; + uint32_t GetVersion() const; + uint32_t GetcontrolCmdType() const; + uint32_t GetFlag() const; + uint32_t CalculateLen() const; + +private: + uint32_t version_ = SOFTWARE_VERSION_CURRENT; + int32_t recvCode_ = 0; + uint32_t controlCmdType_ = 0; + uint32_t flag_ = 0; +}; +} // namespace DistributedDB + +#endif // SINGLE_VER_DATA_SYNC_NEW_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp index 9bce54fc6f506baae2f41ad5f70b01333e794be3..46cd9692e0d2a77f98dffb80477adae70e136b59 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.cpp @@ -16,1212 +16,1319 @@ #include "single_ver_data_sync.h" #include "db_common.h" +#include "db_types.h" +#include "generic_single_ver_kv_entry.h" +#include "intercepted_data_impl.h" #include "log_print.h" -#include "single_ver_sync_state_machine.h" -#include "performance_analysis.h" #include "message_transform.h" -#include "generic_single_ver_kv_entry.h" -#include "single_ver_sync_target.h" -#include "db_constant.h" +#include "performance_analysis.h" +#include "single_ver_sync_state_machine.h" +#include "subscribe_manager.h" +#ifdef RELATIONAL_STORE +#include "relational_db_sync_interface.h" +#endif namespace DistributedDB { namespace { - // index 0 for packetId in data request - // if ack reserve size is 1, reserve is {localWaterMark} - // if ack reserve size is above 2, reserve is {localWaterMark, packetId ...} - constexpr uint32_t REQUEST_PACKET_RESERVED_INDEX_PACKETID = 0; - constexpr uint32_t ACK_PACKET_RESERVED_INDEX_PACKETID = 1; - constexpr uint32_t ACK_PACKET_RESERVED_INDEX_LOCAL_WATER_MARK = 0; // index 0 for localWaterMark - - void SetMessageHeadInfo(Message &message, uint16_t inMsgType, const std::string &inTarget, uint32_t inSequenceId, - uint32_t inSessionId) - { - message.SetMessageType(inMsgType); - message.SetTarget(inTarget); - message.SetSequenceId(inSequenceId); - message.SetSessionId(inSessionId); - } -} - -DataRequestPacket::~DataRequestPacket() -{ - for (auto &entry : data_) { - delete entry; - entry = nullptr; - } -} - -void DataRequestPacket::SetData(std::vector &data) -{ - data_ = std::move(data); -} - -const std::vector &DataRequestPacket::GetData() const -{ - return data_; -} - -void DataRequestPacket::SetEndWaterMark(WaterMark waterMark) +void SetMessageHeadInfo(Message &message, uint16_t inMsgType, const std::string &inTarget, uint32_t inSequenceId, + uint32_t inSessionId) { - endWaterMark_ = waterMark; + message.SetMessageType(inMsgType); + message.SetTarget(inTarget); + message.SetSequenceId(inSequenceId); + message.SetSessionId(inSessionId); } -WaterMark DataRequestPacket::GetEndWaterMark() const +inline bool IsGetDataSuccessfully(int errCode) { - return endWaterMark_; + return errCode == E_OK || errCode == -E_UNFINISHED; } - -void DataRequestPacket::SetLocalWaterMark(WaterMark waterMark) -{ - localWaterMark_ = waterMark; -} - -WaterMark DataRequestPacket::GetLocalWaterMark() const -{ - return localWaterMark_; } -void DataRequestPacket::SetPeerWaterMark(WaterMark waterMark) +SingleVerDataSync::SingleVerDataSync() + : mtuSize_(0), + storage_(nullptr), + communicateHandle_(nullptr), + metadata_(nullptr) { - peerWaterMark_ = waterMark; } -WaterMark DataRequestPacket::GetPeerWaterMark() const +SingleVerDataSync::~SingleVerDataSync() { - return peerWaterMark_; + storage_ = nullptr; + communicateHandle_ = nullptr; + metadata_ = nullptr; } -void DataRequestPacket::SetSendCode(int32_t errCode) +int SingleVerDataSync::Initialize(ISyncInterface *inStorage, ICommunicator *inCommunicateHandle, + std::shared_ptr &inMetadata, const std::string &deviceId) { - sendCode_ = errCode; + if ((inStorage == nullptr) || (inCommunicateHandle == nullptr) || (inMetadata == nullptr)) { + return -E_INVALID_ARGS; + } + storage_ = static_cast(inStorage); + communicateHandle_ = inCommunicateHandle; + metadata_ = inMetadata; + mtuSize_ = DBConstant::MIN_MTU_SIZE; // default size is 1K, it will update when need sync data. + std::vector label = inStorage->GetIdentifier(); + label.resize(3); // only show 3 Bytes enough + label_ = DBCommon::VectorToHexString(label); + deviceId_ = deviceId; + return E_OK; } -int32_t DataRequestPacket::GetSendCode() const +std::string SingleVerDataSync::GetLocalDeviceName() { - return sendCode_; + std::string deviceInfo; + if (communicateHandle_ != nullptr) { + communicateHandle_->GetLocalIdentity(deviceInfo); + } + return deviceInfo; } -void DataRequestPacket::SetMode(int32_t mode) +std::string SingleVerDataSync::TransferForeignOrigDevName(const std::string &deviceName, + const std::string &localHashName) { - mode_ = mode; + if (localHashName == deviceName) { + return ""; + } + return deviceName; } -int32_t DataRequestPacket::GetMode() const +int SingleVerDataSync::Send(SingleVerSyncTaskContext *context, const Message *message, const CommErrHandler &handler, + uint32_t packetLen) { - return mode_; + bool startFeedDogRet = false; + if (packetLen > mtuSize_ && mtuSize_ > NOTIFY_MIN_MTU_SIZE) { + uint32_t time = static_cast(static_cast(packetLen) * + static_cast(context->GetTimeoutTime()) / mtuSize_); // no overflow + startFeedDogRet = context->StartFeedDogForSync(time, SyncDirectionFlag::SEND); + } + int errCode = communicateHandle_->SendMessage(context->GetDeviceId(), message, false, SEND_TIME_OUT, handler); + if (errCode != E_OK) { + LOGE("[DataSync][Send] send message failed, errCode=%d", errCode); + if (startFeedDogRet) { + context->StopFeedDogForSync(SyncDirectionFlag::SEND); + } + } + return errCode; } -void DataRequestPacket::SetSessionId(uint32_t sessionId) +int SingleVerDataSync::GetData(SingleVerSyncTaskContext *context, std::vector &outData, size_t packetSize) { - sessionId_ = sessionId; + int errCode; + UpdateMtuSize(); + if (context->GetRetryStatus() == SyncTaskContext::NEED_RETRY) { + context->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); + LOGI("[DataSync][GetData] resend data"); + errCode = GetUnsyncData(context, outData, packetSize); + } else { + ContinueToken token; + context->GetContinueToken(token); + if (token == nullptr) { + errCode = GetUnsyncData(context, outData, packetSize); + } else { + LOGD("[DataSync][GetData] get data from token"); + // if there is data to send, read out data, and update local watermark, send data + errCode = GetNextUnsyncData(context, outData, packetSize); + } + } + if (errCode == -E_UNFINISHED) { + LOGD("[DataSync][GetData] not finished."); + } + if (IsGetDataSuccessfully(errCode)) { + TransDbDataItemToSendDataItem(context, outData); + } + return errCode; } -uint32_t DataRequestPacket::GetSessionId() const +int SingleVerDataSync::GetDataWithPerformanceRecord(SingleVerSyncTaskContext *context, SyncEntry &syncOutData) { - return sessionId_; -} + uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); + size_t packetSize = (version > SOFTWARE_VERSION_RELEASE_2_0) ? + DBConstant::MAX_HPMODE_PACK_ITEM_SIZE : DBConstant::MAX_NORMAL_PACK_ITEM_SIZE; + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); + PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); + if (performance != nullptr) { + performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_READ_DATA); + } + int errCode = GetData(context, syncOutData.entries, packetSize); + if (performance != nullptr) { + performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_READ_DATA); + } + if (!IsGetDataSuccessfully(errCode)) { + context->SetTaskErrCode(errCode); + return errCode; + } -void DataRequestPacket::SetVersion(uint32_t version) -{ - version_ = version; -} + int innerCode = InterceptData(syncOutData); + if (innerCode != E_OK) { + context->SetTaskErrCode(innerCode); + return innerCode; + } -uint32_t DataRequestPacket::GetVersion() const -{ - return version_; + CompressAlgorithm remoteAlgo = context->ChooseCompressAlgo(); + if (needCompressOnSync && remoteAlgo != CompressAlgorithm::NONE) { + int compressCode = GenericSingleVerKvEntry::Compress(syncOutData.entries, syncOutData.compressedEntries, + { remoteAlgo, version }); + if (compressCode != E_OK) { + return compressCode; + } + } + return errCode; } -void DataRequestPacket::SetReserved(std::vector &reserved) +int SingleVerDataSync::GetUnsyncData(SingleVerSyncTaskContext *context, std::vector &outData, + size_t packetSize) { - reserved_ = std::move(reserved); -} + int errCode; + WaterMark startMark = 0; + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + GetLocalWaterMark(curType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), context, startMark); + WaterMark endMark = MAX_TIMESTAMP; + if ((endMark == 0) || (startMark > endMark)) { + return E_OK; + } + ContinueToken token = nullptr; + context->GetContinueToken(token); + if (token != nullptr) { + storage_->ReleaseContinueToken(token); + } + DataSizeSpecInfo syncDataSizeInfo = GetDataSizeSpecInfo(packetSize); + if (curType != SyncType::QUERY_SYNC_TYPE) { + errCode = storage_->GetSyncData(startMark, endMark, outData, token, syncDataSizeInfo); + } else { + WaterMark deletedStartMark = 0; + GetLocalDeleteSyncWaterMark(context, deletedStartMark); -std::vector DataRequestPacket::GetReserved() const -{ - return reserved_; + QuerySyncObject queryObj = context->GetQuery(); + errCode = storage_->GetSyncData(queryObj, SyncTimeRange{ startMark, deletedStartMark, endMark, endMark}, + syncDataSizeInfo, token, outData); + } + context->SetContinueToken(token); + if (!IsGetDataSuccessfully(errCode)) { + LOGE("[DataSync][GetUnsyncData] get unsync data failed,errCode=%d", errCode); + } + return errCode; } -uint64_t DataRequestPacket::GetPacketId() const +int SingleVerDataSync::GetNextUnsyncData(SingleVerSyncTaskContext *context, std::vector &outData, + size_t packetSize) { - uint64_t packetId = 0; - std::vector DataRequestReserve = GetReserved(); - if (DataRequestReserve.size() > REQUEST_PACKET_RESERVED_INDEX_PACKETID) { - return DataRequestReserve[REQUEST_PACKET_RESERVED_INDEX_PACKETID]; - } else { - return packetId; + ContinueToken token; + context->GetContinueToken(token); + DataSizeSpecInfo syncDataSizeInfo = GetDataSizeSpecInfo(packetSize); + int errCode = storage_->GetSyncDataNext(outData, token, syncDataSizeInfo); + context->SetContinueToken(token); + if (!IsGetDataSuccessfully(errCode)) { + LOGE("[DataSync][GetNextUnsyncData] get next unsync data failed, errCode=%d", errCode); } + return errCode; } -uint32_t DataRequestPacket::CalculateLen() const +int SingleVerDataSync::SaveData(const SingleVerSyncTaskContext *context, const std::vector &inData, + SyncType curType, const QuerySyncObject &query) { - uint64_t totalLen = GenericSingleVerKvEntry::CalculateLens(data_, version_); // for data - totalLen += Parcel::GetUInt64Len(); // endWaterMark - totalLen += Parcel::GetUInt64Len(); // localWaterMark - totalLen += Parcel::GetUInt64Len(); // peerWaterMark - totalLen += Parcel::GetIntLen(); // sendCode - totalLen += Parcel::GetIntLen(); // mode - totalLen += Parcel::GetUInt32Len(); // sessionId - totalLen += Parcel::GetUInt32Len(); // version - totalLen += Parcel::GetVectorLen(reserved_); // reserved - - if (version_ > SOFTWARE_VERSION_RELEASE_2_0) { - totalLen += Parcel::GetUInt32Len(); // flag bit0 used for isLastSequence + if (inData.empty()) { + return E_OK; } - totalLen = Parcel::GetEightByteAlign(totalLen); // 8-byte align - if (totalLen > INT32_MAX) { - return 0; + PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); + if (performance != nullptr) { + performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SAVE_DATA); } - return totalLen; -} -void DataRequestPacket::SetFlag(uint32_t flag) -{ - flag_ = flag; + TransSendDataItemToLocal(context, inData); + int errCode = E_OK; + // query only support prefix key and don't have query in packet in 104 version + errCode = storage_->PutSyncDataWithQuery(query, inData, context->GetDeviceId()); + if (performance != nullptr) { + performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SAVE_DATA); + } + if (errCode != E_OK) { + LOGE("[DataSync][SaveData] save sync data failed,errCode=%d", errCode); + } + return errCode; } -uint32_t DataRequestPacket::GetFlag() const +TimeStamp SingleVerDataSync::GetMaxSendDataTime(const std::vector &inData) { - return flag_; + TimeStamp stamp = 0; + for (size_t i = 0; i < inData.size(); i++) { + if (inData[i] == nullptr) { + continue; + } + TimeStamp tempStamp = inData[i]->GetTimestamp(); + if (stamp < tempStamp) { + stamp = tempStamp; + } + } + return stamp; } -bool DataRequestPacket::IsLastSequence() const +SyncTimeRange SingleVerDataSync::GetSyncDataTimeRange(SyncType syncType, SingleVerSyncTaskContext *context, + const std::vector &inData, UpdateWaterMark &isUpdate) { - return flag_ & IS_LAST_SEQUENCE; + WaterMark localMark = 0; + WaterMark deleteMark = 0; + GetLocalWaterMark(syncType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), context, localMark); + GetLocalDeleteSyncWaterMark(context, deleteMark); + if (syncType != SyncType::QUERY_SYNC_TYPE) { + return GetFullSyncDataTimeRange(inData, localMark, isUpdate); + } + return GetQuerySyncDataTimeRange(inData, localMark, deleteMark, isUpdate); } -void DataRequestPacket::SetLastSequence() +SyncTimeRange SingleVerDataSync::GetRecvDataTimeRange(SyncType syncType, SingleVerSyncTaskContext *context, + const std::vector &data, UpdateWaterMark &isUpdate) { - flag_ = flag_ | IS_LAST_SEQUENCE; + if (syncType != SyncType::QUERY_SYNC_TYPE) { + return GetFullSyncDataTimeRange(data, 0, isUpdate); + } + return GetQuerySyncDataTimeRange(data, 0, 0, isUpdate); } -void DataRequestPacket::SetBasicInfo(int sendCode, uint32_t version, WaterMark localMark, WaterMark peerMark, - int32_t mode) +SyncTimeRange SingleVerDataSync::GetFullSyncDataTimeRange(const std::vector &inData, + WaterMark localMark, UpdateWaterMark &isUpdate) { - SetSendCode(sendCode); - SetVersion(version); - SetLocalWaterMark(localMark); - SetPeerWaterMark(peerMark); - SetMode(mode); + TimeStamp maxTimeStamp = localMark; + TimeStamp minTimeStamp = localMark; + for (size_t i = 0; i < inData.size(); i++) { + if (inData[i] == nullptr) { + continue; + } + TimeStamp tempStamp = inData[i]->GetTimestamp(); + if (maxTimeStamp < tempStamp) { + maxTimeStamp = tempStamp; + } + if (minTimeStamp > tempStamp) { + minTimeStamp = tempStamp; + } + isUpdate.normalUpdateMark = true; + } + return {minTimeStamp, 0, maxTimeStamp, 0}; } -void DataAckPacket::SetData(uint64_t data) +SyncTimeRange SingleVerDataSync::GetQuerySyncDataTimeRange(const std::vector &inData, + WaterMark localMark, WaterMark deletelocalMark, UpdateWaterMark &isUpdate) { - data_ = data; + SyncTimeRange dataTimeRange = {localMark, deletelocalMark, localMark, deletelocalMark}; + for (size_t i = 0; i < inData.size(); i++) { + if (inData[i] == nullptr) { + continue; + } + TimeStamp tempStamp = inData[i]->GetTimestamp(); + if ((inData[i]->GetFlag() & DataItem::DELETE_FLAG) == 0) { // query data + if (dataTimeRange.endTime < tempStamp) { + dataTimeRange.endTime = tempStamp; + } + if (dataTimeRange.beginTime > tempStamp) { + dataTimeRange.beginTime = tempStamp; + } + isUpdate.normalUpdateMark = true; + } + if ((inData[i]->GetFlag() & DataItem::DELETE_FLAG) != 0) { // delete data + if (dataTimeRange.deleteEndTime < tempStamp) { + dataTimeRange.deleteEndTime = tempStamp; + } + if (dataTimeRange.deleteBeginTime > tempStamp) { + dataTimeRange.deleteBeginTime = tempStamp; + } + isUpdate.deleteUpdateMark = true; + } + } + return dataTimeRange; } -uint64_t DataAckPacket::GetData() const +TimeStamp SingleVerDataSync::GetMinSendDataTime(const std::vector &inData, WaterMark localMark) { - return data_; + TimeStamp stamp = localMark; + for (size_t i = 0; i < inData.size(); i++) { + if (inData[i] == nullptr) { + continue; + } + TimeStamp tempStamp = inData[i]->GetTimestamp(); + if (stamp > tempStamp) { + stamp = tempStamp; + } + } + return stamp; } -void DataAckPacket::SetRecvCode(int32_t errorCode) +SyncTimeRange SingleVerDataSync::ReviseLocalMark(SyncType syncType, SyncTimeRange &dataTimeRange, + UpdateWaterMark updateMark) { - recvCode_ = errorCode; + SyncTimeRange tmpDataTime = dataTimeRange; + if (updateMark.deleteUpdateMark && syncType == SyncType::QUERY_SYNC_TYPE) { + tmpDataTime.deleteEndTime += 1; + } + if (updateMark.normalUpdateMark) { + tmpDataTime.endTime += 1; + } + return tmpDataTime; } -int32_t DataAckPacket::GetRecvCode() const +int SingleVerDataSync::SaveLocalWaterMark(SyncType syncType, const SingleVerSyncTaskContext *context, + SyncTimeRange dataTimeRange, bool isCheckBeforUpdate) const { - return recvCode_; + WaterMark localMark = 0; + int errCode = E_OK; + const std::string &deviceId = context->GetDeviceId(); + std::string queryId = GetQuerySyncId(context, context->GetQuery().GetIdentify()); + if (syncType != SyncType::QUERY_SYNC_TYPE) { + if (isCheckBeforUpdate) { + GetLocalWaterMark(syncType, queryId, context, localMark); + if (localMark >= dataTimeRange.endTime) { + return E_OK; + } + } + errCode = metadata_->SaveLocalWaterMark(deviceId, dataTimeRange.endTime); + } else { + bool isNeedUpdateMark = true; + bool isNeedUpdateDeleteMark = true; + if (isCheckBeforUpdate) { + WaterMark deleteDataWaterMark = 0; + GetLocalWaterMark(syncType, queryId, context, localMark); + GetLocalDeleteSyncWaterMark(context, deleteDataWaterMark); + if (localMark >= dataTimeRange.endTime) { + isNeedUpdateMark = false; + } + if (deleteDataWaterMark >= dataTimeRange.deleteEndTime) { + isNeedUpdateDeleteMark = false; + } + } + if (isNeedUpdateMark) { + LOGD("label=%s,dev=%s,endTime=%llu", label_.c_str(), STR_MASK(GetDeviceId()), dataTimeRange.endTime); + errCode = metadata_->SetSendQueryWaterMark(queryId, deviceId, dataTimeRange.endTime); + if (errCode != E_OK) { + LOGE("[DataSync][SaveLocalWaterMark] save query metadata watermark failed,errCode=%d", errCode); + return errCode; + } + } + if (isNeedUpdateDeleteMark) { + LOGD("label=%s,dev=%s,deleteEndTime=%llu", label_.c_str(), STR_MASK(GetDeviceId()), + dataTimeRange.deleteEndTime); + errCode = metadata_->SetSendDeleteSyncWaterMark(GetDeleteSyncId(context), dataTimeRange.deleteEndTime); + } + } + if (errCode != E_OK) { + LOGE("[DataSync][SaveLocalWaterMark] save metadata local watermark failed,errCode=%d", errCode); + } + return errCode; } -void DataAckPacket::SetVersion(uint32_t version) +void SingleVerDataSync::GetPeerWaterMark(SyncType syncType, const std::string &queryIdentify, + const DeviceID &deviceId, WaterMark &waterMark) const { - version_ = version; + if (syncType != SyncType::QUERY_SYNC_TYPE) { + metadata_->GetPeerWaterMark(deviceId, waterMark); + return; + } + metadata_->GetRecvQueryWaterMark(queryIdentify, deviceId, waterMark); } -uint32_t DataAckPacket::GetVersion() const +void SingleVerDataSync::GetPeerDeleteSyncWaterMark(const DeviceID &deviceId, WaterMark &waterMark) { - return version_; + metadata_->GetRecvDeleteSyncWaterMark(deviceId, waterMark); } -void DataAckPacket::SetReserved(std::vector &reserved) +void SingleVerDataSync::GetLocalDeleteSyncWaterMark(const SingleVerSyncTaskContext *context, + WaterMark &waterMark) const { - reserved_ = std::move(reserved); + metadata_->GetSendDeleteSyncWaterMark(GetDeleteSyncId(context), waterMark, context->IsAutoLiftWaterMark()); } -std::vector DataAckPacket::GetReserved() const +void SingleVerDataSync::GetLocalWaterMark(SyncType syncType, const std::string &queryIdentify, + const SingleVerSyncTaskContext *context, WaterMark &waterMark) const { - return reserved_; + if (syncType != SyncType::QUERY_SYNC_TYPE) { + metadata_->GetLocalWaterMark(context->GetDeviceId(), waterMark); + return; + } + metadata_->GetSendQueryWaterMark(queryIdentify, context->GetDeviceId(), + waterMark, context->IsAutoLiftWaterMark()); } -uint64_t DataAckPacket::GetPacketId() const +int SingleVerDataSync::RemoveDeviceDataHandle(SingleVerSyncTaskContext *context, const Message *message, + WaterMark maxSendDataTime) { - uint64_t packetId = 0; - std::vector DataAckReserve = GetReserved(); - if (DataAckReserve.size() > ACK_PACKET_RESERVED_INDEX_PACKETID) { - return DataAckReserve[ACK_PACKET_RESERVED_INDEX_PACKETID]; + bool isNeedClearRemoteData = false; + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_3_0) { + uint64_t clearDeviceDataMark = 0; + metadata_->GetRemoveDataMark(context->GetDeviceId(), clearDeviceDataMark); + isNeedClearRemoteData = (clearDeviceDataMark == REMOVE_DEVICE_DATA_MARK); } else { - return packetId; + const DataRequestPacket *packet = message->GetObject(); + if (packet == nullptr) { + LOGE("[RemoveDeviceDataHandle] get packet object failed"); + return -E_INVALID_ARGS; + } + SyncType curType = SyncOperation::GetSyncType(packet->GetMode()); + WaterMark packetLocalMark = packet->GetLocalWaterMark(); + WaterMark peerMark = 0; + GetPeerWaterMark(curType, GetQuerySyncId(context, packet->GetQueryId()), context->GetDeviceId(), peerMark); + isNeedClearRemoteData = ((packetLocalMark == 0) && (peerMark != 0)); } + if (!isNeedClearRemoteData) { + return E_OK; + } + int errCode = E_OK; + if (context->IsNeedClearRemoteStaleData()) { + // need to clear remote device history data + errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); + if (errCode != E_OK) { + (void)SendDataAck(context, message, errCode, maxSendDataTime); + return errCode; + } + if (context->GetRemoteSoftwareVersion() == SOFTWARE_VERSION_EARLIEST) { + // avoid repeat clear in ack + metadata_->SaveLocalWaterMark(context->GetDeviceId(), 0); + } + } + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_3_0) { + errCode = metadata_->ResetMetaDataAfterRemoveData(context->GetDeviceId()); + if (errCode != E_OK) { + (void)SendDataAck(context, message, errCode, maxSendDataTime); + return errCode; + } + } + return E_OK; } -bool DataAckPacket::IsPacketIdValid(uint64_t packetId) -{ - return (packetId > 0); -} - -uint32_t DataAckPacket::CalculateLen() const +int SingleVerDataSync::DealRemoveDeviceDataByAck(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, + const std::vector &reserved) { - uint64_t len = Parcel::GetUInt64Len(); // ackWaterMark - len += Parcel::GetIntLen(); // recvCode - len += Parcel::GetUInt32Len(); // version - len += Parcel::GetVectorLen(reserved_); // reserved - - len = Parcel::GetEightByteAlign(len); - if (len > INT32_MAX) { - return 0; + bool isNeedClearRemoteData = false; + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_3_0) { + uint64_t clearDeviceDataMark = 0; + metadata_->GetRemoveDataMark(context->GetDeviceId(), clearDeviceDataMark); + isNeedClearRemoteData = (clearDeviceDataMark != 0); + } else if (reserved.empty()) { + WaterMark localMark = 0; + GetLocalWaterMark(curType, context->GetQuery().GetIdentify(), context, localMark); + isNeedClearRemoteData = ((localMark != 0) && (ackWaterMark == 0)); + } else { + WaterMark peerMark = 0; + GetPeerWaterMark(curType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), + context->GetDeviceId(), peerMark); + isNeedClearRemoteData = ((reserved[ACK_PACKET_RESERVED_INDEX_LOCAL_WATER_MARK] == 0) && (peerMark != 0)); + } + if (!isNeedClearRemoteData) { + return E_OK; } - return len; + // need to clear remote historydata + LOGI("[DataSync][WaterMarkException] AckRecv reserved not empty,rebuilted,clear historydata,label=%s,dev=%s", + label_.c_str(), STR_MASK(GetDeviceId())); + int errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); + if (errCode != E_OK) { + return errCode; + } + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_3_0) { + errCode = metadata_->ResetMetaDataAfterRemoveData(context->GetDeviceId()); + } + return errCode; } -SingleVerDataSync::SingleVerDataSync() - : mtuSize_(0), - storage_(nullptr), - communicateHandle_(nullptr), - metadata_(nullptr) -{ -} - -SingleVerDataSync::~SingleVerDataSync() +void SingleVerDataSync::TransDbDataItemToSendDataItem(const SingleVerSyncTaskContext *context, + std::vector &outData) { - storage_ = nullptr; - communicateHandle_ = nullptr; - metadata_ = nullptr; + std::string localHashName = DBCommon::TransferHashString(GetLocalDeviceName()); + for (size_t i = 0; i < outData.size(); i++) { + if (outData[i] == nullptr) { + continue; + } + outData[i]->SetOrigDevice(outData[i]->GetOrigDevice().empty() ? localHashName : outData[i]->GetOrigDevice()); + if (i == 0 || i == (outData.size() - 1)) { + LOGD("[DataSync][TransToSendItem] printData packet=%d,timeStamp=%llu,flag=%llu", i, + outData[i]->GetTimestamp(), outData[i]->GetFlag()); + } + } } -int SingleVerDataSync::RegisterTransformFunc() +void SingleVerDataSync::TransSendDataItemToLocal(const SingleVerSyncTaskContext *context, + const std::vector &data) { - TransformFunc func; - func.computeFunc = std::bind(&SingleVerDataSync::CalculateLen, std::placeholders::_1); - func.serializeFunc = std::bind(&SingleVerDataSync::Serialization, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3); - func.deserializeFunc = std::bind(&SingleVerDataSync::DeSerialization, std::placeholders::_1, - std::placeholders::_2, std::placeholders::_3); - return MessageTransform::RegTransformFunction(DATA_SYNC_MESSAGE, func); -} + TimeOffset offset = context->GetTimeOffset(); + TimeStamp currentLocalTime = context->GetCurrentLocalTime(); + const std::string localHashName = DBCommon::TransferHashString(GetLocalDeviceName()); + for (auto &item : data) { + if (item == nullptr) { + continue; + } + item->SetOrigDevice(TransferForeignOrigDevName(item->GetOrigDevice(), localHashName)); + TimeStamp tempTimestamp = item->GetTimestamp(); + TimeStamp tempWriteTimestamp = item->GetWriteTimestamp(); + item->SetTimestamp(tempTimestamp - offset); + if (tempWriteTimestamp != 0) { + item->SetWriteTimestamp(tempWriteTimestamp - offset); + } -int SingleVerDataSync::Initialize(IKvDBSyncInterface *inStorage, ICommunicator *inCommunicateHandle, - std::shared_ptr &inMetadata, const std::string &deviceId) -{ - if ((inStorage == nullptr) || (inCommunicateHandle == nullptr) || (inMetadata == nullptr)) { - return -E_INVALID_ARGS; + if (item->GetTimestamp() > currentLocalTime) { + item->SetTimestamp(currentLocalTime); + } + if (item->GetWriteTimestamp() > currentLocalTime) { + item->SetWriteTimestamp(currentLocalTime); + } } - storage_ = static_cast(inStorage); - communicateHandle_ = inCommunicateHandle; - metadata_ = inMetadata; - mtuSize_ = communicateHandle_->GetCommunicatorMtuSize(deviceId) * 9 / 10; // get the 9/10 of the size. - std::vector label = inStorage->GetIdentifier(); - label.resize(3); // only show 3 Bytes enough - label_ = DBCommon::VectorToHexString(label); - deviceId_ = deviceId; - return E_OK; } -bool SingleVerDataSync::IsPacketValid(const Message *inMsg) +void SingleVerDataSync::FillDataRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, + SyncEntry &syncData, int sendCode, int mode) { - if (inMsg == nullptr) { - return false; + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); + WaterMark localMark = 0; + WaterMark peerMark = 0; + WaterMark deleteMark = 0; + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); + std::string id = GetQuerySyncId(context, context->GetQuery().GetIdentify()); + GetLocalWaterMark(curType, id, context, localMark); + GetPeerWaterMark(curType, id, context->GetDeviceId(), peerMark); + GetLocalDeleteSyncWaterMark(context, deleteMark); + if (((mode != SyncModeType::RESPONSE_PULL && sendCode == E_OK)) || + (mode == SyncModeType::RESPONSE_PULL && sendCode == SEND_FINISHED)) { + packet->SetLastSequence(); } - if (inMsg->GetMessageId() != DATA_SYNC_MESSAGE) { - LOGE("[DataSync][IsPacketValid] Message Id ERROR! messageId=%d", inMsg->GetMessageId()); - return false; + int tmpMode = mode; + if (mode == SyncModeType::RESPONSE_PULL) { + tmpMode = (curType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH : SyncModeType::PUSH; + } + packet->SetData(syncData.entries); + packet->SetCompressData(syncData.compressedEntries); + packet->SetBasicInfo(sendCode, version, tmpMode); + packet->SetWaterMark(localMark, peerMark, deleteMark); + if (SyncOperation::TransferSyncMode(mode) == SyncModeType::PUSH_AND_PULL) { + packet->SetEndWaterMark(context->GetEndMark()); + packet->SetSessionId(context->GetRequestSessionId()); + } + packet->SetQuery(context->GetQuery()); + packet->SetQueryId(GetQuerySyncId(context, context->GetQuery().GetIdentify())); + CompressAlgorithm curAlgo = context->ChooseCompressAlgo(); + if (needCompressOnSync && curAlgo != CompressAlgorithm::NONE) { + packet->SetCompressDataMark(); + packet->SetCompressAlgo(curAlgo); } - int msgType = inMsg->GetMessageType(); - if (msgType != TYPE_REQUEST && msgType != TYPE_RESPONSE && msgType != TYPE_NOTIFY) { - LOGE("[DataSync][IsPacketValid] Message type ERROR! message type=%d", msgType); - return false; + SetPacketId(packet, context, version); + if (curType == SyncType::QUERY_SYNC_TYPE && (context->GetQuery().HasLimit() || + context->GetQuery().HasOrderBy())) { + packet->SetUpdateWaterMark(); } - return true; + LOGD("[DataSync] curType=%d,local=%llu,del=%llu,end=%llu,label=%s,dev=%s,queryId=%s,isCompress=%d", curType, + localMark, deleteMark, context->GetEndMark(), label_.c_str(), STR_MASK(GetDeviceId()), + STR_MASK(context->GetQuery().GetIdentify()), packet->IsCompressData()); } -int SingleVerDataSync::Serialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +int SingleVerDataSync::RequestStart(SingleVerSyncTaskContext *context, int mode) { - if ((buffer == nullptr) || !(IsPacketValid(inMsg))) { - return -E_MESSAGE_ID_ERROR; + if (!QuerySyncCheck(context)) { + context->SetTaskErrCode(-E_NOT_SUPPORT); + return -E_NOT_SUPPORT; } - - switch (inMsg->GetMessageType()) { - case TYPE_REQUEST: - return DataPacketSerialization(buffer, length, inMsg); - case TYPE_RESPONSE: - case TYPE_NOTIFY: - return AckPacketSerialization(buffer, length, inMsg); - default: - return -E_MESSAGE_TYPE_ERROR; + int errCode = RemoveDeviceDataIfNeed(context); + if (errCode != E_OK) { + return errCode; } -} - -int SingleVerDataSync::DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) -{ - if ((buffer == nullptr) || !(IsPacketValid(inMsg))) { - return -E_MESSAGE_ID_ERROR; + SyncEntry syncData; + // get data + errCode = GetDataWithPerformanceRecord(context, syncData); + // once get data occur E_EKEYREVOKED error, should also send request to remote dev to pull data. + if (SyncOperation::TransferSyncMode(mode) == SyncModeType::PUSH_AND_PULL && + context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && errCode == -E_EKEYREVOKED) { + errCode = E_OK; } - switch (inMsg->GetMessageType()) { - case TYPE_REQUEST: - return DataPacketDeSerialization(buffer, length, inMsg); - case TYPE_RESPONSE: - case TYPE_NOTIFY: - return AckPacketDeSerialization(buffer, length, inMsg); - default: - return -E_MESSAGE_TYPE_ERROR; + if (!IsGetDataSuccessfully(errCode)) { + LOGE("[DataSync][PushStart] get data failed, errCode=%d", errCode); + return errCode; } -} -uint32_t SingleVerDataSync::CalculateLen(const Message *inMsg) -{ - if (!(IsPacketValid(inMsg))) { - return 0; + DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; + if (packet == nullptr) { + LOGE("[DataSync][PushStart] new DataRequestPacket error"); + return -E_OUT_OF_MEMORY; } - uint32_t len = 0; - int errCode; - switch (inMsg->GetMessageType()) { - case TYPE_REQUEST: - errCode = DataPacketCalculateLen(inMsg, len); - if (errCode != E_OK) { - LOGE("[DataSync][CalculateLen] calculate data request packet len failed, errCode=%d", errCode); - return 0; - } - return len; - case TYPE_RESPONSE: - case TYPE_NOTIFY: - errCode = AckPacketCalculateLen(inMsg, len); - if (errCode != E_OK) { - LOGE("[DataSync][CalculateLen] calculate data notify packet len failed errCode=%d", errCode); - return 0; - } - return len; - default: - return 0; + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + UpdateWaterMark isUpdateWaterMark; + SyncTimeRange dataTime = GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); + context->SetSequenceStartAndEndTimeStamp(dataTime); + if (errCode == E_OK) { + context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } -} - -int SingleVerDataSync::DataPacketCalculateLen(const Message *inMsg, uint32_t &len) -{ - const DataRequestPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; + FillDataRequestPacket(packet, context, syncData, errCode, mode); + errCode = SendDataPacket(curType, packet, context); + PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); + if (performance != nullptr) { + performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_MACHINE_START_TO_PUSH_SEND); } - - len = packet->CalculateLen(); - return E_OK; -} - -int SingleVerDataSync::AckPacketCalculateLen(const Message *inMsg, uint32_t &len) -{ - const DataAckPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; + if (errCode == E_OK) { + if (curType == SyncType::QUERY_SYNC_TYPE && (context->GetQuery().HasLimit() || + context->GetQuery().HasOrderBy())) { + LOGI("[DataSync][RequestStart] query contain limit/offset/orderby, no need to update watermark."); + return E_OK; + } + SyncTimeRange tmpDataTime = ReviseLocalMark(curType, dataTime, isUpdateWaterMark); + SaveLocalWaterMark(curType, context, tmpDataTime); } - - len = packet->CalculateLen(); - return E_OK; + return errCode; } -std::string SingleVerDataSync::GetLocalDeviceName() +int SingleVerDataSync::PushStart(SingleVerSyncTaskContext *context) { - std::string deviceInfo; - if (communicateHandle_ != nullptr) { - communicateHandle_->GetLocalIdentity(deviceInfo); + if (context == nullptr) { + return -E_INVALID_ARGS; } - return deviceInfo; + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + return RequestStart(context, + (curType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH : SyncModeType::PUSH); } -std::string SingleVerDataSync::TransferForeignOrigDevName(const std::string &deviceName) +int SingleVerDataSync::PushPullStart(SingleVerSyncTaskContext *context) { - // Get the hash device name. - std::string localName = GetLocalDeviceName(); - if (DBCommon::TransferHashString(localName) == deviceName) { - return ""; + if (context == nullptr) { + return -E_INVALID_ARGS; } - return deviceName; + return RequestStart(context, context->GetMode()); } -std::string SingleVerDataSync::TransferLocalOrigDevName(const std::string &origName) +int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) { - std::string localName = GetLocalDeviceName(); - if (origName.empty()) { - return DBCommon::TransferHashString(localName); + if (context == nullptr) { + return -E_INVALID_ARGS; } - return origName; -} - -int SingleVerDataSync::DataPacketSyncerPartSerialization(Parcel &parcel, const DataRequestPacket *packet) -{ - // endWaterMark - int errCode = parcel.WriteUInt64(packet->GetEndWaterMark()); - if (errCode != E_OK) { - return errCode; + if (!QuerySyncCheck(context)) { + context->SetTaskErrCode(-E_NOT_SUPPORT); + return -E_NOT_SUPPORT; } - // localWaterMark - errCode = parcel.WriteUInt64(packet->GetLocalWaterMark()); + int errCode = RemoveDeviceDataIfNeed(context); if (errCode != E_OK) { return errCode; } - // peerWaterMark - errCode = parcel.WriteUInt64(packet->GetPeerWaterMark()); - if (errCode != E_OK) { - return errCode; + DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; + if (packet == nullptr) { + LOGE("[DataSync][PullRequest]new DataRequestPacket error"); + return -E_OUT_OF_MEMORY; } - // sendCode - errCode = parcel.WriteInt(packet->GetSendCode()); - if (errCode != E_OK) { - return errCode; + SyncType syncType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + WaterMark peerMark = 0; + WaterMark localMark = 0; + WaterMark deleteMark = 0; + GetPeerWaterMark(syncType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), + context->GetDeviceId(), peerMark); + GetLocalWaterMark(syncType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), context, localMark); + GetLocalDeleteSyncWaterMark(context, deleteMark); + uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); + WaterMark endMark = context->GetEndMark(); + SyncTimeRange dataTime = {localMark, deleteMark, localMark, deleteMark}; + context->SetSequenceStartAndEndTimeStamp(dataTime); + + packet->SetBasicInfo(E_OK, version, context->GetMode()); + packet->SetWaterMark(localMark, peerMark, deleteMark); + packet->SetEndWaterMark(endMark); + packet->SetSessionId(context->GetRequestSessionId()); + packet->SetQuery(context->GetQuery()); + packet->SetQueryId(GetQuerySyncId(context, context->GetQuery().GetIdentify())); + packet->SetLastSequence(); + SetPacketId(packet, context, version); + + LOGD("[DataSync][Pull] curType=%d,local=%llu,del=%llu,end=%llu,peer=%llu,label=%s,dev=%s", syncType, localMark, + deleteMark, peerMark, endMark, label_.c_str(), STR_MASK(GetDeviceId())); + return SendDataPacket(syncType, packet, context); +} + +int SingleVerDataSync::PullResponseStart(SingleVerSyncTaskContext *context) +{ + if (context == nullptr) { + return -E_INVALID_ARGS; } - // mode - errCode = parcel.WriteInt(packet->GetMode()); - if (errCode != E_OK) { + SyncEntry syncData; + // get data + int errCode = GetDataWithPerformanceRecord(context, syncData); + if (!IsGetDataSuccessfully(errCode)) { + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { + SendPullResponseDataPkt(errCode, syncData, context); + } return errCode; } - // sessionId - errCode = parcel.WriteUInt32(packet->GetSessionId()); - if (errCode != E_OK) { - return errCode; + // if send finished + int ackCode = E_OK; + ContinueToken token = nullptr; + context->GetContinueToken(token); + if (errCode == E_OK && token == nullptr) { + LOGD("[DataSync][PullResponse] send last frame end"); + ackCode = SEND_FINISHED; } - // reserved - errCode = parcel.WriteVector(packet->GetReserved()); - if (errCode != E_OK) { - return errCode; + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + UpdateWaterMark isUpdateWaterMark; + SyncTimeRange dataTime = + GetSyncDataTimeRange(curType, context, syncData.entries, isUpdateWaterMark); + context->SetSequenceStartAndEndTimeStamp(dataTime); + if (errCode == E_OK) { + context->SetSessionEndTimeStamp(std::max(dataTime.endTime, dataTime.deleteEndTime)); } - if (packet->GetVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - errCode = parcel.WriteUInt32(packet->GetFlag()); - if (errCode != E_OK) { - return errCode; + errCode = SendPullResponseDataPkt(ackCode, syncData, context); + if (errCode == E_OK) { + if (curType == SyncType::QUERY_SYNC_TYPE && (context->GetQuery().HasLimit() || + context->GetQuery().HasOrderBy())) { + LOGI("[DataSync][PullResponseStart] query contain limit/offset/orderby, no need to update watermark."); + return E_OK; } + SyncTimeRange tmpDataTime = ReviseLocalMark(curType, dataTime, isUpdateWaterMark); + SaveLocalWaterMark(curType, context, tmpDataTime); } - parcel.EightByteAlign(); return errCode; } -int SingleVerDataSync::DataPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +void SingleVerDataSync::UpdateQueryPeerWaterMark(SyncType syncType, const std::string &queryId, SyncTimeRange &dataTime, + const SingleVerSyncTaskContext *context, UpdateWaterMark isUpdateWaterMark) { - const DataRequestPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; + WaterMark tmpPeerWatermark = dataTime.endTime; + WaterMark tmpPeerDeletedWatermark = dataTime.deleteEndTime; + if (isUpdateWaterMark.normalUpdateMark) { + tmpPeerWatermark++; } - Parcel parcel(buffer, length); + if (isUpdateWaterMark.deleteUpdateMark) { + tmpPeerDeletedWatermark++; + } + UpdatePeerWaterMark(syncType, queryId, context, tmpPeerWatermark, tmpPeerDeletedWatermark); +} - // version - int errCode = parcel.WriteUInt32(packet->GetVersion()); - if (errCode != E_OK) { - return errCode; +void SingleVerDataSync::UpdatePeerWaterMark(SyncType syncType, const std::string &queryId, + const SingleVerSyncTaskContext *context, WaterMark peerWatermark, WaterMark peerDeletedWatermark) +{ + if (peerWatermark == 0 && peerDeletedWatermark == 0) { + return; + } + int errCode = E_OK; + if (syncType != SyncType::QUERY_SYNC_TYPE) { + errCode = metadata_->SavePeerWaterMark(context->GetDeviceId(), peerWatermark, true); + } else { + if (peerWatermark != 0) { + LOGD("label=%s,dev=%s,endTime=%llu", label_.c_str(), STR_MASK(GetDeviceId()), peerWatermark); + errCode = metadata_->SetRecvQueryWaterMark(queryId, context->GetDeviceId(), peerWatermark); + if (errCode != E_OK) { + LOGE("[DataSync][UpdatePeerWaterMark] save query peer water mark failed,errCode=%d", errCode); + } + } + if (peerDeletedWatermark != 0) { + LOGD("label=%s,dev=%s,peerDeletedTime=%llu", + label_.c_str(), STR_MASK(GetDeviceId()), peerDeletedWatermark); + errCode = metadata_->SetRecvDeleteSyncWaterMark(GetDeleteSyncId(context), peerDeletedWatermark); + } } - // sendDataItems - const std::vector &data = packet->GetData(); - errCode = GenericSingleVerKvEntry::SerializeDatas(data, parcel, packet->GetVersion()); if (errCode != E_OK) { - return errCode; + LOGE("[DataSync][UpdatePeerWaterMark] save peer water mark failed,errCode=%d", errCode); } - return SingleVerDataSync::DataPacketSyncerPartSerialization(parcel, packet); } -int SingleVerDataSync::DataPacketSyncerPartDeSerialization(Parcel &parcel, DataRequestPacket *packet, - uint32_t packLen, uint32_t length, uint32_t version) +int SingleVerDataSync::DoAbilitySyncIfNeed(SingleVerSyncTaskContext *context, const Message *message, bool isControlMsg) { - WaterMark waterMark; - WaterMark localWaterMark; - WaterMark peerWaterMark; - int32_t sendCode; - int32_t mode; - uint32_t sessionId; - uint32_t flag = 0; - std::vector reserved; - - packLen += parcel.ReadUInt64(waterMark); - packLen += parcel.ReadUInt64(localWaterMark); - packLen += parcel.ReadUInt64(peerWaterMark); - packLen += parcel.ReadInt(sendCode); - packLen += parcel.ReadInt(mode); - packLen += parcel.ReadUInt32(sessionId); - packLen += parcel.ReadVector(reserved); - if (version > SOFTWARE_VERSION_RELEASE_2_0) { - packLen += parcel.ReadUInt32(flag); - packet->SetFlag(flag); - } - packLen = Parcel::GetEightByteAlign(packLen); - if (packLen != length || parcel.IsError()) { - LOGE("[DataSync][DataPacketDeSerialization] deserialize failed! input len=%lu,packLen=%lu", length, packLen); - return -E_LENGTH_ERROR; - } - packet->SetEndWaterMark(waterMark); - packet->SetLocalWaterMark(localWaterMark); - packet->SetPeerWaterMark(peerWaterMark); - packet->SetSendCode(sendCode); - packet->SetMode(mode); - packet->SetSessionId(sessionId); - packet->SetReserved(reserved); - return E_OK; + uint16_t remoteCommunicatorVersion = 0; + if (communicateHandle_->GetRemoteCommunicatorVersion(context->GetDeviceId(), remoteCommunicatorVersion) == + -E_NOT_FOUND) { + LOGE("[DataSync][DoAbilitySyncIfNeed] get remote communicator version failed"); + return -E_VERSION_NOT_SUPPORT; + } + // If remote is not the first version, we need check SOFTWARE_VERSION_BASE + if (remoteCommunicatorVersion == 0) { + LOGI("[DataSync] set remote version 0"); + context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_EARLIEST); + return E_OK; + } else { + LOGI("[DataSync][DoAbilitySyncIfNeed] need do ability sync"); + if (isControlMsg) { + SendControlAck(context, message, -E_NEED_ABILITY_SYNC, 0); + } else { + SendDataAck(context, message, -E_NEED_ABILITY_SYNC, 0); + } + return -E_WAIT_NEXT_MESSAGE; + } } -int SingleVerDataSync::DataPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +int SingleVerDataSync::DataRequestRecvPre(SingleVerSyncTaskContext *context, const Message *message) { - std::vector dataItems; - uint32_t version; - Parcel parcel(const_cast(buffer), length); - uint32_t packLen = parcel.ReadUInt32(version); - if (parcel.IsError()) { + if (context == nullptr || message == nullptr) { return -E_INVALID_ARGS; } - DataRequestPacket *packet = new (std::nothrow) DataRequestPacket(); + auto *packet = message->GetObject(); if (packet == nullptr) { - return -E_OUT_OF_MEMORY; + return -E_INVALID_ARGS; } - if (version > SOFTWARE_VERSION_CURRENT) { - packet->SetVersion(version); - packet->SetSendCode(-E_VERSION_NOT_SUPPORT); - int errCode = inMsg->SetExternalObject<>(packet); - if (errCode != E_OK) { - delete packet; - packet = nullptr; - } - return errCode; + if (context->GetRemoteSoftwareVersion() <= SOFTWARE_VERSION_BASE) { + return DoAbilitySyncIfNeed(context, message); + } + int32_t sendCode = packet->GetSendCode(); + if (sendCode == -E_VERSION_NOT_SUPPORT) { + LOGE("[DataSync] Version mismatch: ver=%u, current=%u", packet->GetVersion(), SOFTWARE_VERSION_CURRENT); + (void)SendDataAck(context, message, -E_VERSION_NOT_SUPPORT, 0); + return -E_WAIT_NEXT_MESSAGE; + } + // only deal with pull response packet errCode + if (sendCode != E_OK && sendCode != SEND_FINISHED && + message->GetSessionId() == context->GetRequestSessionId()) { + LOGE("[DataSync][DataRequestRecvPre] remote pullResponse getData sendCode=%d", sendCode); + return sendCode; } - packet->SetVersion(version); - packLen += GenericSingleVerKvEntry::DeSerializeDatas(dataItems, parcel); - packet->SetData(dataItems); - int errCode = SingleVerDataSync::DataPacketSyncerPartDeSerialization(parcel, packet, packLen, length, version); + int errCode = RunPermissionCheck(context, message, packet); if (errCode != E_OK) { - delete packet; - packet = nullptr; return errCode; } - errCode = inMsg->SetExternalObject<>(packet); + if (std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT) > SOFTWARE_VERSION_RELEASE_2_0) { + errCode = CheckSchemaStrategy(context, message); + } + if (errCode == E_OK) { + errCode = RequestQueryCheck(packet); + } if (errCode != E_OK) { - delete packet; - packet = nullptr; + (void)SendDataAck(context, message, errCode, 0); } return errCode; } -int SingleVerDataSync::AckPacketSyncerPartSerializationV1(Parcel &parcel, const DataAckPacket *packet) +int SingleVerDataSync::DataRequestRecv(SingleVerSyncTaskContext *context, const Message *message, + WaterMark &pullEndWatermark) { - int errCode = parcel.WriteUInt64(packet->GetData()); + int errCode = DataRequestRecvPre(context, message); if (errCode != E_OK) { return errCode; } - errCode = parcel.WriteInt(packet->GetRecvCode()); + const DataRequestPacket *packet = message->GetObject(); + const std::vector &data = packet->GetData(); + SyncType curType = SyncOperation::GetSyncType(packet->GetMode()); + LOGI("[DataSync][DataRequestRecv] curType=%d,remote ver=%u,size=%d,errCode=%d,queryId=%s,Label=%s,dev=%s", curType, + packet->GetVersion(), data.size(), packet->GetSendCode(), STR_MASK(packet->GetQueryId()), label_.c_str(), + STR_MASK(GetDeviceId())); + context->SetReceiveWaterMarkErr(false); + UpdateWaterMark isUpdateWaterMark; + SyncTimeRange dataTime = GetRecvDataTimeRange(curType, context, data, isUpdateWaterMark); + errCode = RemoveDeviceDataHandle(context, message, dataTime.endTime); if (errCode != E_OK) { return errCode; } - errCode = parcel.WriteVector(packet->GetReserved()); + if (WaterMarkErrHandle(curType, context, message)) { + return E_OK; + } + GetPullEndWatermark(context, packet, pullEndWatermark); + // save data first + errCode = SaveData(context, data, curType, packet->GetQuery()); if (errCode != E_OK) { + (void)SendDataAck(context, message, errCode, dataTime.endTime); return errCode; } - parcel.EightByteAlign(); - return errCode; -} - -int SingleVerDataSync::AckPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) -{ - const DataAckPacket *packet = inMsg->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; + if (pullEndWatermark > 0 && !storage_->IsReadable()) { // pull mode + pullEndWatermark = 0; + errCode = SendDataAck(context, message, -E_EKEYREVOKED, dataTime.endTime); + } else { + // if data is empty, we don't know the max timestap of this packet. + errCode = SendDataAck(context, message, !data.empty() ? E_OK : WATER_MARK_INVALID, dataTime.endTime); + } + RemotePushFinished(packet->GetSendCode(), packet->GetMode(), message->GetSessionId(), + context->GetRequestSessionId()); + if (curType != SyncType::QUERY_SYNC_TYPE && isUpdateWaterMark.normalUpdateMark) { + UpdatePeerWaterMark(curType, "", context, dataTime.endTime + 1, 0); + } else if (curType == SyncType::QUERY_SYNC_TYPE && packet->IsNeedUpdateWaterMark()) { + UpdateQueryPeerWaterMark(curType, packet->GetQueryId(), dataTime, context, isUpdateWaterMark); } - - Parcel parcel(buffer, length); - int errCode = parcel.WriteUInt32(packet->GetVersion()); if (errCode != E_OK) { return errCode; } - // now V1 compatible for softWareVersion :{101, 102} - return SingleVerDataSync::AckPacketSyncerPartSerializationV1(parcel, packet); -} - -int SingleVerDataSync::AckPacketSyncerPartDeSerializationV1(Parcel &parcel, DataAckPacket &packet) -{ - WaterMark mark; - int32_t errCode; - std::vector reserved; - - parcel.ReadUInt64(mark); - parcel.ReadInt(errCode); - parcel.ReadVector(reserved); - if (parcel.IsError()) { - return -E_INVALID_ARGS; + if (packet->GetSendCode() == SEND_FINISHED) { + return -E_RECV_FINISHED; } - packet.SetData(mark); - packet.SetRecvCode(errCode); - packet.SetReserved(reserved); - return E_OK; + return errCode; } -int SingleVerDataSync::AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +int SingleVerDataSync::SendDataPacket(SyncType syncType, const DataRequestPacket *packet, + SingleVerSyncTaskContext *context) { - DataAckPacket packet; - Parcel parcel(const_cast(buffer), length); - uint32_t version; - - parcel.ReadUInt32(version); - if (parcel.IsError()) { - return -E_INVALID_ARGS; - } - if (version > SOFTWARE_VERSION_CURRENT) { - packet.SetVersion(version); - packet.SetRecvCode(-E_VERSION_NOT_SUPPORT); - return inMsg->SetCopiedObject<>(packet); + Message *message = new (std::nothrow) Message(GetMessageId(syncType)); + if (message == nullptr) { + LOGE("[DataSync][SendDataPacket] new message error"); + delete packet; + packet = nullptr; + return -E_OUT_OF_MEMORY; } - packet.SetVersion(version); - // now V1 compatible for softWareVersion :{101, 102} - int errCode = SingleVerDataSync::AckPacketSyncerPartDeSerializationV1(parcel, packet); + uint32_t packetLen = packet->CalculateLen(GetMessageId(syncType)); + int errCode = message->SetExternalObject(packet); if (errCode != E_OK) { + delete packet; + packet = nullptr; + delete message; + message = nullptr; + LOGE("[DataSync][SendDataPacket] set external object failed errCode=%d", errCode); return errCode; } - - return inMsg->SetCopiedObject<>(packet); -} - -int SingleVerDataSync::Send(SingleVerSyncTaskContext *context, const Message *message, const CommErrHandler &handler, - uint32_t packetLen) -{ - bool startFeedDogRet = false; - if (packetLen > mtuSize_ && mtuSize_ > 0) { - uint32_t time = static_cast(static_cast(packetLen) * - static_cast(DBConstant::AUTO_SYNC_TIMEOUT) / mtuSize_); // no overflow - startFeedDogRet = context->StartFeedDogForSync(time, SyncDirectionFlag::SEND); - } - int errCode = communicateHandle_->SendMessage(context->GetDeviceId(), message, false, SEND_TIME_OUT, handler); - if (errCode != E_OK) { - LOGE("[DataSync][Send] send message failed, errCode=%d", errCode); - if (startFeedDogRet) { - context->StopFeedDogForSync(SyncDirectionFlag::SEND); - } - } - return errCode; -} - -int SingleVerDataSync::GetData(SingleVerSyncTaskContext *context, std::vector &outData, size_t packetSize) -{ - int errCode; - if (context->GetRetryStatus() == SyncTaskContext::NEED_RETRY) { - context->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); - LOGI("[DataSync][GetData] resend data"); - errCode = GetUnsyncData(context, outData, packetSize); - } else { - ContinueToken token; - context->GetContinueToken(token); - if (token == nullptr) { - errCode = GetUnsyncData(context, outData, packetSize); - } else { - LOGD("[DataSync][GetData] get data from token"); - // if there is data to send, read out data, and update local watermark, send data - errCode = GetNextUnsyncData(context, outData, packetSize); - } - } - if (errCode == E_OK || errCode == -E_UNFINISHED) { - TransDbDataItemToSendDataItem(context, outData); - } - if (errCode == -E_UNFINISHED) { - LOGD("[DataSync][GetData] not finished."); - } - if (errCode == -E_EKEYREVOKED) { - context->SetTaskErrCode(-E_EKEYREVOKED); - } - if (errCode == -E_BUSY) { - context->SetTaskErrCode(-E_BUSY); - } - return errCode; -} - -int SingleVerDataSync::GetDataWithRerformanceRecord(SingleVerSyncTaskContext *context, - std::vector &outData) -{ - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - size_t packetSize = (version > SOFTWARE_VERSION_RELEASE_2_0) ? - DBConstant::MAX_HPMODE_PACK_ITEM_SIZE : DBConstant::MAX_NORMAL_PACK_ITEM_SIZE; + SetMessageHeadInfo(*message, TYPE_REQUEST, context->GetDeviceId(), context->GetSequenceId(), + context->GetRequestSessionId()); PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_READ_DATA); - } - int errCode = GetData(context, outData, packetSize); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_READ_DATA); - } - return errCode; -} - -int SingleVerDataSync::GetUnsyncData(SingleVerSyncTaskContext *context, std::vector &outData, - size_t packetSize) -{ - WaterMark startMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), startMark); - WaterMark endMark = context->GetEndMark(); - if ((endMark == 0) || (startMark > endMark)) { - return E_OK; - } - ContinueToken token = nullptr; - context->GetContinueToken(token); - if (token != nullptr) { - storage_->ReleaseContinueToken(token); + performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_SEND_REQUEST_TO_ACK_RECV); } - DataSizeSpecInfo syncDataSizeInfo = {mtuSize_, packetSize}; - int errCode = storage_->GetSyncData(startMark, endMark, outData, token, syncDataSizeInfo); - context->SetContinueToken(token); - if (errCode != E_OK && errCode != -E_UNFINISHED) { - LOGE("[DataSync][GetUnsyncData] get unsync data failed,errCode=%d", errCode); + CommErrHandler handler = std::bind(&SyncTaskContext::CommErrHandlerFunc, std::placeholders::_1, + context, message->GetSessionId()); + errCode = Send(context, message, handler, packetLen); + if (errCode != E_OK) { + delete message; + message = nullptr; } - return errCode; -} -int SingleVerDataSync::GetNextUnsyncData(SingleVerSyncTaskContext *context, std::vector &outData, - size_t packetSize) -{ - ContinueToken token; - context->GetContinueToken(token); - DataSizeSpecInfo syncDataSizeInfo = {mtuSize_, packetSize}; - int errCode = storage_->GetSyncDataNext(outData, token, syncDataSizeInfo); - context->SetContinueToken(token); - if (errCode != E_OK && errCode != -E_UNFINISHED) { - LOGE("[DataSync][GetNextUnsyncData] get next unsync data failed, errCode=%d", errCode); - } return errCode; } -int SingleVerDataSync::SaveData(const SingleVerSyncTaskContext *context, const std::vector &inData) +int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, SyncEntry &syncOutData, + SingleVerSyncTaskContext *context) { - if (inData.empty()) { - return E_OK; + DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; + if (packet == nullptr) { + LOGE("[DataSync][SendPullResponseDataPkt] new data request packet error"); + return -E_OUT_OF_MEMORY; } - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SAVE_DATA); + SyncType syncType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + FillDataRequestPacket(packet, context, syncOutData, ackCode, SyncModeType::RESPONSE_PULL); + uint32_t packetLen = packet->CalculateLen(GetMessageId(syncType)); + Message *message = new (std::nothrow) Message(GetMessageId(syncType)); + if (message == nullptr) { + LOGE("[DataSync][SendPullResponseDataPkt] new message error"); + delete packet; + packet = nullptr; + return -E_OUT_OF_MEMORY; } - - TransSendDataItemToLocal(context, inData); - int errCode = storage_->PutSyncData(inData, context->GetDeviceId()); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SAVE_DATA); + int errCode = message->SetExternalObject(packet); + if (errCode != E_OK) { + delete packet; + packet = nullptr; + delete message; + message = nullptr; + LOGE("[SendPullResponseDataPkt] set external object failed, errCode=%d", errCode); + return errCode; } + SetMessageHeadInfo(*message, TYPE_REQUEST, context->GetDeviceId(), context->GetSequenceId(), + context->GetResponseSessionId()); + SendResetWatchDogPacket(context, packetLen); + errCode = Send(context, message, nullptr, packetLen); if (errCode != E_OK) { - LOGE("[DataSync][SaveData] save sync data failed,errCode=%d", errCode); + delete message; + message = nullptr; } return errCode; } -TimeStamp SingleVerDataSync::GetMaxSendDataTime(const std::vector &inData, bool isNeedInit, - WaterMark localMark) +int SingleVerDataSync::SendDataAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, + WaterMark maxSendDataTime) { - TimeStamp stamp = 0; - if (isNeedInit) { - stamp = localMark; + const DataRequestPacket *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; } - for (size_t i = 0; i < inData.size(); i++) { - if (inData[i] == nullptr) { - continue; - } - TimeStamp tempStamp = inData[i]->GetTimestamp(); - if (stamp < tempStamp) { - stamp = tempStamp; - } + Message *ackMessage = new (std::nothrow) Message(message->GetMessageId()); + if (ackMessage == nullptr) { + LOGE("[DataSync][SendDataAck] new message error"); + return -E_OUT_OF_MEMORY; } - return stamp; -} - -TimeStamp SingleVerDataSync::GetMinSendDataTime(const std::vector &inData, WaterMark localMark) -{ - TimeStamp stamp = localMark; - for (size_t i = 0; i < inData.size(); i++) { - if (inData[i] == nullptr) { - continue; - } - TimeStamp tempStamp = inData[i]->GetTimestamp(); - if (stamp > tempStamp) { - stamp = tempStamp; - } + DataAckPacket ack; + SetAckPacket(ack, context, packet, recvCode, maxSendDataTime); + int errCode = ackMessage->SetCopiedObject(ack); + if (errCode != E_OK) { + delete ackMessage; + ackMessage = nullptr; + LOGE("[DataSync][SendDataAck] set copied object failed, errcode=%d", errCode); + return errCode; } - return stamp; -} + SetMessageHeadInfo(*ackMessage, TYPE_RESPONSE, context->GetDeviceId(), message->GetSequenceId(), + message->GetSessionId()); -int SingleVerDataSync::SaveLocalWaterMark(const DeviceID &deviceId, WaterMark waterMark) -{ - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SAVE_LOCAL_WATERMARK); - } - int errCode = metadata_->SaveLocalWaterMark(deviceId, waterMark); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SAVE_LOCAL_WATERMARK); - } + errCode = Send(context, ackMessage, nullptr, 0); if (errCode != E_OK) { - LOGE("[DataSync][SaveLocalWaterMark] save metadata local watermark failed,errCode=%d", errCode); + delete ackMessage; + ackMessage = nullptr; } return errCode; } -int SingleVerDataSync::SavePeerWaterMark(const DeviceID &deviceId, WaterMark waterMark) -{ - return metadata_->SavePeerWaterMark(deviceId, waterMark); -} - -int SingleVerDataSync::RemoveDeviceData(SingleVerSyncTaskContext *context, const Message *message, - WaterMark maxSendDataTime) +int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message *message) { - int errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); + int errCode = AckMsgErrnoCheck(context, message); if (errCode != E_OK) { - (void)SendAck(context, message, errCode, maxSendDataTime); return errCode; } - if (context->GetRemoteSoftwareVersion() == SOFTWARE_VERSION_EARLIEST) { - (void)SaveLocalWaterMark(context->GetDeviceId(), 0); // avoid repeat clear in ack - } - return E_OK; -} - -void SingleVerDataSync::TransDbDataItemToSendDataItem(const SingleVerSyncTaskContext *context, - std::vector &outData) -{ - for (size_t i = 0; i < outData.size(); i++) { - if (outData[i] == nullptr) { - continue; - } - outData[i]->SetOrigDevice(TransferLocalOrigDevName(outData[i]->GetOrigDevice())); - if (i == 0 || i == (outData.size() - 1)) { - LOGD("[DataSync][TransToSendItem] printData packet=%d,timeStamp=%llu,flag=%llu", i, - outData[i]->GetTimestamp(), outData[i]->GetFlag()); - } + const DataAckPacket *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; } -} - -void SingleVerDataSync::TransSendDataItemToLocal(const SingleVerSyncTaskContext *context, - const std::vector &data) -{ - TimeOffset offset = context->GetTimeOffset(); - TimeStamp currentLocalTime = context->GetCurrentLocalTime(); - for (auto &item : data) { - if (item == nullptr) { - continue; - } - item->SetOrigDevice(TransferForeignOrigDevName(item->GetOrigDevice())); - TimeStamp tempTimestamp = item->GetTimestamp(); - TimeStamp tempWriteTimestamp = item->GetWriteTimestamp(); - item->SetTimestamp(tempTimestamp - offset); - if (tempWriteTimestamp != 0) { - item->SetWriteTimestamp(tempWriteTimestamp - offset); - } - - if (item->GetTimestamp() > currentLocalTime) { - item->SetTimestamp(currentLocalTime); - } - if (item->GetWriteTimestamp() > currentLocalTime) { - item->SetWriteTimestamp(currentLocalTime); - } + int32_t recvCode = packet->GetRecvCode(); + LOGD("[DataSync][AckRecv] ver=%u,recvCode=%d,myversion=%u,label=%s,dev=%s", packet->GetVersion(), recvCode, + SOFTWARE_VERSION_CURRENT, label_.c_str(), STR_MASK(GetDeviceId())); + if (recvCode == -E_VERSION_NOT_SUPPORT) { + LOGE("[DataSync][AckRecv] Version mismatch"); + return -E_VERSION_NOT_SUPPORT; } -} -int SingleVerDataSync::PushStart(SingleVerSyncTaskContext *context) -{ - if (context == nullptr) { - return -E_INVALID_ARGS; + if (recvCode == -E_NEED_ABILITY_SYNC || recvCode == -E_NOT_PERMIT) { + // after set sliding window sender err, we can ReleaseContinueToken, avoid crash + context->SetSlidingWindowSenderErr(true); + LOGI("[DataSync][AckRecv] Data sync abort,recvCode =%d,label =%s,dev=%s", recvCode, label_.c_str(), + STR_MASK(GetDeviceId())); + context->ReleaseContinueToken(); + return recvCode; } - std::vector outData; - WaterMark localMark; - WaterMark peerMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - LOGD("[DataSync][PushStart] localMark=%llu,endmark=%llu,peerMark=%llu,label=%s,dev=%s{private}", localMark, - context->GetEndMark(), peerMark, label_.c_str(), GetDeviceId().c_str()); - // get data - int errCode = GetDataWithRerformanceRecord(context, outData); - if (errCode != E_OK && errCode != -E_UNFINISHED) { - LOGE("[DataSync][PushStart] get data failed, errCode=%d", errCode); - return errCode; + uint64_t data = packet->GetData(); + if (recvCode == LOCAL_WATER_MARK_NOT_INIT) { + return DealWaterMarkException(context, data, packet->GetReserved()); } - DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; - if (packet == nullptr) { - LOGE("[DataSync][PushStart] new DataRequestPacket error"); - return -E_OUT_OF_MEMORY; - } - bool isUpdateWaterMark = (outData.size() > 0); - TimeStamp maxSendDateTime = GetMaxSendDataTime(outData, true, localMark); - TimeStamp minSendDateTime = GetMinSendDataTime(outData, localMark); - context->SetSequenceStartAndEndTimeStamp(minSendDateTime, maxSendDateTime); - if (errCode == E_OK) { - context->SetSessionEndTimeStamp(maxSendDateTime); - packet->SetLastSequence(); + if (recvCode == -E_SAVE_DATA_NOTIFY && data != 0) { + // data only use low 32bit + context->StartFeedDogForSync(static_cast(data), SyncDirectionFlag::RECEIVE); + LOGI("[DataSync][AckRecv] notify ResetWatchDog=%llu,label=%s,dev=%s", data, label_.c_str(), + STR_MASK(GetDeviceId())); } - packet->SetData(outData); - packet->SetBasicInfo(errCode, version, localMark, peerMark, SyncOperation::PUSH); - SetPacketId(packet, context, version); - errCode = SendDataPacket(packet, context); - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_MACHINE_START_TO_PUSH_SEND); + + if (recvCode != E_OK && recvCode != WATER_MARK_INVALID) { + LOGW("[DataSync][AckRecv] Received a uncatched recvCode=%d,label=%s,dev=%s", recvCode, + label_.c_str(), STR_MASK(GetDeviceId())); + return recvCode; } - if (errCode == E_OK && isUpdateWaterMark) { - SaveLocalWaterMark(context->GetDeviceId(), maxSendDateTime + 1); + + // Judge if send finished + ContinueToken token; + context->GetContinueToken(token); + if (((message->GetSessionId() == context->GetResponseSessionId()) || + (message->GetSessionId() == context->GetRequestSessionId())) && (token == nullptr)) { + return -E_NO_DATA_SEND; } - return errCode; + + // send next data + return -E_SEND_DATA; } -int SingleVerDataSync::PushPullStart(SingleVerSyncTaskContext *context) +void SingleVerDataSync::SendSaveDataNotifyPacket(SingleVerSyncTaskContext *context, uint32_t pktVersion, + uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) { - if (context == nullptr) { - return -E_INVALID_ARGS; - } - WaterMark localMark; - WaterMark peerMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - LOGD("[DataSync][PushPull] localMark=%llu,endmark=%llu,peerMark=%llu,label=%s,dev=%s{private}", localMark, - context->GetEndMark(), peerMark, label_.c_str(), GetDeviceId().c_str()); - // get data - std::vector outData; - int errCode = GetDataWithRerformanceRecord(context, outData); - // once get data occur E_EKEYREVOKED error, should also send request to remote dev to pull data. - if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && errCode == -E_EKEYREVOKED) { - errCode = E_OK; + if (inMsgId != DATA_SYNC_MESSAGE && inMsgId != QUERY_SYNC_MESSAGE) { + LOGE("[SingleVerDataSync] messageId not avaiable."); + return; } - if (errCode != E_OK && errCode != -E_UNFINISHED) { - return errCode; + Message *ackMessage = new (std::nothrow) Message(inMsgId); + if (ackMessage == nullptr) { + LOGE("[DataSync][SaveDataNotify] new message failed"); + return; } - DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; - if (packet == nullptr) { - LOGE("[DataSync][PushPullStart] new DataRequestPacket error"); - return -E_OUT_OF_MEMORY; - } - bool isUpdateWaterMark = (outData.size() > 0); - TimeStamp maxSendDateTime = GetMaxSendDataTime(outData, true, localMark); - TimeStamp minSendDateTime = GetMinSendDataTime(outData, localMark); - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - context->SetSequenceStartAndEndTimeStamp(minSendDateTime, maxSendDateTime); - if (errCode == E_OK) { - context->SetSessionEndTimeStamp(maxSendDateTime); - packet->SetLastSequence(); - } - packet->SetData(outData); - packet->SetBasicInfo(errCode, version, localMark, peerMark, SyncOperation::PUSH_AND_PULL); - packet->SetEndWaterMark(context->GetEndMark()); - packet->SetSessionId(context->GetRequestSessionId()); - SetPacketId(packet, context, version); - int sendErrCode = SendDataPacket(packet, context); - if (sendErrCode == E_OK && isUpdateWaterMark) { - SaveLocalWaterMark(context->GetDeviceId(), maxSendDateTime + 1); + DataAckPacket ack; + ack.SetRecvCode(-E_SAVE_DATA_NOTIFY); + ack.SetVersion(pktVersion); + int errCode = ackMessage->SetCopiedObject(ack); + if (errCode != E_OK) { + delete ackMessage; + ackMessage = nullptr; + LOGE("[DataSync][SendSaveDataNotifyPacket] set copied object failed,errcode=%d", errCode); + return; } - return sendErrCode; -} + SetMessageHeadInfo(*ackMessage, TYPE_NOTIFY, context->GetDeviceId(), sequenceId, sessionId); -int SingleVerDataSync::PullRequestStart(SingleVerSyncTaskContext *context) -{ - if (context == nullptr) { - return -E_INVALID_ARGS; - } - DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; - if (packet == nullptr) { - LOGE("[DataSync][PullRequest]new DataRequestPacket error"); - return -E_OUT_OF_MEMORY; + errCode = Send(context, ackMessage, nullptr, 0); + if (errCode != E_OK) { + delete ackMessage; + ackMessage = nullptr; } - WaterMark peerMark; - WaterMark localMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - packet->SetLocalWaterMark(localMark); - packet->SetPeerWaterMark(peerMark); - packet->SetMode(SyncOperation::PULL); - WaterMark endMark = context->GetEndMark(); - packet->SetEndWaterMark(endMark); - packet->SetSessionId(context->GetRequestSessionId()); - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - packet->SetVersion(version); - packet->SetLastSequence(); - SetPacketId(packet, context, version); - LOGD("[DataSync][PullRequest]peerMark=%llu,localMark=%llu,endMark=%llu,IsLastSeq=%d,label=%s,dev=%s{private}", - peerMark, localMark, endMark, packet->IsLastSequence(), label_.c_str(), GetDeviceId().c_str()); - return SendDataPacket(packet, context); + LOGD("[DataSync][SaveDataNotify] Send SaveDataNotify packet Finished,errcode=%d,label=%s,dev=%s", + errCode, label_.c_str(), STR_MASK(GetDeviceId())); } -int SingleVerDataSync::PullResponseStart(SingleVerSyncTaskContext *context) +void SingleVerDataSync::GetPullEndWatermark(const SingleVerSyncTaskContext *context, const DataRequestPacket *packet, + WaterMark &pullEndWatermark) const { - if (context == nullptr) { - return -E_INVALID_ARGS; - } - WaterMark localMark; - WaterMark peerMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - LOGD("[DataSync][PullResponse] localMark=%llu,pullendmark=%llu,peerMark=%llu,label=%s,dev=%s{private}", - localMark, context->GetEndMark(), peerMark, label_.c_str(), GetDeviceId().c_str()); - // get data - std::vector outData; - int errCode = GetDataWithRerformanceRecord(context, outData); - if (errCode != E_OK && errCode != -E_UNFINISHED) { - if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { - SendPullResponseDataPkt(errCode, outData, context); - } - return errCode; - } - - // if send finished - int ackCode = E_OK; - ContinueToken token = nullptr; - context->GetContinueToken(token); - if (errCode == E_OK && token == nullptr) { - LOGD("[DataSync][PullResponse] send last frame end"); - ackCode = SEND_FINISHED; - } - bool isUpdateWaterMark = (outData.size() > 0); - TimeStamp maxSendDateTime = GetMaxSendDataTime(outData, true, localMark); - TimeStamp minSendDateTime = GetMinSendDataTime(outData, localMark); - context->SetSequenceStartAndEndTimeStamp(minSendDateTime, maxSendDateTime); - if (errCode == E_OK) { - context->SetSessionEndTimeStamp(maxSendDateTime); + if (packet == nullptr) { + return; } - errCode = SendPullResponseDataPkt(ackCode, outData, context); - if (errCode == E_OK && isUpdateWaterMark) { - SaveLocalWaterMark(context->GetDeviceId(), maxSendDateTime + 1); + int mode = SyncOperation::TransferSyncMode(packet->GetMode()); + if ((mode == SyncModeType::PULL) || (mode == SyncModeType::PUSH_AND_PULL)) { + WaterMark endMark = packet->GetEndWaterMark(); + TimeOffset offset; + metadata_->GetTimeOffset(context->GetDeviceId(), offset); + pullEndWatermark = endMark - offset; + LOGD("[DataSync][PullEndWatermark] packetEndMark=%llu,offset=%llu,endWaterMark=%llu,label=%s,dev=%s", + endMark, offset, pullEndWatermark, label_.c_str(), STR_MASK(GetDeviceId())); } - return errCode; } -void SingleVerDataSync::UpdatePeerWaterMark(const SingleVerSyncTaskContext *context, WaterMark peerWatermark) +int SingleVerDataSync::DealWaterMarkException(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, + const std::vector &reserved) { - if (peerWatermark == 0) { - return; + // after set sliding window sender err, we can SaveLocalWaterMark, avoid sliding window sender re save a wrong + // waterMark again. + context->SetSlidingWindowSenderErr(true); + WaterMark deletedWaterMark = 0; + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + if (curType == SyncType::QUERY_SYNC_TYPE) { + if (reserved.size() <= ACK_PACKET_RESERVED_INDEX_DELETE_WATER_MARK) { + LOGE("[DataSync] get packet reserve size failed"); + return -E_INVALID_ARGS; + } + deletedWaterMark = reserved[ACK_PACKET_RESERVED_INDEX_DELETE_WATER_MARK]; } - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_SAVE_PEER_WATERMARK); + LOGI("[DataSync][WaterMarkException] AckRecv water error, mark=%llu,deleteMark=%llu,label=%s,dev=%s", ackWaterMark, + deletedWaterMark, label_.c_str(), STR_MASK(GetDeviceId())); + int errCode = SaveLocalWaterMark(curType, context, + {0, 0, ackWaterMark, deletedWaterMark}); + if (errCode != E_OK) { + return errCode; } - - int errCode = SavePeerWaterMark(context->GetDeviceId(), peerWatermark + 1); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_SAVE_PEER_WATERMARK); + context->SetRetryStatus(SyncTaskContext::NEED_RETRY); + context->IncNegotiationCount(); + PushAndPUllKeyRevokHandle(context); + if (!context->IsNeedClearRemoteStaleData()) { + return -E_RE_SEND_DATA; } + errCode = DealRemoveDeviceDataByAck(context, ackWaterMark, reserved); if (errCode != E_OK) { - LOGE("[DataSync][UpdatePeerWaterMark] save peer water mark failed,errCode=%d", errCode); + return errCode; } + return -E_RE_SEND_DATA; } -int SingleVerDataSync::RequestRecvPre(SingleVerSyncTaskContext *context, const Message *message) +int SingleVerDataSync::RunPermissionCheck(SingleVerSyncTaskContext *context, const Message *message, + const DataRequestPacket *packet) { - if (context == nullptr || message == nullptr) { - return -E_INVALID_ARGS; - } - const DataRequestPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - if (context->GetRemoteSoftwareVersion() <= SOFTWARE_VERSION_BASE) { - uint16_t remoteCommunicatorVersion = 0; - if (communicateHandle_->GetRemoteCommunicatorVersion(context->GetDeviceId(), remoteCommunicatorVersion) == - -E_NOT_FOUND) { - LOGE("[DataSync][RequestRecvPre] get remote communicator version failed"); - return -E_VERSION_NOT_SUPPORT; - } - // If remote is not the first version, we need check SOFTWARE_VERSION_BASE - if (remoteCommunicatorVersion == 0) { - LOGI("[DataSync] set remote version 0"); - context->SetRemoteSoftwareVersion(SOFTWARE_VERSION_EARLIEST); - return E_OK; - } else { - LOGI("[DataSync][RequestRecvPre] need do ability sync"); - SendAck(context, message, -E_NEED_ABILITY_SYNC, 0); - return -E_WAIT_NEXT_MESSAGE; - } - } - int32_t sendCode = packet->GetSendCode(); - if (sendCode == -E_VERSION_NOT_SUPPORT) { - LOGE("[DataSync] Version mismatch: ver=%u, current=%u", packet->GetVersion(), SOFTWARE_VERSION_CURRENT); - (void)SendAck(context, message, -E_VERSION_NOT_SUPPORT, 0); - return -E_WAIT_NEXT_MESSAGE; - } - // only deal with pull response packet errCode - if (sendCode != E_OK && sendCode != SEND_FINISHED && - message->GetSessionId() == context->GetRequestSessionId()) { - LOGE("[DataSync][RequestRecvPre] remote pullResponse getData sendCode=%d", sendCode); - return sendCode; + std::string appId = storage_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); + std::string userId = storage_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); + std::string storeId = storage_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); + uint8_t flag; + int mode = SyncOperation::TransferSyncMode(packet->GetMode()); + if (mode == SyncModeType::PUSH) { + flag = CHECK_FLAG_RECEIVE; + } else if (mode == SyncModeType::PULL) { + flag = CHECK_FLAG_SEND; + } else if (mode == SyncModeType::PUSH_AND_PULL) { + flag = CHECK_FLAG_SEND | CHECK_FLAG_RECEIVE; + } else { + // before add permissionCheck, PushStart packet and pullResponse packet do not setMode. + flag = CHECK_FLAG_RECEIVE; } - int errCode = RunPermissionCheck(context, message, packet); + int errCode = RuntimeContext::GetInstance()->RunPermissionCheck(userId, appId, storeId, context->GetDeviceId(), + flag); if (errCode != E_OK) { - return errCode; + LOGE("[DataSync][RunPermissionCheck] check failed flag=%d,Label=%s,dev=%s", flag, label_.c_str(), + STR_MASK(GetDeviceId())); + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_EARLIEST) { // ver 101 can't handle this errCode + (void)SendDataAck(context, message, -E_NOT_PERMIT, 0); + } + return -E_NOT_PERMIT; } + const std::vector &data = packet->GetData(); + WaterMark maxSendDataTime = GetMaxSendDataTime(data); uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - if (version > SOFTWARE_VERSION_RELEASE_2_0) { - errCode = CheckSchemaStrategy(context, message); + if (version > SOFTWARE_VERSION_RELEASE_2_0 && (mode != SyncModeType::PULL) && + !context->GetReceivcPermitCheck()) { + bool permitReceive = CheckPermitReceiveData(context); + if (permitReceive) { + context->SetReceivcPermitCheck(true); + } else { + (void)SendDataAck(context, message, -E_SECURITY_OPTION_CHECK_ERROR, maxSendDataTime); + return -E_SECURITY_OPTION_CHECK_ERROR; + } } return errCode; } -int SingleVerDataSync::RequestRecv(SingleVerSyncTaskContext *context, const Message *message, - WaterMark &pullEndWatermark) +// used in pull response +void SingleVerDataSync::SendResetWatchDogPacket(SingleVerSyncTaskContext *context, uint32_t packetLen) { - int errCode = RequestRecvPre(context, message); - if (errCode != E_OK) { - return errCode; + // When mtu less than 30k, we send data with bluetooth + // In order not to block the bluetooth channel, we don't send notify packet here + if (mtuSize_ >= packetLen || mtuSize_ < NOTIFY_MIN_MTU_SIZE) { + return; } - const DataRequestPacket *packet = message->GetObject(); - const std::vector &data = packet->GetData(); - LOGI("[DataSync][RequestRecv] remote ver=%u,size=%d,errCode=%d,Label=%s,dev=%s{private}", packet->GetVersion(), - data.size(), packet->GetSendCode(), label_.c_str(), GetDeviceId().c_str()); - WaterMark packetLocalMark = packet->GetLocalWaterMark(); - WaterMark peerMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - context->SetReceiveWaterMarkErr(false); - WaterMark maxSendDataTime = GetMaxSendDataTime(data, false); - if (packetLocalMark > peerMark) { - LOGI("[DataSync][RequestRecv] packetLocalMark=%llu,current=%llu", packetLocalMark, peerMark); - return SendLocalWaterMarkAck(context, message); - } else if ((packetLocalMark == 0) && (peerMark != 0) && context->IsNeedClearRemoteStaleData()) { - // need to clear remote device history data - errCode = RemoveDeviceData(context, message, maxSendDataTime); - if (errCode != E_OK) { - return errCode; - } + uint64_t data = static_cast(packetLen) * static_cast(context->GetTimeoutTime()) / mtuSize_; + + Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); + if (ackMessage == nullptr) { + LOGE("[DataSync][ResetWatchDog] new message failed"); + return; } - GetPullEndWatermark(context, packet, pullEndWatermark); - // save data first - errCode = SaveData(context, data); + + DataAckPacket ack; + ack.SetData(data); + ack.SetRecvCode(-E_SAVE_DATA_NOTIFY); + ack.SetVersion(std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT)); + int errCode = ackMessage->SetCopiedObject(ack); if (errCode != E_OK) { - (void)SendAck(context, message, errCode, maxSendDataTime); - return errCode; + delete ackMessage; + ackMessage = nullptr; + LOGE("[DataSync][ResetWatchDog] set copied object failed, errcode=%d", errCode); + return; } - if (pullEndWatermark > 0 && !storage_->IsReadable()) { // pull mode - pullEndWatermark = 0; - errCode = SendAck(context, message, -E_EKEYREVOKED, maxSendDataTime); + SetMessageHeadInfo(*ackMessage, TYPE_NOTIFY, context->GetDeviceId(), context->GetSequenceId(), + context->GetResponseSessionId()); + + errCode = Send(context, ackMessage, nullptr, 0); + if (errCode != E_OK) { + delete ackMessage; + ackMessage = nullptr; + LOGE("[DataSync][ResetWatchDog] Send packet failed,errcode=%d,label=%s,dev=%s", errCode, label_.c_str(), + STR_MASK(GetDeviceId())); } else { - // if data is empty, we don't know the max timestap of this packet. - errCode = SendAck(context, message, !data.empty() ? E_OK : WATER_MARK_INVALID, maxSendDataTime); + LOGI("[DataSync][ResetWatchDog] data = %llu,label=%s,dev=%s", data, label_.c_str(), STR_MASK(GetDeviceId())); } - RemotePushFinished(packet->GetSendCode(), packet->GetMode(), message->GetSessionId(), - context->GetRequestSessionId()); - UpdatePeerWaterMark(context, maxSendDataTime); - if (errCode != E_OK) { +} + +int32_t SingleVerDataSync::ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo) +{ + if (context == nullptr) { + return -E_INVALID_ARGS; + } + SyncEntry syncData; + int errCode = GetReSendData(syncData, context, reSendInfo); + if (!IsGetDataSuccessfully(errCode)) { return errCode; } - if (packet->GetSendCode() == SEND_FINISHED) { - return -E_RECV_FINISHED; + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; + if (packet == nullptr) { + LOGE("[DataSync][ReSend] new DataRequestPacket error"); + return -E_OUT_OF_MEMORY; + } + FillRequestReSendPacket(context, packet, reSendInfo, syncData, errCode); + errCode = SendReSendPacket(packet, context, reSendInfo.sessionId, reSendInfo.sequenceId); + if (errCode == E_OK && SyncOperation::TransferSyncMode(context->GetMode()) != SyncModeType::PULL) { + // resend.end may not update in localwatermark while E_TIMEOUT occurred in send message last time. + SyncTimeRange dataTime {reSendInfo.start, reSendInfo.deleteDataStart, reSendInfo.end, reSendInfo.deleteDataEnd}; + if (reSendInfo.deleteDataEnd > reSendInfo.deleteDataStart && curType == SyncType::QUERY_SYNC_TYPE) { + dataTime.deleteEndTime += 1; + } + if (reSendInfo.end > reSendInfo.start) { + dataTime.endTime += 1; + } + SaveLocalWaterMark(curType, context, dataTime, true); } - return errCode; } -int SingleVerDataSync::SendDataPacket(const DataRequestPacket *packet, SingleVerSyncTaskContext *context) +int SingleVerDataSync::SendReSendPacket(const DataRequestPacket *packet, SingleVerSyncTaskContext *context, + uint32_t sessionId, uint32_t sequenceId) { - Message *message = new (std::nothrow) Message(DATA_SYNC_MESSAGE); + SyncType syncType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + Message *message = new (std::nothrow) Message(GetMessageId(syncType)); if (message == nullptr) { LOGE("[DataSync][SendDataPacket] new message error"); delete packet; packet = nullptr; return -E_OUT_OF_MEMORY; } - uint32_t packetLen = packet->CalculateLen(); + uint32_t packetLen = packet->CalculateLen(GetMessageId(syncType)); int errCode = message->SetExternalObject(packet); if (errCode != E_OK) { delete packet; packet = nullptr; delete message; message = nullptr; - LOGE("[DataSync][SendDataPacket] set external object failed errCode=%d", errCode); + LOGE("[DataSync][SendReSendPacket] SetExternalObject failed errCode=%d", errCode); return errCode; } - SetMessageHeadInfo(*message, TYPE_REQUEST, context->GetDeviceId(), context->GetSequenceId(), - context->GetRequestSessionId()); - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_SEND_REQUEST_TO_ACK_RECV); - } - + SetMessageHeadInfo(*message, TYPE_REQUEST, context->GetDeviceId(), sequenceId, sessionId); CommErrHandler handler = std::bind(&SyncTaskContext::CommErrHandlerFunc, std::placeholders::_1, context, message->GetSessionId()); errCode = Send(context, message, handler, packetLen); @@ -1229,91 +1336,197 @@ int SingleVerDataSync::SendDataPacket(const DataRequestPacket *packet, SingleVer delete message; message = nullptr; } - return errCode; } -int SingleVerDataSync::SendPullResponseDataPkt(int ackCode, std::vector &inData, - SingleVerSyncTaskContext *context) +bool SingleVerDataSync::IsPermitRemoteDeviceRecvData(const std::string &deviceId, + const SecurityOption &remoteSecOption) const { - DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; - if (packet == nullptr) { - LOGE("[DataSync][SendPullResponseDataPkt] new data request packet error"); - return -E_OUT_OF_MEMORY; + SecurityOption localSecOption; + if (remoteSecOption.securityLabel == NOT_SURPPORT_SEC_CLASSIFICATION) { + return true; + } + int errCode = storage_->GetSecurityOption(localSecOption); + if (errCode == -E_NOT_SUPPORT) { + return true; } - size_t size = inData.size(); - WaterMark localMark; - WaterMark peerMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); + return RuntimeContext::GetInstance()->CheckDeviceSecurityAbility(deviceId, localSecOption); +} + +bool SingleVerDataSync::IsPermitLocalDeviceRecvData(const std::string &deviceId, + const SecurityOption &remoteSecOption) const +{ + return RuntimeContext::GetInstance()->CheckDeviceSecurityAbility(deviceId, remoteSecOption); +} + +int SingleVerDataSync::CheckPermitSendData(int inMode, SingleVerSyncTaskContext *context) +{ uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - packet->SetBasicInfo(ackCode, version, localMark, peerMark, SyncOperation::PUSH); - if (ackCode == SEND_FINISHED) { - packet->SetLastSequence(); + int mode = SyncOperation::TransferSyncMode(inMode); + // for pull mode it just need to get data, no need to send data. + if (version <= SOFTWARE_VERSION_RELEASE_2_0 || mode == SyncModeType::PULL) { + return E_OK; } - SetPacketId(packet, context, version); - packet->SetData(inData); - uint32_t packetLen = packet->CalculateLen(); - Message *message = new (std::nothrow) Message(DATA_SYNC_MESSAGE); - if (message == nullptr) { - LOGE("[DataSync][SendPullResponseDataPkt] new message error"); - delete packet; - packet = nullptr; - return -E_OUT_OF_MEMORY; + if (context->GetSendPermitCheck()) { + return E_OK; + } + bool isPermitSync = true; + std::string deviceId = context->GetDeviceId(); + SecurityOption remoteSecOption = context->GetRemoteSeccurityOption(); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::RESPONSE_PULL) { + isPermitSync = IsPermitRemoteDeviceRecvData(deviceId, remoteSecOption); + } + LOGI("[DataSync][PermitSendData] mode=%d,dev=%s,label=%d,flag=%d,PermitSync=%d", mode, STR_MASK(deviceId_), + remoteSecOption.securityLabel, remoteSecOption.securityFlag, isPermitSync); + if (isPermitSync) { + context->SetSendPermitCheck(true); + return E_OK; + } + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { + context->SetTaskErrCode(-E_SECURITY_OPTION_CHECK_ERROR); + return -E_SECURITY_OPTION_CHECK_ERROR; + } + if (mode == SyncModeType::RESPONSE_PULL) { + SyncEntry syncData; + SendPullResponseDataPkt(-E_SECURITY_OPTION_CHECK_ERROR, syncData, context); + return -E_SECURITY_OPTION_CHECK_ERROR; + } + if (mode == SyncModeType::SUBSCRIBE_QUERY) { + return -E_SECURITY_OPTION_CHECK_ERROR; + } + return E_OK; +} + +std::string SingleVerDataSync::GetLabel() const +{ + return label_; +} + +std::string SingleVerDataSync::GetDeviceId() const +{ + return deviceId_; +} + +bool SingleVerDataSync::WaterMarkErrHandle(SyncType syncType, SingleVerSyncTaskContext *context, const Message *message) +{ + const DataRequestPacket *packet = message->GetObject(); + if (packet == nullptr) { + LOGE("[WaterMarkErrHandle] get packet object failed"); + return -E_INVALID_ARGS; + } + WaterMark packetLocalMark = packet->GetLocalWaterMark(); + WaterMark packetDeletedMark = packet->GetDeletedWaterMark(); + WaterMark peerMark = 0; + WaterMark deletedMark = 0; + GetPeerWaterMark(syncType, packet->GetQueryId(), context->GetDeviceId(), peerMark); + if (syncType == SyncType::QUERY_SYNC_TYPE) { + GetPeerDeleteSyncWaterMark(GetDeleteSyncId(context), deletedMark); + } + if (syncType != SyncType::QUERY_SYNC_TYPE && packetLocalMark > peerMark) { + LOGI("[DataSync][DataRequestRecv] packetLocalMark=%llu,current=%llu", packetLocalMark, peerMark); + context->SetReceiveWaterMarkErr(true); + SendDataAck(context, message, LOCAL_WATER_MARK_NOT_INIT, 0); + return true; + } + if (syncType == SyncType::QUERY_SYNC_TYPE && (packetLocalMark > peerMark || packetDeletedMark > deletedMark)) { + LOGI("[DataSync][DataRequestRecv] packetDeletedMark=%llu,deletedMark=%llu,packetLocalMark=%llu,peerMark=%llu", + packetDeletedMark, deletedMark, packetLocalMark, peerMark); + context->SetReceiveWaterMarkErr(true); + SendDataAck(context, message, LOCAL_WATER_MARK_NOT_INIT, 0); + return true; + } + return false; +} + +bool SingleVerDataSync::CheckPermitReceiveData(const SingleVerSyncTaskContext *context) +{ + SecurityOption remoteSecOption = context->GetRemoteSeccurityOption(); + std::string localDeviceId; + if (communicateHandle_ == nullptr || remoteSecOption.securityLabel == NOT_SURPPORT_SEC_CLASSIFICATION) { + return true; + } + communicateHandle_->GetLocalIdentity(localDeviceId); + bool isPermitSync = IsPermitLocalDeviceRecvData(localDeviceId, remoteSecOption); + if (isPermitSync) { + return isPermitSync; + } + LOGE("[DataSync][PermitReceiveData] check failed: permitReceive=%d, localDev=%s, seclabel=%d, secflag=%d", + isPermitSync, STR_MASK(localDeviceId), remoteSecOption.securityLabel, remoteSecOption.securityFlag); + return isPermitSync; +} + +int SingleVerDataSync::CheckSchemaStrategy(SingleVerSyncTaskContext *context, const Message *message) +{ + SyncStrategy localStrategy = context->GetSyncStrategy(); + if (!context->GetIsSchemaSync()) { + LOGE("[DataSync][CheckSchemaStrategy] isSchemaSync=%d check failed", context->GetIsSchemaSync()); + (void)SendDataAck(context, message, -E_NEED_ABILITY_SYNC, 0); + return -E_NEED_ABILITY_SYNC; + } + if (!localStrategy.permitSync) { + LOGE("[DataSync][CheckSchemaStrategy] Strategy permitSync=%d check failed", localStrategy.permitSync); + (void)SendDataAck(context, message, -E_SCHEMA_MISMATCH, 0); + return -E_SCHEMA_MISMATCH; + } + return E_OK; +} + +void SingleVerDataSync::RemotePushFinished(int sendCode, int inMode, uint32_t msgSessionId, uint32_t contextSessionId) +{ + int mode = SyncOperation::TransferSyncMode(inMode); + if ((mode != SyncModeType::PUSH) && (mode != SyncModeType::PUSH_AND_PULL) && (mode != SyncModeType::QUERY_PUSH) && + (mode != SyncModeType::QUERY_PUSH_PULL)) { + return; } - LOGI("[DataSync][SendPullResponseDataPkt] size=%d,code=%d,LastSequence=%d,label=%s,dev=%s{private}", - size, ackCode, packet->IsLastSequence(), label_.c_str(), GetDeviceId().c_str()); - int errCode = message->SetExternalObject(packet); - if (errCode != E_OK) { - delete packet; - packet = nullptr; - delete message; - message = nullptr; - LOGE("[SendPullResponseDataPkt] set external object failed, errCode=%d", errCode); - return errCode; + + if ((sendCode == E_OK) && (msgSessionId != 0) && (msgSessionId != contextSessionId)) { + storage_->NotifyRemotePushFinished(deviceId_); } - SetMessageHeadInfo(*message, TYPE_REQUEST, context->GetDeviceId(), context->GetSequenceId(), - context->GetResponseSessionId()); - SendResetWatchDogPacket(context, packetLen); +} - errCode = Send(context, message, nullptr, packetLen); - if (errCode != E_OK) { - delete message; - message = nullptr; +void SingleVerDataSync::SetPacketId(DataRequestPacket *packet, SingleVerSyncTaskContext *context, uint32_t version) +{ + if (version > SOFTWARE_VERSION_RELEASE_2_0) { + context->IncPacketId(); // begin from 1 + std::vector reserved {context->GetPacketId()}; + packet->SetReserved(reserved); } - return errCode; } -void SingleVerDataSync::SetAckData(DataAckPacket &ackPacket, SingleVerSyncTaskContext *context, int32_t recvCode, - WaterMark maxSendDataTime) const +int SingleVerDataSync::GetMessageId(SyncType syncType) const { - // send ack packet - if ((recvCode == E_OK) && (maxSendDataTime != 0)) { - ackPacket.SetData(maxSendDataTime + 1); // + 1 to next start - } else if (recvCode != WATER_MARK_INVALID) { - WaterMark mark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), mark); - ackPacket.SetData(mark); + if (syncType == SyncType::QUERY_SYNC_TYPE) { + return QUERY_SYNC_MESSAGE; } + return DATA_SYNC_MESSAGE; } -int SingleVerDataSync::SendAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, - WaterMark maxSendDataTime) +void SingleVerDataSync::PushAndPUllKeyRevokHandle(SingleVerSyncTaskContext *context) { - const DataRequestPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; + // for push_and_pull mode it may be EKEYREVOKED error before receive watermarkexception + // should clear errCode and restart pushpull request. + int mode = SyncOperation::TransferSyncMode(context->GetMode()); + if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && mode == SyncModeType::PUSH_AND_PULL && + context->GetTaskErrCode() == -E_EKEYREVOKED) { + context->SetTaskErrCode(E_OK); } - Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); - if (ackMessage == nullptr) { - LOGE("[DataSync][SendAck] new message error"); - return -E_OUT_OF_MEMORY; +} + +void SingleVerDataSync::SetAckPacket(DataAckPacket &ackPacket, SingleVerSyncTaskContext *context, + const DataRequestPacket *packet, int32_t recvCode, WaterMark maxSendDataTime) +{ + SyncType curType = SyncOperation::GetSyncType(packet->GetMode()); + WaterMark localMark = 0; + GetLocalWaterMark(curType, packet->GetQueryId(), context, localMark); + ackPacket.SetRecvCode(recvCode); + // send ack packet + if ((recvCode == E_OK) && (maxSendDataTime != 0)) { + ackPacket.SetData(maxSendDataTime + 1); // + 1 to next start + } else if (recvCode != WATER_MARK_INVALID) { + WaterMark mark = 0; + GetPeerWaterMark(curType, packet->GetQueryId(), context->GetDeviceId(), mark); + ackPacket.SetData(mark); } - DataAckPacket ack; - ack.SetRecvCode(recvCode); - SetAckData(ack, context, recvCode, maxSendDataTime); - WaterMark localMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); std::vector reserved {localMark}; uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); uint64_t packetId = 0; @@ -1323,404 +1536,358 @@ int SingleVerDataSync::SendAck(SingleVerSyncTaskContext *context, const Message if (version > SOFTWARE_VERSION_RELEASE_2_0 && packetId > 0) { reserved.push_back(packetId); } - ack.SetReserved(reserved); - ack.SetVersion(version); - int errCode = ackMessage->SetCopiedObject(ack); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - LOGE("[DataSync][SendAck] set copied object failed, errcode=%d", errCode); - return errCode; - } - SetMessageHeadInfo(*ackMessage, TYPE_RESPONSE, context->GetDeviceId(), message->GetSequenceId(), - message->GetSessionId()); - - errCode = Send(context, ackMessage, nullptr, 0); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - } - PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); - if (performance != nullptr) { - performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); + // while recv is not E_OK, data is peerMark, reserve[2] is deletedPeerMark value + if (curType == SyncType::QUERY_SYNC_TYPE && recvCode != WATER_MARK_INVALID) { + WaterMark deletedPeerMark; + GetPeerDeleteSyncWaterMark(GetDeleteSyncId(context), deletedPeerMark); + reserved.push_back(deletedPeerMark); // query sync mode, reserve[2] store deletedPeerMark value } - return errCode; + ackPacket.SetReserved(reserved); + ackPacket.SetVersion(version); } -int SingleVerDataSync::SendLocalWaterMarkAck(SingleVerSyncTaskContext *context, const Message *message) +int SingleVerDataSync::GetReSendData(SyncEntry &syncData, SingleVerSyncTaskContext *context, + DataSyncReSendInfo reSendInfo) { - context->SetReceiveWaterMarkErr(true); - const DataRequestPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; - } - Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); - if (ackMessage == nullptr) { - LOGE("[DataSync][LocalWaterMarkAck] new message error"); - return -E_OUT_OF_MEMORY; + int mode = SyncOperation::TransferSyncMode(context->GetMode()); + if (mode == SyncModeType::PULL) { + return E_OK; } - - DataAckPacket ack; - WaterMark peerMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); + ContinueToken token = nullptr; uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - ack.SetData(peerMark); - WaterMark localMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - uint64_t packetId = 0; - if (version > SOFTWARE_VERSION_RELEASE_2_0) { - packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value + size_t packetSize = (version > SOFTWARE_VERSION_RELEASE_2_0) ? + DBConstant::MAX_HPMODE_PACK_ITEM_SIZE : DBConstant::MAX_NORMAL_PACK_ITEM_SIZE; + DataSizeSpecInfo reSendDataSizeInfo = GetDataSizeSpecInfo(packetSize); + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + int errCode; + if (curType != SyncType::QUERY_SYNC_TYPE) { + errCode = storage_->GetSyncData(reSendInfo.start, reSendInfo.end + 1, syncData.entries, token, + reSendDataSizeInfo); + } else { + QuerySyncObject queryObj = context->GetQuery(); + errCode = storage_->GetSyncData(queryObj, SyncTimeRange { reSendInfo.start, reSendInfo.deleteDataStart, + reSendInfo.end + 1, reSendInfo.deleteDataEnd + 1 }, reSendDataSizeInfo, token, syncData.entries); } - std::vector reserved {localMark}; - if (version > SOFTWARE_VERSION_RELEASE_2_0 && packetId > 0) { - reserved.push_back(packetId); + if (token != nullptr) { + storage_->ReleaseContinueToken(token); } - ack.SetReserved(reserved); - ack.SetRecvCode(LOCAL_WATER_MARK_NOT_INIT); - ack.SetVersion(version); - int errCode = ackMessage->SetCopiedObject(ack); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - LOGE("[DataSync][LocalWaterMarkAck] set copied object failed, errcode=%d", errCode); + if (errCode == -E_BUSY || errCode == -E_EKEYREVOKED) { + context->SetTaskErrCode(errCode); return errCode; } - SetMessageHeadInfo(*ackMessage, TYPE_RESPONSE, context->GetDeviceId(), message->GetSequenceId(), - message->GetSessionId()); - errCode = Send(context, ackMessage, nullptr, 0); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; + if (!IsGetDataSuccessfully(errCode)) { + return errCode; + } + + int innerCode = InterceptData(syncData); + if (innerCode != E_OK) { + context->SetTaskErrCode(innerCode); + return innerCode; + } + + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); + CompressAlgorithm remoteAlgo = context->ChooseCompressAlgo(); + if (needCompressOnSync && remoteAlgo != CompressAlgorithm::NONE) { + int compressCode = GenericSingleVerKvEntry::Compress(syncData.entries, syncData.compressedEntries, + { remoteAlgo, version }); + if (compressCode != E_OK) { + return compressCode; + } } - LOGI("[DataSync][LocalWaterMarkAck] LOCAL_WATER_MARK_NOT_INIT peerMark=%llu,localMark=%llu", peerMark, localMark); return errCode; } -int SingleVerDataSync::AckRecv(SingleVerSyncTaskContext *context, const Message *message) +int SingleVerDataSync::GetReSendMode(int mode, uint32_t sequenceId, SyncType syncType) { - if (context == nullptr || message == nullptr) { - return -E_INVALID_ARGS; + int curMode = SyncOperation::TransferSyncMode(mode); + if (curMode == SyncModeType::PUSH || curMode == SyncModeType::PULL) { + return mode; } - - const DataAckPacket *packet = message->GetObject(); - if (packet == nullptr) { - return -E_INVALID_ARGS; + if (curMode == SyncModeType::RESPONSE_PULL) { + return (syncType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH : SyncModeType::PUSH; } - int32_t recvCode = packet->GetRecvCode(); - LOGD("[DataSync][AckRecv] ver=%u,recvCode=%d,myversion=%u,label=%s,dev=%s{private}", packet->GetVersion(), - recvCode, SOFTWARE_VERSION_CURRENT, label_.c_str(), GetDeviceId().c_str()); - if (recvCode == -E_VERSION_NOT_SUPPORT) { - LOGE("[DataSync][AckRecv] Version mismatch"); - return -E_VERSION_NOT_SUPPORT; + // set push_and_pull mode when resend first sequenceId to inform remote to run RESPONSE_PULL task + // for sequenceId which is larger than first, only need to send data, means to set push or query_push mode + if (sequenceId == 1) { + return (syncType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH_PULL : SyncModeType::PUSH_AND_PULL; } + return (syncType == SyncType::QUERY_SYNC_TYPE) ? SyncModeType::QUERY_PUSH : SyncModeType::PUSH; +} - if (recvCode == -E_NEED_ABILITY_SYNC || recvCode == -E_NOT_PERMIT) { - // after set sliding window sender err, we can ReleaseContinueToken, avoid crash - context->SetSlidingWindowSenderErr(true); - LOGI("[DataSync][AckRecv] Data sync abort,recvCode = %d,label = %s,dev = %s{private}", recvCode, - label_.c_str(), GetDeviceId().c_str()); - context->ReleaseContinueToken(); - return recvCode; - } - uint64_t data = packet->GetData(); - if (recvCode == LOCAL_WATER_MARK_NOT_INIT) { - return DealWaterMarkException(context, data, packet->GetReserved()); +int SingleVerDataSync::RemoveDeviceDataIfNeed(SingleVerSyncTaskContext *context) +{ + if (context->GetRemoteSoftwareVersion() <= SOFTWARE_VERSION_RELEASE_3_0) { + return E_OK; } - - if (recvCode == -E_SAVE_DATA_NOTIFY && data != 0) { - // data only use low 32bit - context->StartFeedDogForSync(static_cast(data), SyncDirectionFlag::RECEIVE); - LOGI("[DataSync][AckRecv] notify ResetWatchDog=%llu,label=%s,dev=%s{private}", data, label_.c_str(), - GetDeviceId().c_str()); + uint64_t clearRemoteDataMark = 0; + metadata_->GetRemoveDataMark(context->GetDeviceId(), clearRemoteDataMark); + if (clearRemoteDataMark == 0) { + return E_OK; } - - if (recvCode != E_OK && recvCode != WATER_MARK_INVALID) { - LOGW("[DataSync][AckRecv] Received a uncatched recvCode=%d,label=%s,dev=%s{private}", recvCode, - label_.c_str(), GetDeviceId().c_str()); - return recvCode; + int errCode = E_OK; + if (context->IsNeedClearRemoteStaleData() && clearRemoteDataMark == REMOVE_DEVICE_DATA_MARK) { + errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); + if (errCode != E_OK) { + LOGE("clear remote %s data failed,errCode=%d", STR_MASK(GetDeviceId()), errCode); + return errCode; + } } - - // Judge if send finished - ContinueToken token; - context->GetContinueToken(token); - if (((message->GetSessionId() == context->GetResponseSessionId()) || - (message->GetSessionId() == context->GetRequestSessionId())) && (token == nullptr)) { - return -E_NO_DATA_SEND; + if (clearRemoteDataMark == REMOVE_DEVICE_DATA_MARK) { + errCode = metadata_->ResetMetaDataAfterRemoveData(context->GetDeviceId()); + if (errCode != E_OK) { + LOGE("set %s removeDataWaterMark to false failed,errCode=%d", STR_MASK(GetDeviceId()), errCode); + return errCode; + } } + return E_OK; +} - // send next data - return -E_SEND_DATA; +void SingleVerDataSync::UpdateMtuSize() +{ + mtuSize_ = communicateHandle_->GetCommunicatorMtuSize(deviceId_) * 9 / 10; // get the 9/10 of the size } -void SingleVerDataSync::SendSaveDataNotifyPacket(SingleVerSyncTaskContext *context, uint32_t pktVersion, - uint32_t sessionId, uint32_t sequenceId) +void SingleVerDataSync::FillRequestReSendPacket(const SingleVerSyncTaskContext *context, DataRequestPacket *packet, + DataSyncReSendInfo reSendInfo, SyncEntry &syncData, int sendCode) { - Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); - if (ackMessage == nullptr) { - LOGE("[DataSync][SaveDataNotify] new message failed"); - return; + SyncType curType = (context->IsQuerySync()) ? SyncType::QUERY_SYNC_TYPE : SyncType::MANUAL_FULL_SYNC_TYPE; + WaterMark peerMark = 0; + GetPeerWaterMark(curType, GetQuerySyncId(context, context->GetQuery().GetIdentify()), context->GetDeviceId(), + peerMark); + uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); + // transer reSend mode, RESPONSE_PULL transfer to push or query push + // PUSH_AND_PULL mode which sequenceId lager than first transfer to push or query push + int reSendMode = GetReSendMode(context->GetMode(), reSendInfo.sequenceId, curType); + if (context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) || + SyncOperation::TransferSyncMode(context->GetMode()) == SyncModeType::PULL) { + LOGI("[DataSync][ReSend] set lastid,label=%s,dev=%s", label_.c_str(), STR_MASK(GetDeviceId())); + packet->SetLastSequence(); } - - DataAckPacket ack; - ack.SetRecvCode(-E_SAVE_DATA_NOTIFY); - ack.SetVersion(pktVersion); - int errCode = ackMessage->SetCopiedObject(ack); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - LOGE("[DataSync][SendSaveDataNotifyPacket] set copied object failed,errcode=%d", errCode); - return; + if (sendCode == E_OK && context->GetSessionEndTimeStamp() == std::max(reSendInfo.end, reSendInfo.deleteDataEnd) && + context->GetMode() == SyncModeType::RESPONSE_PULL) { + sendCode = SEND_FINISHED; } - SetMessageHeadInfo(*ackMessage, TYPE_NOTIFY, context->GetDeviceId(), sequenceId, sessionId); - - errCode = Send(context, ackMessage, nullptr, 0); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; + packet->SetData(syncData.entries); + packet->SetCompressData(syncData.compressedEntries); + packet->SetBasicInfo(sendCode, version, reSendMode); + packet->SetWaterMark(reSendInfo.start, peerMark, reSendInfo.deleteDataStart); + if (SyncOperation::TransferSyncMode(reSendMode) != SyncModeType::PUSH) { + packet->SetEndWaterMark(context->GetEndMark()); + packet->SetQuery(context->GetQuery()); + } + packet->SetQueryId(GetQuerySyncId(context, context->GetQuery().GetIdentify())); + if (version > SOFTWARE_VERSION_RELEASE_2_0) { + std::vector reserved {reSendInfo.packetId}; + packet->SetReserved(reserved); + } + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); + CompressAlgorithm curAlgo = context->ChooseCompressAlgo(); + if (needCompressOnSync && curAlgo != CompressAlgorithm::NONE) { + packet->SetCompressDataMark(); + packet->SetCompressAlgo(curAlgo); } - LOGD("[DataSync][SaveDataNotify] Send SaveDataNotify packet Finished,errcode=%d,label=%s,dev=%s{private}", - errCode, label_.c_str(), GetDeviceId().c_str()); } -void SingleVerDataSync::GetPullEndWatermark(const SingleVerSyncTaskContext *context, const DataRequestPacket *packet, - WaterMark &pullEndWatermark) const +DataSizeSpecInfo SingleVerDataSync::GetDataSizeSpecInfo(size_t packetSize) { - if (packet == nullptr) { - return; - } - if ((packet->GetMode() == SyncOperation::PULL) || (packet->GetMode() == SyncOperation::PUSH_AND_PULL)) { - WaterMark endMark = packet->GetEndWaterMark(); - TimeOffset offset; - metadata_->GetTimeOffset(context->GetDeviceId(), offset); - pullEndWatermark = endMark - offset; - LOGD("[DataSync][PullEndWatermark] packetEndMark=%llu,offset=%llu,endWaterMark=%llu,label=%s,dev=%s{private}", - endMark, offset, pullEndWatermark, label_.c_str(), GetDeviceId().c_str()); - } + bool needCompressOnSync = false; + uint8_t compressionRate = DBConstant::DEFAULT_COMPTRESS_RATE; + (void)storage_->GetCompressionOption(needCompressOnSync, compressionRate); + uint32_t blockSize = std::min(static_cast(DBConstant::MAX_SYNC_BLOCK_SIZE), + mtuSize_ * 100 / compressionRate); // compressionRate max is 100 + return {blockSize, packetSize}; } -int SingleVerDataSync::DealWaterMarkException(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, - const std::vector &reserved) +int SingleVerDataSync::AckMsgErrnoCheck(const SingleVerSyncTaskContext *context, const Message *message) const { - // after set sliding window sender err, we can SaveLocalWaterMark, avoid sliding window sender re save a wrong - // waterMark again. - context->SetSlidingWindowSenderErr(true); - WaterMark localMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - LOGI("[DataSync][WaterMarkException] AckRecv LOCAL_WATER_MARK_NOT_INIT mark=%llu,label=%s,dev=%s{private}", - ackWaterMark, label_.c_str(), GetDeviceId().c_str()); - int errCode = SaveLocalWaterMark(context->GetDeviceId(), ackWaterMark); - if (errCode != E_OK) { - return errCode; - } - context->SetRetryStatus(SyncTaskContext::NEED_RETRY); - // for push_and_pull mode it may be EKEYREVOKED error before receive watermarkexception - // should clear errCode and restart pushpull request. - if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && - context->GetMode() == SyncOperation::PUSH_AND_PULL && context->GetTaskErrCode() == -E_EKEYREVOKED) { - context->SetTaskErrCode(E_OK); + if (context == nullptr || message == nullptr) { + return -E_INVALID_ARGS; } - - if (reserved.empty()) { - if (localMark != 0 && ackWaterMark == 0 && context->IsNeedClearRemoteStaleData()) { - // need to clear remote historydata - LOGI("[DataSync][WaterMarkException] AckRecv rebuilted,clear historydata,label=%s,dev=%s{private}", - label_.c_str(), GetDeviceId().c_str()); - errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); - if (errCode != E_OK) { - return errCode; - } - } - } else { - WaterMark peerMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - if (reserved[ACK_PACKET_RESERVED_INDEX_LOCAL_WATER_MARK] == 0 && peerMark != 0 && - context->IsNeedClearRemoteStaleData()) { - // need to clear remote historydata - LOGI("[DataSync][WaterMarkException] AckRecv reserved not empty,rebuilted,clear historydata,label=%s," - "dev = %s{private}", label_.c_str(), GetDeviceId().c_str()); - errCode = storage_->RemoveDeviceData(context->GetDeviceId(), true); - if (errCode != E_OK) { - return errCode; - } - } + if (message->IsFeedbackError()) { + LOGE("[DataSync][AckMsgErrnoCheck] message errNo=%d", message->GetErrorNo()); + return -(message->GetErrorNo()); } - return -E_RE_SEND_DATA; + return E_OK; } -int SingleVerDataSync::RunPermissionCheck(SingleVerSyncTaskContext *context, const Message *message, - const DataRequestPacket *packet) +bool SingleVerDataSync::QuerySyncCheck(const SingleVerSyncTaskContext *context) const { - std::string appId = storage_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); - std::string userId = storage_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); - std::string storeId = storage_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); - uint8_t flag; - int32_t mode = packet->GetMode(); - if (mode == SyncOperation::PUSH) { - flag = CHECK_FLAG_RECEIVE; - } else if (mode == SyncOperation::PULL) { - flag = CHECK_FLAG_SEND; - } else if (mode == SyncOperation::PUSH_AND_PULL) { - flag = CHECK_FLAG_SEND | CHECK_FLAG_RECEIVE; - } else { - // before add permissionCheck, PushStart packet and pullResponse packet do not setMode. - flag = CHECK_FLAG_RECEIVE; - } - int errCode = RuntimeContext::GetInstance()->RunPermissionCheck(userId, appId, storeId, context->GetDeviceId(), - flag); - if (errCode != E_OK) { - LOGE("[DataSync][RunPermissionCheck] check failed flag=%d,Label=%s,dev=%s{private}", flag, label_.c_str(), - GetDeviceId().c_str()); - if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_EARLIEST) { // ver 101 can't handle this errCode - (void)SendAck(context, message, -E_NOT_PERMIT, 0); - } - return -E_NOT_PERMIT; + if (!context->IsQuerySync()) { + return true; } - const std::vector &data = packet->GetData(); - WaterMark maxSendDataTime = GetMaxSendDataTime(data, false); uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - if (version > SOFTWARE_VERSION_RELEASE_2_0 && (mode != SyncOperation::PULL) && !context->GetReceivcPermitCheck()) { - bool permitReceive = CheckPermitReceiveData(context); - if (permitReceive) { - context->SetReceivcPermitCheck(true); - } else { - (void)SendAck(context, message, -E_SECURITY_OPTION_CHECK_ERROR, maxSendDataTime); - return -E_SECURITY_OPTION_CHECK_ERROR; - } + // for 101 version, no need to do abilitySync, just send request to remote + if (version <= SOFTWARE_VERSION_RELEASE_1_0) { + return true; } - return errCode; + if (version < SOFTWARE_VERSION_RELEASE_4_0) { + LOGE("[SingleVerDataSync] not support query sync when remote ver lower than 104"); + return false; + } + if (version < SOFTWARE_VERSION_RELEASE_5_0 && !(context->GetQuery().IsQueryOnlyByKey())) { + LOGE("[SingleVerDataSync] remote version only support prefix key"); + return false; + } + return true; } -// used in pull response -void SingleVerDataSync::SendResetWatchDogPacket(SingleVerSyncTaskContext *context, uint32_t packetLen) +int SingleVerDataSync::InterceptData(SyncEntry &syncEntry) { - if (mtuSize_ >= packetLen || mtuSize_ == 0) { - return; + if (storage_ == nullptr) { + LOGE("Invalid DB. Can not intercept data."); + return -E_INVALID_DB; } - uint64_t data = static_cast(packetLen) * static_cast(DBConstant::AUTO_SYNC_TIMEOUT) / mtuSize_; - Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); - if (ackMessage == nullptr) { - LOGE("[DataSync][ResetWatchDog] new message failed"); - return; - } + // GetLocalDeviceName get local device ID. + // GetDeviceId get remote device ID. + // If intercept data fail, entries will be released. + return storage_->InterceptData(syncEntry.entries, GetLocalDeviceName(), GetDeviceId()); +} - DataAckPacket ack; - ack.SetData(data); - ack.SetRecvCode(-E_SAVE_DATA_NOTIFY); - ack.SetVersion(std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT)); - int errCode = ackMessage->SetCopiedObject(ack); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - LOGE("[DataSync][ResetWatchDog] set copied object failed, errcode=%d", errCode); - return; +int SingleVerDataSync::RequestQueryCheck(const DataRequestPacket *packet) const +{ + if (SyncOperation::GetSyncType(packet->GetMode()) != SyncType::QUERY_SYNC_TYPE) { + return E_OK; } - SetMessageHeadInfo(*ackMessage, TYPE_NOTIFY, context->GetDeviceId(), context->GetSequenceId(), - context->GetResponseSessionId()); - - errCode = Send(context, ackMessage, nullptr, 0); + QuerySyncObject syncQuery = packet->GetQuery(); + int errCode = storage_->CheckAndInitQueryCondition(syncQuery); if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - LOGE("[DataSync][ResetWatchDog] Send packet failed,errcode=%d,label=%s,dev = %s{private}", errCode, - label_.c_str(), GetDeviceId().c_str()); - } else { - LOGI("[DataSync][ResetWatchDog] data = %llu,label=%s,dev=%s{private}", data, label_.c_str(), - GetDeviceId().c_str()); + LOGE("[SingleVerDataSync] check sync query failed,errCode=%d", errCode); + return errCode; } + return E_OK; } -void SingleVerDataSync::SendAck(SingleVerSyncTaskContext *context, uint32_t sessionId, uint32_t sequenceId, - uint64_t packetId) +int SingleVerDataSync::ControlCmdStart(SingleVerSyncTaskContext *context) { - Message *ackMessage = new (std::nothrow) Message(DATA_SYNC_MESSAGE); - if (ackMessage == nullptr) { - LOGE("[DataSync][SendAck] new message error"); - return; + if (context == nullptr) { + return -E_INVALID_ARGS; } - DataAckPacket ack; - ack.SetRecvCode(E_OK); - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - WaterMark localMark; - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - std::vector reserved {localMark}; - if (version > SOFTWARE_VERSION_RELEASE_2_0 && packetId > 0) { - reserved.push_back(packetId); + std::shared_ptr subManager = context->GetSubscribeManager(); + if (subManager == nullptr) { + return -E_INVALID_ARGS; } - ack.SetReserved(reserved); - ack.SetVersion(version); - int errCode = ackMessage->SetCopiedObject(ack); + int errCode = ControlCmdStartCheck(context); if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; - LOGE("[DataSync][SendAck] set copied object failed, errcode=%d", errCode); - return; + return errCode; } - SetMessageHeadInfo(*ackMessage, TYPE_RESPONSE, context->GetDeviceId(), sequenceId, sessionId); - errCode = Send(context, ackMessage, nullptr, 0); - if (errCode != E_OK) { - delete ackMessage; - ackMessage = nullptr; + ControlRequestPacket* packet = new (std::nothrow) SubscribeRequest(); + if (packet == nullptr) { + LOGE("[DataSync][ControlCmdStart] new SubscribeRequest error"); + return -E_OUT_OF_MEMORY; + } + if (context->GetMode() == SyncModeType::SUBSCRIBE_QUERY) { + errCode = subManager->ReserveLocalSubscribeQuery(context->GetDeviceId(), context->GetQuery()); + if (errCode != E_OK) { + LOGE("[DataSync][ControlCmdStart] reserve local subscribe query failed,err=%d", errCode); + delete packet; + packet = nullptr; + return errCode; + } } + FillControlRequestPacket(packet, context); + errCode = SendControlPacket(packet, context); + if (errCode != E_OK && context->GetMode() == SyncModeType::SUBSCRIBE_QUERY) { + subManager->DeleteLocalSubscribeQuery(context->GetDeviceId(), context->GetQuery()); + } + return errCode; } -int32_t SingleVerDataSync::ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo) +int SingleVerDataSync::ControlCmdRequestRecv(SingleVerSyncTaskContext *context, const Message *message) { - if (context == nullptr) { + const ControlRequestPacket *packet = message->GetObject(); + if (packet == nullptr) { return -E_INVALID_ARGS; } - LOGI("[DataSync][ReSend] start=%llu,end=%llu,label=%s,dev=%s{private}", reSendInfo.start, reSendInfo.end, - label_.c_str(), GetDeviceId().c_str()); - std::vector outData; - ContinueToken token = nullptr; - uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - size_t packetSize = (version > SOFTWARE_VERSION_RELEASE_2_0) ? - DBConstant::MAX_HPMODE_PACK_ITEM_SIZE : DBConstant::MAX_NORMAL_PACK_ITEM_SIZE; - DataSizeSpecInfo reSendDataSizeInfo = {mtuSize_, packetSize}; - int errCode = storage_->GetSyncData(reSendInfo.start, reSendInfo.end + 1, outData, token, reSendDataSizeInfo); - if (token != nullptr) { - storage_->ReleaseContinueToken(token); - } - if (errCode == -E_BUSY || errCode == -E_EKEYREVOKED) { - context->SetTaskErrCode(errCode); + LOGI("[SingleVerDataSync] recv control cmd message,label=%s,dev=%s,controlType=%u", label_.c_str(), + STR_MASK(GetDeviceId()), packet->GetcontrolCmdType()); + int errCode = ControlCmdRequestRecvPre(context, message); + if (errCode != E_OK) { return errCode; } - if (errCode != E_OK && errCode != -E_UNFINISHED) { + if (packet->GetcontrolCmdType() == ControlCmdType::SUBSCRIBE_QUERY_CMD) { + errCode = SubscribeRequestRecv(context, message); + } else if (packet->GetcontrolCmdType() == ControlCmdType::UNSUBSCRIBE_QUERY_CMD) { + errCode = UnsubscribeRequestRecv(context, message); + } + return errCode; +} + +int SingleVerDataSync::ControlCmdAckRecv(SingleVerSyncTaskContext *context, const Message *message) +{ + std::shared_ptr subManager = context->GetSubscribeManager(); + if (subManager == nullptr) { + return -E_INVALID_ARGS; + } + int errCode = AckMsgErrnoCheck(context, message); + if (errCode != E_OK) { + ControlAckErrorHandle(context, subManager); return errCode; } - WaterMark localMark; - WaterMark peerMark; - metadata_->GetPeerWaterMark(context->GetDeviceId(), peerMark); - metadata_->GetLocalWaterMark(context->GetDeviceId(), localMark); - DataRequestPacket *packet = new (std::nothrow) DataRequestPacket; + const ControlAckPacket *packet = message->GetObject(); if (packet == nullptr) { - LOGE("[DataSync][ReSend] new DataRequestPacket error"); - return -E_OUT_OF_MEMORY; + return -E_INVALID_ARGS; } - if (context->GetSessionEndTimeStamp() == reSendInfo.end) { - LOGI("[DataSync][ReSend] lastid,label=%s,dev=%s{private}", label_.c_str(), GetDeviceId().c_str()); - packet->SetLastSequence(); + int32_t recvCode = packet->GetRecvCode(); + uint32_t cmdType = packet->GetcontrolCmdType(); + if (recvCode != E_OK) { + LOGE("[DataSync][AckRecv] control sync abort,recvCode=%d,label=%s,dev=%s,type=%u", recvCode, label_.c_str(), + STR_MASK(GetDeviceId()), cmdType); + // for unsubscribe no need to do something + ControlAckErrorHandle(context, subManager); + return recvCode; } - packet->SetData(outData); - packet->SetBasicInfo(errCode, version, reSendInfo.start, peerMark, SyncOperation::PUSH); - if (version > SOFTWARE_VERSION_RELEASE_2_0) { - std::vector reserved {reSendInfo.packetId}; - packet->SetReserved(reserved); + if (cmdType == ControlCmdType::SUBSCRIBE_QUERY_CMD) { + errCode = subManager->ActiveLocalSubscribeQuery(context->GetDeviceId(), context->GetQuery()); + } else if (cmdType == ControlCmdType::UNSUBSCRIBE_QUERY_CMD) { + subManager->RemoveLocalSubscribeQuery(context->GetDeviceId(), context->GetQuery()); } - errCode = SendReSendPacket(packet, context, reSendInfo.sessionId, reSendInfo.sequenceId); - if (errCode == E_OK && localMark < reSendInfo.end) { - // resend.end may not update in localwatermark while E_TIMEOUT occurred in send message last time. - SaveLocalWaterMark(context->GetDeviceId(), reSendInfo.end + 1); + if (errCode != E_OK) { + LOGE("[DataSync] ack handle failed,label =%s,dev=%s,type=%u", label_.c_str(), STR_MASK(GetDeviceId()), cmdType); + return errCode; } - return errCode; + return -E_NO_DATA_SEND; // means control msg send finished } -int SingleVerDataSync::SendReSendPacket(const DataRequestPacket *packet, SingleVerSyncTaskContext *context, - uint32_t sessionId, uint32_t sequenceId) +int SingleVerDataSync::ControlCmdStartCheck(SingleVerSyncTaskContext *context) +{ + if ((context->GetMode() != SyncModeType::SUBSCRIBE_QUERY) && + (context->GetMode() != SyncModeType::UNSUBSCRIBE_QUERY)) { + LOGE("[ControlCmdStartCheck] not support controlCmd"); + return -E_INVALID_ARGS; + } + if ((context->GetMode() != SyncModeType::SUBSCRIBE_QUERY) || context->GetReceivcPermitCheck()) { + return E_OK; + } + bool permitReceive = CheckPermitReceiveData(context); + if (permitReceive) { + context->SetReceivcPermitCheck(true); + } else { + return -E_SECURITY_OPTION_CHECK_ERROR; + } + return E_OK; +} + +void SingleVerDataSync::FillControlRequestPacket(ControlRequestPacket *packet, SingleVerSyncTaskContext *context) +{ + uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); + uint32_t flag = 0; + if (context->GetMode() == SyncModeType::SUBSCRIBE_QUERY && context->IsAutoSubscribe()) { + flag = flag | SubscribeRequest::IS_AUTO_SUBSCRIBE; + } + packet->SetPacketHead(E_OK, version, GetControlCmdType(context->GetMode()), flag); + packet->SetQuery(context->GetQuery()); +} + +int SingleVerDataSync::SendControlPacket(const ControlRequestPacket *packet, SingleVerSyncTaskContext *context) { - Message *message = new (std::nothrow) Message(DATA_SYNC_MESSAGE); + Message *message = new (std::nothrow) Message(CONTROL_SYNC_MESSAGE); if (message == nullptr) { - LOGE("[DataSync][SendDataPacket] new message error"); + LOGE("[DataSync][SendControlPacket] new message error"); delete packet; packet = nullptr; return -E_OUT_OF_MEMORY; @@ -1732,10 +1899,11 @@ int SingleVerDataSync::SendReSendPacket(const DataRequestPacket *packet, SingleV packet = nullptr; delete message; message = nullptr; - LOGE("[DataSync][SendReSendPacket] SetExternalObject failed errCode=%d", errCode); + LOGE("[DataSync][SendControlPacket] set external object failed errCode=%d", errCode); return errCode; } - SetMessageHeadInfo(*message, TYPE_REQUEST, context->GetDeviceId(), sequenceId, sessionId); + SetMessageHeadInfo(*message, TYPE_REQUEST, context->GetDeviceId(), context->GetSequenceId(), + context->GetRequestSessionId()); CommErrHandler handler = std::bind(&SyncTaskContext::CommErrHandlerFunc, std::placeholders::_1, context, message->GetSessionId()); errCode = Send(context, message, handler, packetLen); @@ -1746,120 +1914,238 @@ int SingleVerDataSync::SendReSendPacket(const DataRequestPacket *packet, SingleV return errCode; } -bool SingleVerDataSync::IsPermitRemoteDeviceRecvData(const std::string &deviceId, - const SecurityOption &remoteSecOption) const +ControlCmdType SingleVerDataSync::GetControlCmdType(int mode) { - SecurityOption localSecOption; - if (remoteSecOption.securityLabel == NOT_SURPPORT_SEC_CLASSIFICATION) { - return true; - } - int errCode = storage_->GetSecurityOption(localSecOption); - if (errCode == -E_NOT_SUPPORT) { - return true; + if (mode == SyncModeType::SUBSCRIBE_QUERY) { + return ControlCmdType::SUBSCRIBE_QUERY_CMD; + } else if (mode == SyncModeType::UNSUBSCRIBE_QUERY) { + return ControlCmdType::UNSUBSCRIBE_QUERY_CMD; } - return RuntimeContext::GetInstance()->CheckDeviceSecurityAbility(deviceId, localSecOption); + return ControlCmdType::INVALID_CONTROL_CMD; } -bool SingleVerDataSync::IsPermitLocalDeviceRecvData(const std::string &deviceId, - const SecurityOption &remoteSecOption) const +int SingleVerDataSync::GetModeByControlCmdType(ControlCmdType controlCmd) { - return RuntimeContext::GetInstance()->CheckDeviceSecurityAbility(deviceId, remoteSecOption); + if (controlCmd == ControlCmdType::SUBSCRIBE_QUERY_CMD) { + return SyncModeType::SUBSCRIBE_QUERY; + } else if (controlCmd == ControlCmdType::UNSUBSCRIBE_QUERY_CMD) { + return SyncModeType::UNSUBSCRIBE_QUERY; + } + return SyncModeType::INVALID_MODE; } -int SingleVerDataSync::CheckPermitSendData(int mode, SingleVerSyncTaskContext *context) +int SingleVerDataSync::SendControlAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, + uint32_t controlCmdType, const CommErrHandler &handler) { + Message *ackMessage = new (std::nothrow) Message(message->GetMessageId()); + if (ackMessage == nullptr) { + LOGE("[DataSync][SendControlAck] new message error"); + return -E_OUT_OF_MEMORY; + } uint32_t version = std::min(context->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT); - // for pull mode it just need to get data, no need to send data. - if (version <= SOFTWARE_VERSION_RELEASE_2_0 || mode == SyncOperation::PULL) { - return E_OK; + ControlAckPacket ack; + ack.SetPacketHead(recvCode, version, controlCmdType, 0); + int errCode = ackMessage->SetCopiedObject(ack); + if (errCode != E_OK) { + delete ackMessage; + ackMessage = nullptr; + LOGE("[DataSync][SendControlAck] set copied object failed, errcode=%d", errCode); + return errCode; } - if (context->GetSendPermitCheck()) { - return E_OK; + SetMessageHeadInfo(*ackMessage, TYPE_RESPONSE, context->GetDeviceId(), message->GetSequenceId(), + message->GetSessionId()); + errCode = Send(context, ackMessage, handler, 0); + if (errCode != E_OK) { + delete ackMessage; + ackMessage = nullptr; } - bool isPermitSync = true; - std::string deviceId = context->GetDeviceId(); - SecurityOption remoteSecOption = context->GetRemoteSeccurityOption(); - if (mode == SyncOperation::PUSH || mode == SyncOperation::PUSH_AND_PULL || mode == SyncOperation::RESPONSE_PULL) { - isPermitSync = IsPermitRemoteDeviceRecvData(deviceId, remoteSecOption); + return errCode; +} + +int SingleVerDataSync::ControlCmdRequestRecvPre(SingleVerSyncTaskContext *context, const Message *message) +{ + if (context == nullptr || message == nullptr) { + return -E_INVALID_ARGS; } - LOGI("[DataSync][PermitSendData] mode=%d,dev=%s{private},label=%d,flag=%d,PermitSync=%d", mode, deviceId.c_str(), - remoteSecOption.securityLabel, remoteSecOption.securityFlag, isPermitSync); - if (isPermitSync) { - context->SetSendPermitCheck(true); - return E_OK; + const ControlRequestPacket *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; } - if (mode == SyncOperation::PUSH || mode == SyncOperation::PUSH_AND_PULL) { - context->SetTaskErrCode(-E_SECURITY_OPTION_CHECK_ERROR); - return -E_SECURITY_OPTION_CHECK_ERROR; + uint32_t controlCmdType = packet->GetcontrolCmdType(); + if (context->GetRemoteSoftwareVersion() <= SOFTWARE_VERSION_BASE) { + return DoAbilitySyncIfNeed(context, message, true); } - if (mode == SyncOperation::RESPONSE_PULL) { - std::vector outData; - SendPullResponseDataPkt(-E_SECURITY_OPTION_CHECK_ERROR, outData, context); - return -E_SECURITY_OPTION_CHECK_ERROR; + if (controlCmdType >= ControlCmdType::INVALID_CONTROL_CMD) { + SendControlAck(context, message, -E_NOT_SUPPORT, controlCmdType); + return -E_WAIT_NEXT_MESSAGE; } return E_OK; } -std::string SingleVerDataSync::GetLabel() const +int SingleVerDataSync::SubscribeRequestRecvPre(SingleVerSyncTaskContext *context, const SubscribeRequest *packet, + const Message *message) { - return label_; + uint32_t controlCmdType = packet->GetcontrolCmdType(); + if (controlCmdType != ControlCmdType::SUBSCRIBE_QUERY_CMD) { + return E_OK; + } + QuerySyncObject syncQuery = packet->GetQuery(); + int errCode; + if (!packet->IsAutoSubscribe()) { + errCode = storage_->CheckAndInitQueryCondition(syncQuery); + if (errCode != E_OK) { + LOGE("[SingleVerDataSync] check sync query failed,errCode=%d", errCode); + SendControlAck(context, message, errCode, controlCmdType); + return -E_WAIT_NEXT_MESSAGE; + } + } + int mode = GetModeByControlCmdType(static_cast(packet->GetcontrolCmdType())); + if (mode >= SyncModeType::INVALID_MODE) { + LOGE("[SingleVerDataSync] invalid mode"); + SendControlAck(context, message, -E_INVALID_ARGS, controlCmdType); + return -E_WAIT_NEXT_MESSAGE; + } + errCode = CheckPermitSendData(mode, context); + if (errCode != E_OK) { + LOGE("[SingleVerDataSync] check sync query failed,errCode=%d", errCode); + SendControlAck(context, message, errCode, controlCmdType); + } + return errCode; } -std::string SingleVerDataSync::GetDeviceId() const +int SingleVerDataSync::SubscribeRequestRecv(SingleVerSyncTaskContext *context, const Message *message) { - return deviceId_; + const SubscribeRequest *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + int errCode = SubscribeRequestRecvPre(context, packet, message); + if (errCode != E_OK) { + return errCode; + } + uint32_t controlCmdType = packet->GetcontrolCmdType(); + std::shared_ptr subscribeManager = context->GetSubscribeManager(); + if (subscribeManager == nullptr) { + LOGE("[SingleVerDataSync] subscribeManager check failed"); + SendControlAck(context, message, -E_NOT_REGISTER, controlCmdType); + return -E_INVALID_ARGS; + } + errCode = storage_->AddSubscribe(packet->GetQuery().GetIdentify(), packet->GetQuery(), packet->IsAutoSubscribe()); + if (errCode != E_OK) { + LOGE("[SingleVerDataSync] add trigger failed,err=%d,label=%s,dev=%s", errCode, label_.c_str(), + STR_MASK(GetDeviceId())); + SendControlAck(context, message, errCode, controlCmdType); + return errCode; + } + errCode = subscribeManager->ReserveRemoteSubscribeQuery(context->GetDeviceId(), packet->GetQuery()); + if (errCode != E_OK) { + LOGE("[SingleVerDataSync] add remote subscribe query failed,err=%d,label=%s,dev=%s", errCode, label_.c_str(), + STR_MASK(GetDeviceId())); + storage_->RemoveSubscribe(packet->GetQuery().GetIdentify()); + SendControlAck(context, message, errCode, controlCmdType); + return errCode; + } + errCode = SendControlAck(context, message, E_OK, controlCmdType); + if (errCode != E_OK) { + storage_->RemoveSubscribe(packet->GetQuery().GetIdentify()); + subscribeManager->DeleteRemoteSubscribeQuery(context->GetDeviceId(), packet->GetQuery()); + LOGE("[SingleVerDataSync] send control msg failed,err=%d,label=%s,dev=%s", errCode, label_.c_str(), + STR_MASK(GetDeviceId())); + return errCode; + } + subscribeManager->ActiveRemoteSubscribeQuery(context->GetDeviceId(), packet->GetQuery()); + return errCode; } -bool SingleVerDataSync::CheckPermitReceiveData(const SingleVerSyncTaskContext *context) +int SingleVerDataSync::UnsubscribeRequestRecv(SingleVerSyncTaskContext *context, const Message *message) { - SecurityOption remoteSecOption = context->GetRemoteSeccurityOption(); - std::string localDeviceId; - if (communicateHandle_ == nullptr || remoteSecOption.securityLabel == NOT_SURPPORT_SEC_CLASSIFICATION) { - return true; + const SubscribeRequest *packet = message->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; } - communicateHandle_->GetLocalIdentity(localDeviceId); - bool isPermitSync = IsPermitLocalDeviceRecvData(localDeviceId, remoteSecOption); - if (isPermitSync) { - return isPermitSync; + uint32_t controlCmdType = packet->GetcontrolCmdType(); + std::shared_ptr subscribeManager = context->GetSubscribeManager(); + if (subscribeManager == nullptr) { + LOGE("[SingleVerDataSync] subscribeManager check failed"); + SendControlAck(context, message, -E_NOT_REGISTER, controlCmdType); + return -E_INVALID_ARGS; } - LOGE("[DataSync][PermitReceiveData] check failed: permitReceive=%d, localDev=%s{private}, seclabel=%d, secflag=%d", - isPermitSync, localDeviceId.c_str(), remoteSecOption.securityLabel, remoteSecOption.securityFlag); - return isPermitSync; + int errCode; + if (subscribeManager->IsRemoteContainSubscribe(context->GetDeviceId(), packet->GetQuery())) { + errCode = storage_->RemoveSubscribe(packet->GetQuery().GetIdentify()); + if (errCode != E_OK) { + LOGE("[SingleVerDataSync] remove trigger failed,err=%d,label=%s,dev=%s", errCode, label_.c_str(), + STR_MASK(GetDeviceId())); + SendControlAck(context, message, errCode, controlCmdType); + return errCode; + } + } + errCode = SendControlAck(context, message, E_OK, controlCmdType); + if (errCode != E_OK) { + LOGE("[SingleVerDataSync] send control msg failed,err=%d,label=%s,dev=%s", errCode, label_.c_str(), + STR_MASK(GetDeviceId())); + return errCode; + } + subscribeManager->RemoveRemoteSubscribeQuery(context->GetDeviceId(), packet->GetQuery()); + metadata_->RemoveQueryFromRecordSet(context->GetDeviceId(), packet->GetQuery().GetIdentify()); + return errCode; } -int SingleVerDataSync::CheckSchemaStrategy(SingleVerSyncTaskContext *context, const Message *message) +bool SingleVerDataSync::IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query) { - SyncStrategy localStrategy = context->GetSyncStrategy(); - if (!context->GetIsSchemaSync()) { - LOGE("[DataSync][CheckSchemaStrategy] isSchemaSync=%d check failed", context->GetIsSchemaSync()); - (void)SendAck(context, message, -E_NEED_ABILITY_SYNC, 0); - return -E_NEED_ABILITY_SYNC; + if (inMsg == nullptr) { + return false; } - if (!localStrategy.permitSync) { - LOGE("[DataSync][CheckSchemaStrategy] Strategy permitSync=%d check failed", localStrategy.permitSync); - (void)SendAck(context, message, -E_SCHEMA_MISMATCH, 0); - return -E_SCHEMA_MISMATCH; + if (inMsg->GetMessageId() != CONTROL_SYNC_MESSAGE) { + return false; } - return E_OK; + const ControlRequestPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + return false; + } + uint32_t controlCmdType = packet->GetcontrolCmdType(); + if (controlCmdType == ControlCmdType::SUBSCRIBE_QUERY_CMD && inMsg->GetMessageType() == TYPE_REQUEST) { + const SubscribeRequest *subPacket = inMsg->GetObject(); + if (packet == nullptr) { + return false; + } + query = subPacket->GetQuery(); + LOGI("[SingleVerDataSync] receive sub scribe query cmd,begin to trigger query auto sync"); + return true; + } + return false; } -void SingleVerDataSync::RemotePushFinished(int sendCode, int mode, uint32_t msgSessionId, uint32_t contextSessionId) +void SingleVerDataSync::ControlAckErrorHandle(const SingleVerSyncTaskContext *context, + const std::shared_ptr &subManager) const { - if ((mode != SyncOperation::PUSH) && (mode != SyncOperation::PUSH_AND_PULL)) { - return; + if (context->GetMode() == SyncModeType::SUBSCRIBE_QUERY) { + // reserve before need clear + subManager->DeleteLocalSubscribeQuery(context->GetDeviceId(), context->GetQuery()); } +} - if ((sendCode == E_OK) && (msgSessionId != 0) && (msgSessionId != contextSessionId)) { - storage_->NotifyRemotePushFinished(deviceId_); +std::string SingleVerDataSync::GetQuerySyncId(const SingleVerSyncTaskContext *context, const std::string &queryId) const +{ + std::string id; +#ifdef RELATIONAL_STORE + if (storage_->GetInterfaceType() == ISyncInterface::SYNC_RELATION) { + id += context->GetQuery().GetRelationTableName(); } +#endif + id += queryId; + return id; } -void SingleVerDataSync::SetPacketId(DataRequestPacket *packet, SingleVerSyncTaskContext *context, uint32_t version) +std::string SingleVerDataSync::GetDeleteSyncId(const SingleVerSyncTaskContext *context) const { - if (version > SOFTWARE_VERSION_RELEASE_2_0) { - context->IncPacketId(); // begin from 1 - std::vector reserved {context->GetPacketId()}; - packet->SetReserved(reserved); + std::string id; + id += context->GetDeviceId(); +#ifdef RELATIONAL_STORE + if (storage_->GetInterfaceType() == ISyncInterface::SYNC_RELATION) { + id += context->GetQuery().GetRelationTableName(); } +#endif + return id; } } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h index 1aee6bfb3cafab2e7e1f56bddd263c32d9e05e5b..bcb2b0b973954280de5e408e9bd3c02c4ad3acac 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync.h @@ -17,12 +17,15 @@ #define SINGLE_VER_DATA_SYNC_NEW_H #include "icommunicator.h" +#include "isync_interface.h" #include "meta_data.h" +#include "parcel.h" +#include "single_ver_data_packet.h" #include "single_ver_kvdb_sync_interface.h" #include "single_ver_sync_task_context.h" +#include "sync_generic_interface.h" #include "sync_types.h" #include "version.h" -#include "parcel.h" namespace DistributedDB { using SendDataItem = SingleVerKvEntry *; @@ -30,134 +33,26 @@ using SendDataItem = SingleVerKvEntry *; struct DataSyncReSendInfo { uint32_t sessionId = 0; uint32_t sequenceId = 0; - TimeStamp start = 0; // means localwatermark + TimeStamp start = 0; // means normal or sync data localwatermark TimeStamp end = 0; + TimeStamp deleteDataStart = 0; // means delete data localwatermark + TimeStamp deleteDataEnd = 0; uint64_t packetId = 0; }; -class DataRequestPacket { -public: - DataRequestPacket() {}; - ~DataRequestPacket(); - - void SetData(std::vector &data); - - const std::vector &GetData() const; - - void SetEndWaterMark(WaterMark waterMark); - - WaterMark GetEndWaterMark() const; - - void SetLocalWaterMark(WaterMark waterMark); - - WaterMark GetLocalWaterMark() const; - - void SetPeerWaterMark(WaterMark waterMark); - - WaterMark GetPeerWaterMark() const; - - void SetSendCode(int32_t errCode); - - int32_t GetSendCode() const; - - void SetMode(int32_t mode); - - int32_t GetMode() const; - - void SetSessionId(uint32_t sessionId); - - uint32_t GetSessionId() const; - - void SetVersion(uint32_t version); - - uint32_t GetVersion() const; - - uint32_t CalculateLen() const; - - void SetReserved(std::vector &reserved); - - std::vector GetReserved() const; - - uint64_t GetPacketId() const; - - void SetFlag(uint32_t flag); - - uint32_t GetFlag() const; - - bool IsLastSequence() const; - - void SetLastSequence(); - - void SetBasicInfo(int sendCode, uint32_t version, WaterMark localMark, WaterMark peerMark, int32_t mode); - -private: - std::vector data_; - WaterMark endWaterMark_ = 0; - WaterMark localWaterMark_ = 0; - WaterMark peerWaterMark_ = 0; - int32_t sendCode_ = 0; - int32_t mode_ = SyncOperation::INVALID; - uint32_t sessionId_ = 0; - uint32_t version_ = SOFTWARE_VERSION_CURRENT; - std::vector reserved_; - uint32_t flag_ = 0; // bit 0 used for isLastSequence - static const uint32_t IS_LAST_SEQUENCE = 0x1; // bit 0 used for isLastSequence, 1: is last, 0: not last -}; - -class DataAckPacket { -public: - DataAckPacket() {}; - ~DataAckPacket() {}; - - void SetData(uint64_t data); - - uint64_t GetData() const; - - void SetRecvCode(int32_t errorCode); - - int32_t GetRecvCode() const; - - void SetVersion(uint32_t version); - - uint32_t GetVersion() const; - - void SetReserved(std::vector &reserved); - - std::vector GetReserved() const; - - uint64_t GetPacketId() const; - - static bool IsPacketIdValid(uint64_t packetId); - - uint32_t CalculateLen() const; - -private: - /* - * data_ is waterMark when revCode_ == LOCAL_WATER_MARK_NOT_INIT || revCode_ == E_OK; - * data_ is timer in milliSeconds when revCode_ == -E_SAVE_DATA_NOTIFY && data_ != 0. - */ - uint64_t data_ = 0; - int32_t recvCode_ = 0; - uint32_t version_ = SOFTWARE_VERSION_CURRENT; - std::vector reserved_; +struct SyncEntry { + std::vector entries; + std::vector compressedEntries; }; class SingleVerDataSync { public: SingleVerDataSync(); - ~SingleVerDataSync(); + virtual ~SingleVerDataSync(); DISABLE_COPY_ASSIGN_MOVE(SingleVerDataSync); - static int Serialization(uint8_t *buffer, uint32_t length, const Message *inMsg); - - static int DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); - - static uint32_t CalculateLen(const Message *inMsg); - - static int RegisterTransformFunc(); - - int Initialize(IKvDBSyncInterface *inStorage, ICommunicator *inCommunicateHandle, + int Initialize(ISyncInterface *inStorage, ICommunicator *inCommunicateHandle, std::shared_ptr &inMetadata, const std::string &deviceId); int PushStart(SingleVerSyncTaskContext *context); @@ -168,52 +63,68 @@ public: int PullResponseStart(SingleVerSyncTaskContext *context); - int RequestRecv(SingleVerSyncTaskContext *context, const Message *message, WaterMark &pullEndWatermark); + int DataRequestRecv(SingleVerSyncTaskContext *context, const Message *message, WaterMark &pullEndWatermark); int AckRecv(SingleVerSyncTaskContext *context, const Message *message); void SendSaveDataNotifyPacket(SingleVerSyncTaskContext *context, uint32_t pktVersion, uint32_t sessionId, - uint32_t sequenceId); + uint32_t sequenceId, uint32_t inMsgId); - void SendAck(SingleVerSyncTaskContext *context, uint32_t sessionId, uint32_t sequenceId, uint64_t packetId); + virtual int SendDataAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, + WaterMark maxSendDataTime); int32_t ReSend(SingleVerSyncTaskContext *context, DataSyncReSendInfo reSendInfo); - int CheckPermitSendData(int mode, SingleVerSyncTaskContext *context); + int CheckPermitSendData(int inMode, SingleVerSyncTaskContext *context); std::string GetLabel() const; std::string GetDeviceId() const; -private: + bool WaterMarkErrHandle(SyncType syncType, SingleVerSyncTaskContext *context, const Message *message); + + int ControlCmdStart(SingleVerSyncTaskContext *context); + + int ControlCmdRequestRecv(SingleVerSyncTaskContext *context, const Message *message); + + int ControlCmdAckRecv(SingleVerSyncTaskContext *context, const Message *message); + + bool IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query); + + void ControlAckErrorHandle(const SingleVerSyncTaskContext *context, + const std::shared_ptr &subManager) const; + +protected: static const int SEND_FINISHED = 0xff; static const int LOCAL_WATER_MARK_NOT_INIT = 0xaa; static const int PEER_WATER_MARK_NOT_INIT = 0x55; static const int WATER_MARK_INVALID = 0xbb; static const int MTU_SIZE = 28311552; // 27MB - static int AckPacketCalculateLen(const Message *inMsg, uint32_t &len); + TimeStamp GetMaxSendDataTime(const std::vector &inData); - static int DataPacketCalculateLen(const Message *inMsg, uint32_t &len); + TimeStamp GetMinSendDataTime(const std::vector &inData, WaterMark localMark); - static int DataPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + void FillDataRequestPacket(DataRequestPacket *packet, SingleVerSyncTaskContext *context, + SyncEntry &syncData, int sendCode, int mode); - static int DataPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + int RequestStart(SingleVerSyncTaskContext *context, int mode); - static int AckPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + SyncTimeRange GetSyncDataTimeRange(SyncType syncType, SingleVerSyncTaskContext *context, + const std::vector &inData, UpdateWaterMark &isUpdate); - static int AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + SyncTimeRange GetRecvDataTimeRange(SyncType syncType, SingleVerSyncTaskContext *context, + const std::vector &data, UpdateWaterMark &isUpdate); - static bool IsPacketValid(const Message *inMsg); + SyncTimeRange GetFullSyncDataTimeRange(const std::vector &inData, WaterMark localMark, + UpdateWaterMark &isUpdate); - static TimeStamp GetMaxSendDataTime(const std::vector &inData, bool isNeedInit, - WaterMark localMark = 0); - - static TimeStamp GetMinSendDataTime(const std::vector &inData, WaterMark localMark); + SyncTimeRange GetQuerySyncDataTimeRange(const std::vector &inData, WaterMark localMark, + WaterMark deletelocalMark, UpdateWaterMark &isUpdate); int GetData(SingleVerSyncTaskContext *context, std::vector &outData, size_t packetSize); - int GetDataWithRerformanceRecord(SingleVerSyncTaskContext *context, std::vector &outData); + int GetDataWithPerformanceRecord(SingleVerSyncTaskContext *context, SyncEntry &syncOutData); int Send(SingleVerSyncTaskContext *context, const Message *message, const CommErrHandler &handler, uint32_t packetLen); @@ -222,13 +133,30 @@ private: int GetNextUnsyncData(SingleVerSyncTaskContext *context, std::vector &outData, size_t packetSize); - int SaveData(const SingleVerSyncTaskContext *context, const std::vector &inData); + int GetMessageId(SyncType syncType) const; + + int SaveData(const SingleVerSyncTaskContext *context, const std::vector &inData, SyncType curType, + const QuerySyncObject &query); + + SyncTimeRange ReviseLocalMark(SyncType syncType, SyncTimeRange &dataTimeRange, UpdateWaterMark updateMark); + + int SaveLocalWaterMark(SyncType syncType, const SingleVerSyncTaskContext *context, + SyncTimeRange dataTimeRange, bool isCheckBeforUpdate = false) const; + + void GetLocalWaterMark(SyncType syncType, const std::string &queryIdentify, const SingleVerSyncTaskContext *context, + WaterMark &watermark) const; + + void GetPeerWaterMark(SyncType syncType, const std::string &queryIdentify, const DeviceID &deviceId, + WaterMark &watermark) const; - int SaveLocalWaterMark(const DeviceID &deviceId, WaterMark waterMark); + void GetPeerDeleteSyncWaterMark(const DeviceID &deviceId, WaterMark &waterMark); - int SavePeerWaterMark(const DeviceID &deviceId, WaterMark waterMark); + void GetLocalDeleteSyncWaterMark(const SingleVerSyncTaskContext *context, WaterMark &waterMark) const; - int RemoveDeviceData(SingleVerSyncTaskContext *context, const Message *message, WaterMark maxSendDataTime); + int RemoveDeviceDataHandle(SingleVerSyncTaskContext *context, const Message *message, WaterMark maxSendDataTime); + + int DealRemoveDeviceDataByAck(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, + const std::vector &reserved); void TransSendDataItemToLocal(const SingleVerSyncTaskContext *context, const std::vector &data); @@ -236,25 +164,21 @@ private: void TransDbDataItemToSendDataItem(const SingleVerSyncTaskContext *context, std::vector &outData); - int SendDataPacket(const DataRequestPacket *packet, SingleVerSyncTaskContext *context); - - void SetAckData(DataAckPacket &ackPacket, SingleVerSyncTaskContext *context, int32_t recvCode, - WaterMark maxSendDataTime) const; - - int SendAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, - WaterMark maxSendDataTime); + int SendDataPacket(SyncType syncType, const DataRequestPacket *packet, SingleVerSyncTaskContext *context); - int SendLocalWaterMarkAck(SingleVerSyncTaskContext *context, const Message *message); + void UpdateQueryPeerWaterMark(SyncType syncType, const std::string &queryId, SyncTimeRange &dataTime, + const SingleVerSyncTaskContext *context, UpdateWaterMark isUpdateWaterMark); - void UpdatePeerWaterMark(const SingleVerSyncTaskContext *context, WaterMark peerWatermark); + void UpdatePeerWaterMark(SyncType syncType, const std::string &queryId, const SingleVerSyncTaskContext *context, + WaterMark peerWatermark, WaterMark peerDeletedWatermark); std::string GetLocalDeviceName(); - std::string TransferForeignOrigDevName(const std::string &deviceName); + std::string TransferForeignOrigDevName(const std::string &deviceName, const std::string &localHashName); - std::string TransferLocalOrigDevName(const std::string &origName); + int DoAbilitySyncIfNeed(SingleVerSyncTaskContext *context, const Message *message, bool isControlMsg = false); - int RequestRecvPre(SingleVerSyncTaskContext *context, const Message *message); + int DataRequestRecvPre(SingleVerSyncTaskContext *context, const Message *message); void GetPullEndWatermark(const SingleVerSyncTaskContext *context, const DataRequestPacket *packet, WaterMark &pullEndWatermark) const; @@ -262,15 +186,6 @@ private: int DealWaterMarkException(SingleVerSyncTaskContext *context, WaterMark ackWaterMark, const std::vector &reserved); - static int DataPacketSyncerPartSerialization(Parcel &parcel, const DataRequestPacket *packet); - - static int DataPacketSyncerPartDeSerialization(Parcel &parcel, DataRequestPacket *packet, uint32_t packLen, - uint32_t length, uint32_t version); - - static int AckPacketSyncerPartSerializationV1(Parcel &parcel, const DataAckPacket *packet); - - static int AckPacketSyncerPartDeSerializationV1(Parcel &parcel, DataAckPacket &packet); - int RunPermissionCheck(SingleVerSyncTaskContext *context, const Message *message, const DataRequestPacket *packet); @@ -279,8 +194,7 @@ private: int SendReSendPacket(const DataRequestPacket *packet, SingleVerSyncTaskContext *context, uint32_t sessionId, uint32_t sequenceId); - int SendPullResponseDataPkt(int ackCode, std::vector &inData, - SingleVerSyncTaskContext *context); + int SendPullResponseDataPkt(int ackCode, SyncEntry &syncOutData, SingleVerSyncTaskContext *context); void SetPacketId(DataRequestPacket *packet, SingleVerSyncTaskContext *context, uint32_t version); @@ -292,10 +206,56 @@ private: int CheckSchemaStrategy(SingleVerSyncTaskContext *context, const Message *message); - void RemotePushFinished(int sendCode, int mode, uint32_t msgSessionId, uint32_t contetSessionId); + void RemotePushFinished(int sendCode, int inMode, uint32_t msgSessionId, uint32_t contextSessionId); + + void PushAndPUllKeyRevokHandle(SingleVerSyncTaskContext *context); + + void SetAckPacket(DataAckPacket &ackPacket, SingleVerSyncTaskContext *context, const DataRequestPacket *packet, + int32_t recvCode, WaterMark maxSendDataTime); + + int GetReSendData(SyncEntry &syncData, SingleVerSyncTaskContext *context, + DataSyncReSendInfo reSendInfo); + + int RemoveDeviceDataIfNeed(SingleVerSyncTaskContext *context); + + int GetReSendMode(int mode, uint32_t sequenceId, SyncType syncType); + + void FillRequestReSendPacket(const SingleVerSyncTaskContext *context, DataRequestPacket *packet, + DataSyncReSendInfo reSendInfo, SyncEntry &syncData, int sendCode); + + void UpdateMtuSize(); + + DataSizeSpecInfo GetDataSizeSpecInfo(size_t packetSize); + + int AckMsgErrnoCheck(const SingleVerSyncTaskContext *context, const Message *message) const; + + bool QuerySyncCheck(const SingleVerSyncTaskContext *context) const; + + int InterceptData(SyncEntry &syncEntry); + + int RequestQueryCheck(const DataRequestPacket *packet) const; + int ControlCmdStartCheck(SingleVerSyncTaskContext *context); + + void FillControlRequestPacket(ControlRequestPacket *packet, SingleVerSyncTaskContext *context); + + int SendControlPacket(const ControlRequestPacket *packet, SingleVerSyncTaskContext *context); + + ControlCmdType GetControlCmdType(int mode); + int GetModeByControlCmdType(ControlCmdType controlCmd); + + int ControlCmdRequestRecvPre(SingleVerSyncTaskContext *context, const Message *message); + int SubscribeRequestRecvPre(SingleVerSyncTaskContext *context, const SubscribeRequest *packet, + const Message *message); + int SubscribeRequestRecv(SingleVerSyncTaskContext *context, const Message *message); + int UnsubscribeRequestRecv(SingleVerSyncTaskContext *context, const Message *message); + int SendControlAck(SingleVerSyncTaskContext *context, const Message *message, int32_t recvCode, + uint32_t controlCmdType, const CommErrHandler &handler = nullptr); + + std::string GetQuerySyncId(const SingleVerSyncTaskContext *context, const std::string &queryId) const; + std::string GetDeleteSyncId(const SingleVerSyncTaskContext *context) const; uint32_t mtuSize_; - SingleVerKvDBSyncInterface* storage_; + SyncGenericInterface* storage_; ICommunicator* communicateHandle_; std::shared_ptr metadata_; std::string label_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h index 0c18c9bde17bf08c02eefcdf2174034d4c684b63..6d77f0ccfc7a011fa5aea8f8d811cb7040223d05 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_data_sync_with_sliding_window.h @@ -16,8 +16,8 @@ #ifndef SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H #define SINGLE_VER_DATA_SYNC_WITH_SLIDING_WINDOW_H -#include "sliding_window_sender.h" #include "sliding_window_receiver.h" +#include "sliding_window_sender.h" namespace DistributedDB { class SingleVerDataSyncWithSlidingWindow { diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..95396a1160d8e616258185b0cd0cc803b611e87a --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.cpp @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "single_ver_kv_syncer.h" + +#include +#include +#include + +#include "db_common.h" +#include "ikvdb_sync_interface.h" +#include "log_print.h" +#include "meta_data.h" +#include "single_ver_sync_engine.h" +#include "sqlite_single_ver_natural_store.h" + +namespace DistributedDB { +SingleVerKVSyncer::SingleVerKVSyncer() + : autoSyncEnable_(false), + triggerSyncTask_(true) +{ +} + +SingleVerKVSyncer::~SingleVerKVSyncer() +{ +} + +void SingleVerKVSyncer::EnableAutoSync(bool enable) +{ + LOGI("[Syncer] EnableAutoSync enable = %d, Label=%s", enable, label_.c_str()); + if (autoSyncEnable_ == enable) { + return; + } + + autoSyncEnable_ = enable; + if (!enable) { + return; + } + + if (!initialized_) { + LOGE("[Syncer] Syncer has not Init"); + return; + } + + std::vector devices; + GetOnlineDevices(devices); + if (devices.empty()) { + LOGI("[Syncer] EnableAutoSync no online devices"); + return; + } + int errCode = Sync(devices, SyncModeType::AUTO_PUSH, nullptr, nullptr, false); + if (errCode != E_OK) { + LOGE("[Syncer] sync start by EnableAutoSync failed err %d", errCode); + } +} + +int SingleVerKVSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +{ + if (metadata_ == nullptr) { + return -E_NOT_INIT; + } + return metadata_->EraseDeviceWaterMark(deviceId, isNeedHash); +} + +// Local data changed callback +void SingleVerKVSyncer::LocalDataChanged(int notifyEvent) +{ + if (!initialized_) { + LOGE("[Syncer] Syncer has not Init"); + return; + } + + if (notifyEvent != SQLITE_GENERAL_FINISH_MIGRATE_EVENT && + notifyEvent != SQLITE_GENERAL_NS_PUT_EVENT) { + LOGD("[Syncer] ignore event:%d", notifyEvent); + return; + } + if (!triggerSyncTask_) { + LOGI("[Syncer] some sync task is scheduling"); + return; + } + triggerSyncTask_ = false; + RefObject::IncObjRef(syncEngine_); + // To avoid many task were produced and waiting in the queue. For example, put value in a loop. + // It will consume thread pool resources, so other task will delay until these task finish. + // In extreme situation, 10 thread run the localDataChanged task and 1 task waiting in queue. + int errCode = RuntimeContext::GetInstance()->ScheduleTask([this] { + triggerSyncTask_ = true; + std::vector devices; + GetOnlineDevices(devices); + if (devices.empty()) { + LOGI("[Syncer] LocalDataChanged no online devices, Label=%s", label_.c_str()); + RefObject::DecObjRef(syncEngine_); + return; + } + if (!TryFullSync(devices)) { + TriggerSubQuerySync(devices); + } + RefObject::DecObjRef(syncEngine_); + }); + // if task schedule failed, but triggerSyncTask_ is not set to true, other thread may skip the schedule time + // when task schedule failed, it means unormal status, it is unable to shedule next time probably + // so it is ok if other thread skip the schedule if last task schedule failed + if (errCode != E_OK) { + triggerSyncTask_ = true; + LOGE("[TriggerSync] LocalDataChanged retCode:%d", errCode); + RefObject::DecObjRef(syncEngine_); + } + return; +} + +// remote device online callback +void SingleVerKVSyncer::RemoteDataChanged(const std::string &device) +{ + LOGI("[SingleVerKVSyncer] device online dev %s", STR_MASK(device)); + // while remote db is online again, need to do abilitySync + static_cast(syncEngine_)->SetIsNeedResetAbilitySync(device, true); + if (autoSyncEnable_) { + RefObject::IncObjRef(syncEngine_); + int retCode = RuntimeContext::GetInstance()->ScheduleTask([this, device] { + std::vector devices; + devices.push_back(device); + int errCode = Sync(devices, SyncModeType::AUTO_PUSH, nullptr, nullptr, false); + if (errCode != E_OK) { + LOGE("[SingleVerKVSyncer] sync start by RemoteDataChanged failed err %d", errCode); + } + RefObject::DecObjRef(syncEngine_); + }); + if (retCode != E_OK) { + LOGE("[AutoLaunch] RemoteDataChanged triggler sync retCode:%d", retCode); + RefObject::DecObjRef(syncEngine_); + } + } + // db online again ,trigger subscribe + // if remote device online, subscribequery num is 0 + std::vector syncQueries; + static_cast(syncEngine_)->GetLocalSubscribeQueries(device, syncQueries); + if (syncQueries.size() == 0) { + LOGI("no need to trigger auto subscribe"); + return; + } + LOGI("[SingleVerKVSyncer] trigger local subscribe sync, queryNums=%d", syncQueries.size()); + for (const auto &query : syncQueries) { + TriggerSubscribe(device, query); + } + static_cast(syncEngine_)->PutUnfiniedSubQueries(device, syncQueries); +} + +void SingleVerKVSyncer::QueryAutoSync(const InternalSyncParma ¶m) +{ + LOGI("[SingleVerKVSyncer] trigger query sync_type=%u,dev=%s", param.mode, GetSyncDevicesStr(param.devices).c_str()); + RefObject::IncObjRef(syncEngine_); + int retCode = RuntimeContext::GetInstance()->ScheduleTask([this, param] { + int errCode = Sync(param); + if (errCode != E_OK) { + LOGE("[SingleVerKVSyncer] sync start by QueryAutoSync failed err %d", errCode); + } + RefObject::DecObjRef(syncEngine_); + }); + if (retCode != E_OK) { + LOGE("[SingleVerKVSyncer] QueryAutoSync triggler sync retCode:%d", retCode); + RefObject::DecObjRef(syncEngine_); + } +} + +int SingleVerKVSyncer::SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, + const std::vector &devices) const +{ + if (!isQuerySync) { + return E_OK; + } + int errCode = static_cast(syncInterface_)->CheckAndInitQueryCondition(query); + if (errCode != E_OK) { + LOGE("[SingleVerKVSyncer] QuerySyncObject check failed"); + return errCode; + } + if (mode != SUBSCRIBE_QUERY) { + return E_OK; + } + if (query.HasLimit() || query.HasOrderBy()) { + LOGE("[SingleVerKVSyncer] subscribe query not support limit,offset or orderby"); + return -E_NOT_SUPPORT; + } + if (devices.size() > MAX_DEVICES_NUM) { + LOGE("[SingleVerKVSyncer] devices is overlimit"); + return -E_MAX_LIMITS; + } + return syncEngine_->SubscribeLimitCheck(devices, query); +} + +void SingleVerKVSyncer::TriggerSubscribe(const std::string &device, const QuerySyncObject &query) +{ + RefObject::IncObjRef(syncEngine_); + int retCode = RuntimeContext::GetInstance()->ScheduleTask([this, device, query] { + std::vector devices; + devices.push_back(device); + SyncParma param; + param.devices = devices; + param.mode = SyncModeType::AUTO_SUBSCRIBE_QUERY; + param.onComplete = nullptr; + param.onFinalize = nullptr; + param.wait = false; + param.isQuerySync = true; + param.syncQuery = query; + int errCode = Sync(param); + if (errCode != E_OK) { + LOGE("[SingleVerKVSyncer] subscribe start by RemoteDataChanged failed err %d", errCode); + } + RefObject::DecObjRef(syncEngine_); + }); + if (retCode != E_OK) { + LOGE("[Syncer] triggler query subscribe start failed err %d", retCode); + RefObject::DecObjRef(syncEngine_); + } +} + +bool SingleVerKVSyncer::TryFullSync(const std::vector &devices) +{ + if (!autoSyncEnable_) { + LOGD("[Syncer] autoSync no enable"); + return false; + } + int errCode = Sync(devices, SyncModeType::AUTO_PUSH, nullptr, nullptr, false); + if (errCode != E_OK) { + LOGE("[Syncer] sync start by RemoteDataChanged failed err %d", errCode); + return false; + } + return true; +} + +void SingleVerKVSyncer::TriggerSubQuerySync(const std::vector &devices) +{ + int errCode; + for (auto &device : devices) { + std::vector queries; + static_cast(syncEngine_)->GetRemoteSubscribeQueries(device, queries); + for (auto &query : queries) { + std::string queryId = query.GetIdentify(); + uint64_t lastTimestamp = metadata_->GetQueryLastTimestamp(device, queryId); + WaterMark queryWaterMark = 0; + errCode = metadata_->GetSendQueryWaterMark(queryId, device, queryWaterMark, false); + if (errCode != E_OK) { + LOGE("[Syncer] get queryId=%s,dev=%s watermark failed", STR_MASK(queryId), STR_MASK(device)); + continue; + } + if (lastTimestamp < queryWaterMark || lastTimestamp == 0) { + continue; + } + LOGD("[Syncer] lastTime=%llu vs WaterMark=%llu,trigger queryId=%s,dev=%s", lastTimestamp, queryWaterMark, + STR_MASK(queryId), STR_MASK(device)); + InternalSyncParma param; + std::vector targetDevices; + targetDevices.push_back(device); + param.devices = targetDevices; + param.mode = SyncModeType::AUTO_PUSH; + param.isQuerySync = true; + param.syncQuery = query; + QueryAutoSync(param); + } + } +} +} // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.h new file mode 100644 index 0000000000000000000000000000000000000000..cb02a472c2bcc4cae744bccf3fc0d22b63397ef4 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_kv_syncer.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef KV_SYNCER_H +#define KV_SYNCER_H + +#include "single_ver_syncer.h" + +namespace DistributedDB { +class SingleVerKVSyncer final : public SingleVerSyncer { +public: + SingleVerKVSyncer(); + ~SingleVerKVSyncer() override; + + // Enable auto sync function + void EnableAutoSync(bool enable) override; + + // delete specified device's watermark + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; + + // Local data changed callback + void LocalDataChanged(int notifyEvent) override; + + // Remote data changed callback + void RemoteDataChanged(const std::string &device) override; + + void QueryAutoSync(const InternalSyncParma ¶m) override; + + void TriggerSubscribe(const std::string &device, const QuerySyncObject &query); + +protected: + int SyncConditionCheck(QuerySyncObject &query, int mode, bool isQuerySync, + const std::vector &devices) const override; + +private: + // if trigger full sync, no need to trigger query sync again + bool TryFullSync(const std::vector &devices); + void TriggerSubQuerySync(const std::vector &devices); + bool autoSyncEnable_; + std::atomic triggerSyncTask_; +}; +} // namespace DistributedDB + +#endif // KV_SYNCER_H diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..31fc0d11482587a21a8e033ea65ef83c591bf528 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "single_ver_relational_syncer.h" +#ifdef RELATIONAL_STORE +#include "db_common.h" +#include "relational_db_sync_interface.h" +#include "single_ver_sync_engine.h" + +namespace DistributedDB { +int SingleVerRelationalSyncer::PrepareSync(const SyncParma ¶m, uint32_t syncId) +{ + const auto &syncInterface = static_cast(syncInterface_); + std::vector tablesQuery; + if (param.isQuerySync) { + tablesQuery.push_back(param.syncQuery); + } else { + tablesQuery = syncInterface->GetTablesQuery(); + } + std::set subSyncIdSet; + int errCode = GenerateEachSyncTask(param, syncId, tablesQuery, subSyncIdSet); + if (errCode != E_OK) { + DoRollBack(subSyncIdSet); + return errCode; + } + return E_OK; +} + +int SingleVerRelationalSyncer::GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, + const std::vector &tablesQuery, std::set &subSyncIdSet) +{ + SyncParma subParam = param; + subParam.isQuerySync = true; + int errCode = E_OK; + for (const QuerySyncObject &table : tablesQuery) { + uint32_t subSyncId = GenerateSyncId(); + LOGI("[SingleVerRelationalSyncer] SubSyncId %d create by SyncId %d, tableName = %s", + subSyncId, syncId, table.GetRelationTableName().c_str()); + subParam.syncQuery = table; + subParam.onComplete = std::bind(&SingleVerRelationalSyncer::DoOnSubSyncComplete, this, subSyncId, + syncId, param, std::placeholders::_1); + { + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].insert(subSyncId); + } + errCode = GenericSyncer::PrepareSync(subParam, subSyncId); + if (errCode != E_OK) { + LOGW("[SingleVerRelationalSyncer] PrepareSync failed errCode:%d", errCode); + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].erase(subSyncId); + break; + } + subSyncIdSet.insert(subSyncId); + } + return errCode; +} + +void SingleVerRelationalSyncer::DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, + const SyncParma ¶m, const std::map &devicesMap) +{ + static std::map statusMap = { + { static_cast(SyncOperation::OP_FINISHED_ALL), OK }, + { static_cast(SyncOperation::OP_TIMEOUT), TIME_OUT }, + { static_cast(SyncOperation::OP_PERMISSION_CHECK_FAILED), PERMISSION_CHECK_FORBID_SYNC }, + { static_cast(SyncOperation::OP_COMM_ABNORMAL), COMM_FAILURE }, + { static_cast(SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE), SECURITY_OPTION_CHECK_ERROR }, + { static_cast(SyncOperation::OP_EKEYREVOKED_FAILURE), EKEYREVOKED_ERROR }, + { static_cast(SyncOperation::OP_SCHEMA_INCOMPATIBLE), SCHEMA_MISMATCH }, + { static_cast(SyncOperation::OP_BUSY_FAILURE), BUSY }, + { static_cast(SyncOperation::OP_QUERY_FORMAT_FAILURE), INVALID_QUERY_FORMAT }, + { static_cast(SyncOperation::OP_QUERY_FIELD_FAILURE), INVALID_QUERY_FIELD }, + { static_cast(SyncOperation::OP_NOT_SUPPORT), NOT_SUPPORT }, + }; + bool allFinish = true; + { + std::lock_guard lockGuard(syncMapLock_); + fullSyncIdMap_[syncId].erase(subSyncId); + allFinish = fullSyncIdMap_[syncId].empty(); + for (const auto &item : devicesMap) { + DBStatus status = DB_ERROR; + if (statusMap.find(item.second) != statusMap.end()) { + status = statusMap[item.second]; + } + resMap_[syncId][item.first][param.syncQuery.GetRelationTableName()] = status; + } + } + if (allFinish) { + DoOnComplete(param, syncId); + } +} + +void SingleVerRelationalSyncer::DoRollBack(std::set &subSyncIdSet) +{ + for (const auto &removeId : subSyncIdSet) { + int retCode = RemoveSyncOperation((int)removeId); + if (retCode != E_OK) { + LOGW("[SingleVerRelationalSyncer] RemoveSyncOperation failed errCode:%d, syncId:%d", retCode, removeId); + } + } +} + +void SingleVerRelationalSyncer::DoOnComplete(const SyncParma ¶m, uint32_t syncId) +{ + if (!param.relationOnComplete) { + return; + } + std::map> syncRes; + std::map> tmpMap; + { + std::lock_guard lockGuard(syncMapLock_); + tmpMap = resMap_[syncId]; + } + for (const auto &devicesRes : tmpMap) { + for (const auto &tableRes : devicesRes.second) { + syncRes[devicesRes.first].push_back( + {tableRes.first, static_cast(tableRes.second)}); + } + } + param.relationOnComplete(syncRes); + { + std::lock_guard lockGuard(syncMapLock_); + resMap_.erase(syncId); + } +} + +void SingleVerRelationalSyncer::EnableAutoSync(bool enable) +{ +} + +int SingleVerRelationalSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) +{ + return E_OK; +} + +void SingleVerRelationalSyncer::LocalDataChanged(int notifyEvent) +{ +} + +void SingleVerRelationalSyncer::RemoteDataChanged(const std::string &device) +{ +} +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h new file mode 100644 index 0000000000000000000000000000000000000000..159a011eaadaae1938b108c1be16add91083fc17 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_relational_syncer.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_SYNCER_H +#define RELATIONAL_SYNCER_H +#ifdef RELATIONAL_STORE +#include "single_ver_syncer.h" +namespace DistributedDB { +class SingleVerRelationalSyncer final : public SingleVerSyncer { +public: + SingleVerRelationalSyncer() = default; + ~SingleVerRelationalSyncer() override = default; + + void EnableAutoSync(bool enable) override; + + int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; + + void LocalDataChanged(int notifyEvent) override; + +protected: + + int PrepareSync(const SyncParma ¶m, uint32_t syncId) override; + + void RemoteDataChanged(const std::string &device) override; + +private: + + int GenerateEachSyncTask(const SyncParma ¶m, uint32_t syncId, + const std::vector &tablesQuery, std::set &subSyncIdSet); + + void DoRollBack(std::set &subSyncIdSet); + + void DoOnComplete(const SyncParma ¶m, uint32_t syncId); + void DoOnSubSyncComplete(const uint32_t subSyncId, const uint32_t syncId, + const SyncParma ¶m, const std::map &devicesMap); + + mutable std::mutex syncMapLock_; + std::map> fullSyncIdMap_; + std::map>> resMap_; +}; +} + +#endif +#endif // RELATIONAL_SYNCER_H diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..94defbb15c166a996df422d82f135c7e2ac5f23f --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.cpp @@ -0,0 +1,712 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "single_ver_serialize_manager.h" +#include "icommunicator.h" +#include "log_print.h" +#include "message_transform.h" +#include "parcel.h" +#include "generic_single_ver_kv_entry.h" +#include "sync_types.h" +#include "version.h" +#include "db_common.h" + +namespace DistributedDB { +int SingleVerSerializeManager::Serialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + if ((buffer == nullptr) || !(IsPacketValid(inMsg))) { + return -E_MESSAGE_ID_ERROR; + } + if (inMsg->GetMessageId() == CONTROL_SYNC_MESSAGE) { + return ControlSerialization(buffer, length, inMsg); + } + return DataSerialization(buffer, length, inMsg); +} + +int SingleVerSerializeManager::DataSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + return DataPacketSerialization(buffer, length, inMsg); + case TYPE_RESPONSE: + case TYPE_NOTIFY: + return AckPacketSerialization(buffer, length, inMsg); + default: + return -E_MESSAGE_TYPE_ERROR; + } +} + +int SingleVerSerializeManager::ControlSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + return ControlPacketSerialization(buffer, length, inMsg); + case TYPE_RESPONSE: + return AckControlPacketSerialization(buffer, length, inMsg); + default: + return -E_MESSAGE_TYPE_ERROR; + } +} + +int SingleVerSerializeManager::DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + if ((buffer == nullptr) || !(IsPacketValid(inMsg))) { + return -E_MESSAGE_ID_ERROR; + } + if (inMsg->GetMessageId() == CONTROL_SYNC_MESSAGE) { + return ControlDeSerialization(buffer, length, inMsg); + } + return DataDeSerialization(buffer, length, inMsg); +} + +int SingleVerSerializeManager::DataDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + return DataPacketDeSerialization(buffer, length, inMsg); + case TYPE_RESPONSE: + case TYPE_NOTIFY: + return AckPacketDeSerialization(buffer, length, inMsg); + default: + return -E_MESSAGE_TYPE_ERROR; + } +} + +int SingleVerSerializeManager::ControlDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + return ControlPacketDeSerialization(buffer, length, inMsg); + case TYPE_RESPONSE: + return AckControlPacketDeSerialization(buffer, length, inMsg); + default: + return -E_MESSAGE_TYPE_ERROR; + } +} + +uint32_t SingleVerSerializeManager::CalculateLen(const Message *inMsg) +{ + if (!(IsPacketValid(inMsg))) { + return 0; + } + if (inMsg->GetMessageId() == CONTROL_SYNC_MESSAGE) { + return CalculateControlLen(inMsg); + } + return CalculateDataLen(inMsg); +} + +uint32_t SingleVerSerializeManager::CalculateDataLen(const Message *inMsg) +{ + uint32_t len = 0; + int errCode; + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + errCode = DataPacketCalculateLen(inMsg, len); + if (errCode != E_OK) { + LOGE("[CalculateDataLen] calculate data request packet len failed, errCode=%d", errCode); + return 0; + } + return len; + case TYPE_RESPONSE: + case TYPE_NOTIFY: + errCode = AckPacketCalculateLen(inMsg, len); + if (errCode != E_OK) { + LOGE("[CalculateDataLen] calculate data notify packet len failed errCode=%d", errCode); + return 0; + } + return len; + default: + return 0; + } +} + +uint32_t SingleVerSerializeManager::CalculateControlLen(const Message *inMsg) +{ + uint32_t len = 0; + int errCode; + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + errCode = ControlPacketCalculateLen(inMsg, len); + if (errCode != E_OK) { + LOGE("[CalculateControlLen] calculate control request packet len failed, errCode=%d", errCode); + return 0; + } + return len; + case TYPE_RESPONSE: + case TYPE_NOTIFY: + errCode = AckControlPacketCalculateLen(inMsg, len); + if (errCode != E_OK) { + LOGE("[CalculateControlLen] calculate control request packet len failed, errCode=%d", errCode); + return 0; + } + return len; + default: + return 0; + } +} + +int SingleVerSerializeManager::RegisterTransformFunc() +{ + TransformFunc func; + func.computeFunc = std::bind(&SingleVerSerializeManager::CalculateLen, std::placeholders::_1); + func.serializeFunc = std::bind(&SingleVerSerializeManager::Serialization, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3); + func.deserializeFunc = std::bind(&SingleVerSerializeManager::DeSerialization, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3); + + int errCode = MessageTransform::RegTransformFunction(QUERY_SYNC_MESSAGE, func); + if (errCode != E_OK) { + return errCode; + } + errCode = MessageTransform::RegTransformFunction(DATA_SYNC_MESSAGE, func); + if (errCode != E_OK) { + return errCode; + } + return MessageTransform::RegTransformFunction(CONTROL_SYNC_MESSAGE, func); +} + +int SingleVerSerializeManager::DataPacketSyncerPartSerialization(Parcel &parcel, const DataRequestPacket *packet) +{ + parcel.WriteUInt64(packet->GetEndWaterMark()); + parcel.WriteUInt64(packet->GetLocalWaterMark()); + parcel.WriteUInt64(packet->GetPeerWaterMark()); + parcel.WriteInt(packet->GetSendCode()); + parcel.WriteInt(packet->GetMode()); + parcel.WriteUInt32(packet->GetSessionId()); + parcel.WriteVector(packet->GetReserved()); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + if (packet->GetVersion() > SOFTWARE_VERSION_RELEASE_2_0) { + parcel.WriteUInt32(packet->GetFlag()); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + } + parcel.EightByteAlign(); + return E_OK; +} + +int SingleVerSerializeManager::DataPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + auto packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + Parcel parcel(buffer, length); + + // version + int errCode = parcel.WriteUInt32(packet->GetVersion()); + if (errCode != E_OK) { + LOGE("[DataPacketSerialization] Serialize version failed"); + return errCode; + } + // sendDataItems + errCode = GenericSingleVerKvEntry::SerializeDatas( + (packet->IsCompressData() ? std::vector {} : packet->GetData()), parcel, packet->GetVersion()); + if (errCode != E_OK) { + LOGE("[DataPacketSerialization] Serialize Data failed"); + return errCode; + } + + // data sync + errCode = DataPacketSyncerPartSerialization(parcel, packet); + if (errCode != E_OK) { + LOGE("[DataPacketSerialization] Serialize Data failed"); + return errCode; + } + if (inMsg->GetMessageId() == QUERY_SYNC_MESSAGE) { + errCode = DataPacketQuerySyncSerialization(parcel, packet); // for query sync + if (errCode != E_OK) { + return errCode; + } + } + if (packet->IsCompressData()) { + // serialize compress data + errCode = GenericSingleVerKvEntry::SerializeCompressedDatas(packet->GetData(), packet->GetCompressData(), + parcel, packet->GetVersion(), packet->GetCompressAlgo()); + if (errCode != E_OK) { + LOGE("[DataPacketSerialization] Serialize compress Data failed"); + return errCode; + } + } + return E_OK; +} + +int SingleVerSerializeManager::DataPacketQuerySyncSerialization(Parcel &parcel, const DataRequestPacket *packet) +{ + // deleted record send watermark + int errCode = parcel.WriteUInt64(packet->GetDeletedWaterMark()); + if (errCode != E_OK) { + LOGE("[QuerySerialization] Serialize deleted record send watermark failed!"); + return errCode; + } + + // query identify + QuerySyncObject queryObj = packet->GetQuery(); + errCode = parcel.WriteString(packet->GetQueryId()); + if (errCode != E_OK) { + LOGE("[QuerySerialization] Serialize query id failed!"); + return errCode; + } + if ((packet->GetVersion() > SOFTWARE_VERSION_RELEASE_4_0) || packet->GetMode() != QUERY_PUSH) { + // need to check. + errCode = queryObj.SerializeData(parcel, SOFTWARE_VERSION_CURRENT); + } + return errCode; +} + +int SingleVerSerializeManager::DataPacketCalculateLen(const Message *inMsg, uint32_t &len) +{ + const DataRequestPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + + len = packet->CalculateLen(inMsg->GetMessageId()); + return E_OK; +} + +int SingleVerSerializeManager::AckPacketCalculateLen(const Message *inMsg, uint32_t &len) +{ + const DataAckPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + + len = packet->CalculateLen(); + return E_OK; +} + +bool SingleVerSerializeManager::IsPacketValid(const Message *inMsg) +{ + if (inMsg == nullptr) { + return false; + } + + int msgType = inMsg->GetMessageType(); + if (msgType != TYPE_REQUEST && msgType != TYPE_RESPONSE && msgType != TYPE_NOTIFY) { + LOGE("[DataSync][IsPacketValid] Message type ERROR! message type=%d", msgType); + return false; + } + return true; +} + +int SingleVerSerializeManager::AckPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + const DataAckPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + + Parcel parcel(buffer, length); + parcel.WriteUInt32(packet->GetVersion()); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + // now V1 compatible for softWareVersion :{101, 102} + return AckPacketSyncerPartSerializationV1(parcel, packet); +} + +int SingleVerSerializeManager::AckPacketSyncerPartSerializationV1(Parcel &parcel, const DataAckPacket *packet) +{ + parcel.WriteUInt64(packet->GetData()); + parcel.WriteInt(packet->GetRecvCode()); + parcel.WriteVector(packet->GetReserved()); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + parcel.EightByteAlign(); + return E_OK; +} + +int SingleVerSerializeManager::DataPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + std::vector dataItems; + uint32_t version; + Parcel parcel(const_cast(buffer), length); + uint32_t packLen = parcel.ReadUInt32(version); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + + if (version > SOFTWARE_VERSION_CURRENT) { + return -E_VERSION_NOT_SUPPORT; + } + + auto packet = new (std::nothrow) DataRequestPacket(); + if (packet == nullptr) { + return -E_OUT_OF_MEMORY; + } + + packet->SetVersion(version); + packLen += GenericSingleVerKvEntry::DeSerializeDatas(dataItems, parcel); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + + packet->SetData(dataItems); + int errCode = DataPacketSyncerPartDeSerialization(parcel, packet, packLen, length, version); + if (errCode != E_OK) { + goto ERROR; + } + if (inMsg->GetMessageId() == QUERY_SYNC_MESSAGE) { + errCode = DataPacketQuerySyncDeSerialization(parcel, packet); + if (errCode != E_OK) { + goto ERROR; + } + } + if (packet->IsCompressData()) { + errCode = DataPacketCompressDataDeSerialization(parcel, packet); + if (errCode != E_OK) { + goto ERROR; + } + } + + errCode = inMsg->SetExternalObject<>(packet); + if (errCode != E_OK) { + goto ERROR; + } + return errCode; + +ERROR: + delete packet; + packet = nullptr; + return errCode; +} + +int SingleVerSerializeManager::DataPacketQuerySyncDeSerialization(Parcel &parcel, DataRequestPacket *packet) +{ + WaterMark deletedWatermark = 0; + parcel.ReadUInt64(deletedWatermark); + std::string queryId; + parcel.ReadString(queryId); + if (parcel.IsError()) { + return -E_PARSE_FAIL; + } + // query identify + QuerySyncObject querySyncObj; + int errCode = E_OK; + // for version 105, query is always sent. + if ((packet->GetVersion() > SOFTWARE_VERSION_RELEASE_4_0) || packet->GetMode() != QUERY_PUSH) { + // need to check. + errCode = QuerySyncObject::DeSerializeData(parcel, querySyncObj); + } + if (errCode != E_OK) { + LOGI("[SingleVerSerializeManager] DeSerializeData object failed."); + return errCode; + } + packet->SetDeletedWaterMark(deletedWatermark); + packet->SetQueryId(queryId); + if ((packet->GetVersion() > SOFTWARE_VERSION_RELEASE_4_0) || packet->GetMode() != QUERY_PUSH) { + packet->SetQuery(querySyncObj); + } + return E_OK; +} + +int SingleVerSerializeManager::DataPacketCompressDataDeSerialization(Parcel &parcel, DataRequestPacket *packet) +{ + std::vector originalData; + int errCode = GenericSingleVerKvEntry::DeSerializeCompressedDatas(originalData, parcel); + if (errCode != E_OK) { + LOGE("[SingleVerSerializeManager] DeSerializeComptressData failed, errCode=%d", errCode); + return errCode; + } + packet->SetData(originalData); + return E_OK; +} + +int SingleVerSerializeManager::DataPacketSyncerPartDeSerialization(Parcel &parcel, DataRequestPacket *packet, + uint32_t packLen, uint32_t length, uint32_t version) +{ + WaterMark waterMark; + WaterMark localWaterMark; + WaterMark peerWaterMark; + int32_t sendCode; + int32_t mode; + uint32_t sessionId; + uint32_t flag = 0; + std::vector reserved; + + packLen += parcel.ReadUInt64(waterMark); + packLen += parcel.ReadUInt64(localWaterMark); + packLen += parcel.ReadUInt64(peerWaterMark); + packLen += parcel.ReadInt(sendCode); + packLen += parcel.ReadInt(mode); + packLen += parcel.ReadUInt32(sessionId); + packLen += parcel.ReadVector(reserved); + if (version > SOFTWARE_VERSION_RELEASE_2_0) { + packLen += parcel.ReadUInt32(flag); + packet->SetFlag(flag); + } + packLen = Parcel::GetEightByteAlign(packLen); + if (parcel.IsError()) { + LOGE("[DataSync][DataPacketDeSerialization] deserialize failed! input len=%lu,packLen=%lu", length, packLen); + return -E_LENGTH_ERROR; + } + parcel.EightByteAlign(); + packet->SetEndWaterMark(waterMark); + packet->SetLocalWaterMark(localWaterMark); + packet->SetPeerWaterMark(peerWaterMark); + packet->SetSendCode(sendCode); + packet->SetMode(mode); + packet->SetSessionId(sessionId); + packet->SetReserved(reserved); + return E_OK; +} + +int SingleVerSerializeManager::AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + DataAckPacket packet; + Parcel parcel(const_cast(buffer), length); + uint32_t version; + + parcel.ReadUInt32(version); + if (parcel.IsError()) { + return -E_INVALID_ARGS; + } + if (version > SOFTWARE_VERSION_CURRENT) { + packet.SetVersion(version); + packet.SetRecvCode(-E_VERSION_NOT_SUPPORT); + return inMsg->SetCopiedObject<>(packet); + } + packet.SetVersion(version); + // now V1 compatible for softWareVersion :{101, 102} + int errCode = AckPacketSyncerPartDeSerializationV1(parcel, packet); + if (errCode != E_OK) { + return errCode; + } + + return inMsg->SetCopiedObject<>(packet); +} + +int SingleVerSerializeManager::AckPacketSyncerPartDeSerializationV1(Parcel &parcel, DataAckPacket &packet) +{ + WaterMark mark; + int32_t errCode; + std::vector reserved; + + parcel.ReadUInt64(mark); + parcel.ReadInt(errCode); + parcel.ReadVector(reserved); + if (parcel.IsError()) { + LOGE("[AckPacketSyncerPartDeSerializationV1] DeSerialization failed"); + return -E_INVALID_ARGS; + } + packet.SetData(mark); + packet.SetRecvCode(errCode); + packet.SetReserved(reserved); + return E_OK; +} + +int SingleVerSerializeManager::ControlPacketCalculateLen(const Message *inMsg, uint32_t &len) +{ + auto packet = inMsg->GetObject(); + if (packet == nullptr || packet->GetcontrolCmdType() >= INVALID_CONTROL_CMD) { + LOGE("[ControlPacketSerialization] invalid control cmd"); + return -E_INVALID_ARGS; + } + if (packet->GetcontrolCmdType() == SUBSCRIBE_QUERY_CMD || packet->GetcontrolCmdType() == UNSUBSCRIBE_QUERY_CMD) { + return SingleVerSerializeManager::SubscribeCalculateLen(inMsg, len); + } + return E_OK; +} + +int SingleVerSerializeManager::ControlPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + auto packet = inMsg->GetObject(); + if (packet == nullptr || packet->GetcontrolCmdType() >= INVALID_CONTROL_CMD) { + LOGE("[ControlPacketSerialization] invalid control cmd"); + return -E_INVALID_ARGS; + } + if (packet->GetcontrolCmdType() == SUBSCRIBE_QUERY_CMD || packet->GetcontrolCmdType() == UNSUBSCRIBE_QUERY_CMD) { + return SingleVerSerializeManager::SubscribeSerialization(buffer, length, inMsg); + } + return E_OK; +} + +int SingleVerSerializeManager::ControlPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + Parcel parcel(const_cast(buffer), length); + ControlRequestPacket packet; + int errCode = ControlRequestDeSerialization(parcel, packet); + if (errCode != E_OK) { + return errCode; + } + if (packet.GetcontrolCmdType() == SUBSCRIBE_QUERY_CMD || packet.GetcontrolCmdType() == UNSUBSCRIBE_QUERY_CMD) { + errCode = SubscribeDeSerialization(parcel, inMsg, packet); + } + return errCode; +} + +int SingleVerSerializeManager::AckControlPacketCalculateLen(const Message *inMsg, uint32_t &len) +{ + auto packet = inMsg->GetObject(); + if (packet == nullptr) { + LOGE("[AckControlPacketCalculateLen] invalid control cmd"); + return -E_INVALID_ARGS; + } + len = packet->CalculateLen(); + return E_OK; +} + +int SingleVerSerializeManager::AckControlPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + auto packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + Parcel parcel(buffer, length); + parcel.WriteUInt32(packet->GetVersion()); + parcel.WriteInt(packet->GetRecvCode()); + parcel.WriteUInt32(packet->GetcontrolCmdType()); + parcel.WriteUInt32(packet->GetFlag()); + if (parcel.IsError()) { + LOGE("[AckControlPacketSerialization] Serialization failed"); + return -E_INVALID_ARGS; + } + parcel.EightByteAlign(); + return E_OK; +} + +int SingleVerSerializeManager::AckControlPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg) +{ + auto packet = new (std::nothrow) ControlAckPacket(); + if (packet == nullptr) { + return -E_OUT_OF_MEMORY; + } + Parcel parcel(const_cast(buffer), length); + int32_t recvCode = 0; + uint32_t version = 0; + uint32_t controlCmdType = 0; + uint32_t flag = 0; + parcel.ReadUInt32(version); + parcel.ReadInt(recvCode); + parcel.ReadUInt32(controlCmdType); + parcel.ReadUInt32(flag); + int errCode; + if (parcel.IsError()) { + LOGE("[AckControlPacketDeSerialization] DeSerialization failed"); + errCode = -E_INVALID_ARGS; + goto ERROR; + } + packet->SetPacketHead(recvCode, version, controlCmdType, flag); + errCode = inMsg->SetExternalObject<>(packet); + if (errCode != E_OK) { + goto ERROR; + } + return errCode; +ERROR: + delete packet; + packet = nullptr; + return errCode; +} + +int SingleVerSerializeManager::ControlRequestSerialization(Parcel &parcel, const Message *inMsg) +{ + auto packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + parcel.WriteUInt32(packet->GetVersion()); + parcel.WriteInt(packet->GetSendCode()); + parcel.WriteUInt32(packet->GetcontrolCmdType()); + parcel.WriteUInt32(packet->GetFlag()); + if (parcel.IsError()) { + LOGE("[ControlRequestSerialization] Serialization failed"); + return -E_INVALID_ARGS; + } + parcel.EightByteAlign(); + return E_OK; +} + +int SingleVerSerializeManager::ControlRequestDeSerialization(Parcel &parcel, ControlRequestPacket &packet) +{ + uint32_t version = 0; + int32_t sendCode = 0; + uint32_t controlCmdType = 0; + uint32_t flag = 0; + parcel.ReadUInt32(version); + if (version > SOFTWARE_VERSION_CURRENT) { + return -E_VERSION_NOT_SUPPORT; + } + parcel.ReadInt(sendCode); + parcel.ReadUInt32(controlCmdType); + parcel.ReadUInt32(flag); + if (parcel.IsError()) { + LOGE("[ControlRequestDeSerialization] deserialize failed!"); + return -E_LENGTH_ERROR; + } + packet.SetPacketHead(sendCode, version, controlCmdType, flag); + return E_OK; +} + +int SingleVerSerializeManager::SubscribeCalculateLen(const Message *inMsg, uint32_t &len) +{ + auto packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + len = packet->CalculateLen(); + return E_OK; +} + +int SingleVerSerializeManager::SubscribeSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg) +{ + auto packet = inMsg->GetObject(); + if (packet == nullptr) { + return -E_INVALID_ARGS; + } + Parcel parcel(buffer, length); + int errCode = ControlRequestSerialization(parcel, inMsg); + if (errCode != E_OK) { + LOGE("[SubscribeSerialization] ControlRequestPacket Serialization failed, errCode=%d", errCode); + return errCode; + } + QuerySyncObject queryObj = packet->GetQuery(); + errCode = queryObj.SerializeData(parcel, SOFTWARE_VERSION_CURRENT); + if (errCode != E_OK) { + LOGE("[SubscribeSerialization] query object Serialization failed, errCode=%d", errCode); + return errCode; + } + return E_OK; +} + +int SingleVerSerializeManager::SubscribeDeSerialization(Parcel &parcel, Message *inMsg, + ControlRequestPacket &controlPacket) +{ + auto packet = new (std::nothrow) SubscribeRequest(); + if (packet == nullptr) { + return -E_OUT_OF_MEMORY; + } + QuerySyncObject querySyncObj; + int errCode = QuerySyncObject::DeSerializeData(parcel, querySyncObj); + if (errCode != E_OK) { + goto ERROR; + } + packet->SetPacketHead(controlPacket.GetSendCode(), controlPacket.GetVersion(), controlPacket.GetcontrolCmdType(), + controlPacket.GetFlag()); + packet->SetQuery(querySyncObj); + errCode = inMsg->SetExternalObject<>(packet); + if (errCode != E_OK) { + goto ERROR; + } + return errCode; +ERROR: + delete packet; + packet = nullptr; + return errCode; +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.h new file mode 100644 index 0000000000000000000000000000000000000000..162a23da194eba92410b09b2309e607b9119ab49 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_serialize_manager.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SINGLE_VER_SERIALIZE_MANAGER_NEW_H +#define SINGLE_VER_SERIALIZE_MANAGER_NEW_H + +#include "icommunicator.h" +#include "parcel.h" +#include "single_ver_data_packet.h" + +namespace DistributedDB { +class SingleVerSerializeManager { +public: + SingleVerSerializeManager(); + virtual ~SingleVerSerializeManager(); + + DISABLE_COPY_ASSIGN_MOVE(SingleVerSerializeManager); + + static int Serialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + + static int DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + + static uint32_t CalculateLen(const Message *inMsg); + + static int RegisterTransformFunc(); +private: + static bool IsPacketValid(const Message *inMsg); + + static int DataSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + static int ControlSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + + static int DataDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + static int ControlDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + + static uint32_t CalculateDataLen(const Message *inMsg); + static uint32_t CalculateControlLen(const Message *inMsg); + + static int DataPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + static int DataPacketSyncerPartSerialization(Parcel &parcel, const DataRequestPacket *packet); + static int DataPacketQuerySyncSerialization(Parcel &parcel, const DataRequestPacket *packet); + static int DataPacketCalculateLen(const Message *inMsg, uint32_t &len); + + static int DataPacketQuerySyncDeSerialization(Parcel &parcel, DataRequestPacket *packet); + static int DataPacketCompressDataDeSerialization(Parcel &parcel, DataRequestPacket *packet); + static int DataPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + static int DataPacketSyncerPartDeSerialization(Parcel &parcel, DataRequestPacket *packet, uint32_t packLen, + uint32_t length, uint32_t version); + + static int AckPacketCalculateLen(const Message *inMsg, uint32_t &len); + static int AckPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + static int AckPacketSyncerPartSerializationV1(Parcel &parcel, const DataAckPacket *packet); + + static int AckPacketSyncerPartDeSerializationV1(Parcel &parcel, DataAckPacket &packet); + static int AckPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + + static int ControlPacketCalculateLen(const Message *inMsg, uint32_t &len); + static int ControlPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + static int ControlPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + + static int AckControlPacketCalculateLen(const Message *inMsg, uint32_t &len); + static int AckControlPacketSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + static int AckControlPacketDeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); + + static int ControlRequestSerialization(Parcel &parcel, const Message *inMsg); + static int ControlRequestDeSerialization(Parcel &parcel, ControlRequestPacket &packet); + static int SubscribeCalculateLen(const Message *inMsg, uint32_t &len); + static int SubscribeSerialization(uint8_t *buffer, uint32_t length, const Message *inMsg); + static int SubscribeDeSerialization(Parcel &parcel, Message *inMsg, ControlRequestPacket &controlPacket); +}; +} // namespace DistributedDB + +#endif // SINGLE_VER_SERIALIZE_MANAGER_NEW_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ea78cbe1b0f044cb39ed574c1388b11c07a7f03c --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.cpp @@ -0,0 +1,394 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "single_ver_subscribe_manager.h" +#include "db_common.h" +#include "sync_types.h" + +namespace DistributedDB { +void SingleVerSubscribeManager::ClearRemoteSubscribeQuery(const std::string &device) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + ClearSubscribeQuery(device, remoteSubscribedMap_, remoteSubscribedTotalMap_); +} + +void SingleVerSubscribeManager::ClearAllRemoteQuery() +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + remoteSubscribedMap_.clear(); + remoteSubscribedTotalMap_.clear(); +} + +void SingleVerSubscribeManager::ClearLocalSubscribeQuery(const std::string &device) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + unFinishedLocalAutoSubMap_.erase(device); + ClearSubscribeQuery(device, localSubscribeMap_, localSubscribeTotalMap_); +} + +int SingleVerSubscribeManager::ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + int errCode = ReserveSubscribeQuery(device, query, remoteSubscribedMap_, remoteSubscribedTotalMap_); + LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote reserve err=%d", + STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); + return errCode; +} + +int SingleVerSubscribeManager::ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + std::string queryId = query.GetIdentify(); + int errCode = ActiveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); + LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remote active err=%d", + STR_MASK(device), STR_MASK(queryId), errCode); + return errCode; +} + +int SingleVerSubscribeManager::ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + int errCode = ReserveSubscribeQuery(device, query, localSubscribeMap_, localSubscribeTotalMap_); + LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local reserve err=%d", + STR_MASK(device), STR_MASK(query.GetIdentify()), errCode); + return errCode; +} + +int SingleVerSubscribeManager::ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + std::string queryId = query.GetIdentify(); + int errCode = ActiveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); + LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s local active err=%d", + STR_MASK(device), STR_MASK(queryId), errCode); + if (errCode != E_OK) { + return errCode; + } + if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && + unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { + unFinishedLocalAutoSubMap_[device].erase(queryId); + } + return errCode; +} + +void SingleVerSubscribeManager::DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + std::string queryId = query.GetIdentify(); + DeleteSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); +} + +void SingleVerSubscribeManager::DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + std::string queryId = query.GetIdentify(); + DeleteSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); +} + +void SingleVerSubscribeManager::PutLocalUnFiniedSubQueries(const std::string &device, + std::vector &subscribeQueries) +{ + LOGI("[SingleVerSubscribeManager] put local unfinished subscribe queries, nums=%d", subscribeQueries.size()); + std::unique_lock lockGuard(localSubscribeMapLock_); + if (subscribeQueries.size() == 0) { + unFinishedLocalAutoSubMap_.erase(device); + return; + } + unFinishedLocalAutoSubMap_[device].clear(); + auto iter = unFinishedLocalAutoSubMap_.find(device); + for (const auto &query : subscribeQueries) { + iter->second.insert(query.GetIdentify()); + } +} + +void SingleVerSubscribeManager::GetAllUnFinishSubQueries( + std::map> &allSyncQueries) const +{ + std::shared_lock lock(localSubscribeMapLock_); + for (auto &item : unFinishedLocalAutoSubMap_) { + if (item.second.size() == 0) { + continue; + } + allSyncQueries[item.first] = {}; + auto iter = allSyncQueries.find(item.first); + for (const auto &queryId : item.second) { + auto iterTmp = localSubscribeTotalMap_.find(queryId); + if (iterTmp == localSubscribeTotalMap_.end()) { + LOGI("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryId)); + continue; + } + iter->second.push_back(iterTmp->second.first); + } + } +} + +void SingleVerSubscribeManager::RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + std::string queryId = query.GetIdentify(); + RemoveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); +} + +void SingleVerSubscribeManager::RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + std::string queryId = query.GetIdentify(); + RemoveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); + if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && + unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { + unFinishedLocalAutoSubMap_[device].erase(queryId); + LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s delete from UnFinishedMap", + STR_MASK(device), STR_MASK(queryId)); + if (unFinishedLocalAutoSubMap_[device].size() == 0) { + LOGI("[SingleVerSubscribeManager] dev=%s delete from unFinish map", STR_MASK(device)); + unFinishedLocalAutoSubMap_.erase(device); + } + } +} + +void SingleVerSubscribeManager::GetLocalSubscribeQueries(const std::string &device, + std::vector &subscribeQueries) const +{ + std::shared_lock lock(localSubscribeMapLock_); + GetSubscribeQueries(device, localSubscribeMap_, localSubscribeTotalMap_, subscribeQueries); +} + +void SingleVerSubscribeManager::GetRemoteSubscribeQueries(const std::string &device, + std::vector &subscribeQueries) const +{ + std::shared_lock lockGuard(remoteSubscribedMapLock_); + GetSubscribeQueries(device, remoteSubscribedMap_, remoteSubscribedTotalMap_, subscribeQueries); +} + +bool SingleVerSubscribeManager::IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const +{ + std::shared_lock lockGuard(remoteSubscribedMapLock_); + auto iter = remoteSubscribedMap_.find(device); + if (iter == remoteSubscribedMap_.end()) { + LOGD("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); + return false; + } + std::string queryId = query.GetIdentify(); + auto subIter = iter->second.find(queryId); + if (subIter == iter->second.end()) { + LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryId)); + return false; + } + return true; +} + +void SingleVerSubscribeManager::GetRemoteSubscribeQueryIds(const std::string &device, + std::vector &subscribeQueryIds) const +{ + std::shared_lock lockGuard(remoteSubscribedMapLock_); + auto iter = remoteSubscribedMap_.find(device); + if (iter == remoteSubscribedMap_.end()) { + LOGI("[SingleVerSubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); + return; + } + for (const auto &queryInfo : iter->second) { + if (remoteSubscribedTotalMap_.find(queryInfo.first) == remoteSubscribedTotalMap_.end()) { + LOGE("[SingleVerSubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryInfo.first)); + continue; + } + subscribeQueryIds.push_back(queryInfo.first); + } +} + +int SingleVerSubscribeManager::LocalSubscribeLimitCheck(const std::vector &devices, + QuerySyncObject &query) const +{ + std::shared_lock lock(localSubscribeMapLock_); + int devNum = localSubscribeMap_.size(); + for (const auto &device : devices) { + if (localSubscribeMap_.find(device) != localSubscribeMap_.end()) { + continue; + } + devNum++; + if (devNum > MAX_DEVICES_NUM) { + LOGE("[SingleVerSubscribeManager] local subscribe devices is over limit"); + return -E_MAX_LIMITS; + } + } + std::string queryId = query.GetIdentify(); + auto allIter = localSubscribeTotalMap_.find(queryId); + if (allIter == localSubscribeTotalMap_.end() && localSubscribeTotalMap_.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { + LOGE("[SingleVerSubscribeManager] all local subscribe sums is over limit"); + return -E_MAX_LIMITS; + } + return E_OK; +} + +void SingleVerSubscribeManager::ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap) +{ + if (subscribeMap.find(device) == subscribeMap.end()) { + LOGI("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); + return; + } + for (const auto &queryInfo : subscribeMap[device]) { + if (subscribedTotalMap.find(queryInfo.first) != subscribedTotalMap.end()) { + if (subscribedTotalMap[queryInfo.first].second > 0) { + subscribedTotalMap[queryInfo.first].second--; + } + if (subscribedTotalMap[queryInfo.first].second == 0) { + LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryInfo.first)); + subscribedTotalMap.erase(queryInfo.first); + } + } + } + subscribeMap.erase(device); + LOGI("[SingleVerSubscribeManager] clear dev=%s` remote subscribe queies finished", STR_MASK(device)); +} + +int SingleVerSubscribeManager::ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, + SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) +{ + std::string queryId = query.GetIdentify(); + auto iter = subscribeMap.find(device); + auto allIter = subscribedTotalMap.find(queryId); + // limit check + if (allIter == subscribedTotalMap.end() && subscribedTotalMap.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { + LOGE("[SingleVerSubscribeManager] all subscribe sums is over limit"); + return -E_MAX_LIMITS; + } + if (iter == subscribeMap.end() && subscribeMap.size() >= MAX_DEVICES_NUM) { + LOGE("[SingleVerSubscribeManager] subscribe devices is over limit"); + return -E_MAX_LIMITS; + } + if (iter != subscribeMap.end() && iter->second.find(queryId) == iter->second.end() && + iter->second.size() >= MAX_SUBSCRIBE_NUM_PER_DEV) { + LOGE("[SingleVerSubscribeManager] subscribe sums is over limit"); + return -E_MAX_LIMITS; + } + if (iter != subscribeMap.end() && iter->second.find(queryId) != iter->second.end() && + iter->second[queryId] == SubscribeStatus::ACTIVE) { + LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s already active in map", + STR_MASK(device), STR_MASK(queryId)); + return E_OK; + } + + if (iter == subscribeMap.end()) { + subscribeMap[device] = std::map {}; + } + bool isNeedInc = false; + if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { + subscribeMap[device][queryId] = SubscribeStatus::NOT_ACTIVE; + isNeedInc = true; + } + if (allIter == subscribedTotalMap.end()) { + subscribedTotalMap[queryId] = {query, 1}; + } else if (isNeedInc) { + subscribedTotalMap[queryId].second++; + } + return E_OK; +} + +int SingleVerSubscribeManager::ActiveSubscribeQuery(const std::string &device, const std::string &queryId, + SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) +{ + if (subscribedTotalMap.find(queryId) == subscribedTotalMap.end()) { + LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); + return -E_INTERNAL_ERROR; + } + if (subscribeMap.find(device) == subscribeMap.end()) { + LOGE("[SingleVerSubscribeManager] can not find dev=%s in localSubscribeMap", STR_MASK(device)); + return -E_INTERNAL_ERROR; + } + if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { + LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); + return -E_INTERNAL_ERROR; + } + subscribeMap[device][queryId] = SubscribeStatus::ACTIVE; + return E_OK; +} + +void SingleVerSubscribeManager::DeleteSubscribeQuery(const std::string &device, const std::string &queryId, + SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) +{ + if (subscribeMap.find(device) == subscribeMap.end()) { + LOGE("[SingleVerSubscribeManager] can not find dev=%s in map", STR_MASK(device)); + return; + } + if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { + LOGE("[SingleVerSubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); + return; + } + SubscribeStatus queryStatus = subscribeMap[device][queryId]; + // not permit to delete the query when something wrong this time,because it is subscribed successfully last time + if (queryStatus == SubscribeStatus::ACTIVE) { + LOGE("[SingleVerSubscribeManager] dev=%s,queryId=%s is active, no need to del", + STR_MASK(device), STR_MASK(queryId)); + return; + } + subscribeMap[device].erase(queryId); + auto iter = subscribedTotalMap.find(queryId); + if (iter == subscribedTotalMap.end()) { + LOGE("[SingleVerSubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); + return; + } + iter->second.second--; + if (iter->second.second <= 0) { + LOGI("[SingleVerSubscribeManager] del queryId=%s from SubscribeTotalMap", STR_MASK(queryId)); + subscribedTotalMap.erase(queryId); + } + LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribeMap success", + STR_MASK(device), STR_MASK(queryId)); +} + +void SingleVerSubscribeManager::RemoveSubscribeQuery(const std::string &device, const std::string &queryId, + SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) +{ + auto iter = subscribeMap.find(device); + if (iter == subscribeMap.end()) { + LOGE("[SingleVerSubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); + return; + } + if (iter->second.find(queryId) == subscribeMap[device].end()) { + LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s not in SubscribedMap", STR_MASK(device), STR_MASK(queryId)); + return; + } + iter->second.erase(queryId); + auto allIter = subscribedTotalMap.find(queryId); + if (allIter == subscribedTotalMap.end()) { + LOGI("[SingleVerSubscribeManager] queryId=%s not in TotalMap", STR_MASK(queryId)); + return; + } + allIter->second.second--; + if (allIter->second.second <= 0) { + subscribedTotalMap.erase(queryId); + LOGI("[SingleVerSubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryId)); + } + LOGI("[SingleVerSubscribeManager] dev=%s,queryId=%s remove from SubscribedMap success", + STR_MASK(device), STR_MASK(queryId)); +} + +void SingleVerSubscribeManager::GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, + const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const +{ + auto iter = subscribeMap.find(device); + if (iter == subscribeMap.end()) { + LOGD("[SingleVerSubscribeManager] dev=%s not in localSubscribeMap", STR_MASK(device)); + return; + } + for (const auto &queryInfo : iter->second) { + auto iterTmp = subscribedTotalMap.find(queryInfo.first); + if (iterTmp == subscribedTotalMap.end()) { + LOGE("[SingleVerSubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryInfo.first)); + continue; + } + subscribeQueries.push_back(iterTmp->second.first); + } +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h new file mode 100644 index 0000000000000000000000000000000000000000..c152e0f29e0f966436f3e8c2d048add4aa3e6f2c --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_subscribe_manager.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SINGLE_VER_SUBSCRIBE_MANAGER_H +#define SINGLE_VER_SUBSCRIBE_MANAGER_H + +#include +#include +#include "query_sync_object.h" + +namespace DistributedDB { +enum class SubscribeStatus { + NOT_ACTIVE = 0, + ACTIVE = 1, +}; + +using SubscribeMap = std::map>; +using SubscribedTotalMap = std::map>; + +class SingleVerSubscribeManager { +public: + SingleVerSubscribeManager() = default; + ~SingleVerSubscribeManager() {}; + + DISABLE_COPY_ASSIGN_MOVE(SingleVerSubscribeManager); + + // clear remoteSubscribeMap_[device] list when remote db is closed or dev offline. + void ClearRemoteSubscribeQuery(const std::string &device); + + // clear localSubscribeMap_[device] list when device is offline. + void ClearLocalSubscribeQuery(const std::string &device); + + void ClearAllRemoteQuery(); + + // add query when receive subscribe command + int ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // active query to ACTIVE when send ack ok + int ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // reserve query when user call SubscribeRemoteQuery, status set to NOT_ACTIVE + int ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // active query to ACTIVE when receive ack ok + int ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // delete local subscribe query when recv wrong errCode, only not_active status allowed to del + void DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // delete remote subscribe query when send msg failed, only not_active status allowed to del + void DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // put subscribe queries into unfinished map when remote db online + void PutLocalUnFiniedSubQueries(const std::string &device, std::vector &subscribeQueries); + + // get all device unFinished subscribe queries which triggered by auto subscribe and need retry subscribe + void GetAllUnFinishSubQueries(std::map> &allSyncQueries) const; + + // remove query when receive unsubscribe command + void RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // remove query when user call UnSubscribeRemoteQuery + void RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // get device active subscribeQueries from localSubscribeMap_ + void GetLocalSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; + + // get device remote queryId from remoteSubscribedMap_ while data change + void GetRemoteSubscribeQueryIds(const std::string &device, std::vector &subscribeQueryIds) const; + // get device remote subscribeQueries from remoteSubscribedMap_ while data change + void GetRemoteSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; + + bool IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const; + + int LocalSubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const; +private: + void ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + int ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + int ActiveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + void DeleteSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + void RemoveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + void GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, + const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const; + + mutable std::shared_mutex localSubscribeMapLock_; + // subscribe sponsor, key: device, value: pair map + // status 0: active, 1: not active + SubscribeMap localSubscribeMap_; + + // used retry subscribe in db open scene, key: device value: set + std::map> unFinishedLocalAutoSubMap_; + + // subscribe sponsor total query info, key:queryId, value: + // while use_num is 0, delete item from the map + SubscribedTotalMap localSubscribeTotalMap_; + + mutable std::shared_mutex remoteSubscribedMapLock_; + // subscribed, key: device, value: pair map + // status 0: active, 1: not active + SubscribeMap remoteSubscribedMap_; + + // subscribed total query info, key:queryId, value: + // while use_num is 0, delete item from the map + SubscribedTotalMap remoteSubscribedTotalMap_; +}; +} // namespace DistributedDB + +#endif // SINGLE_VER_SUBSCRIBE_MANAGER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp index 1c332d3885aa017d7fa7d89f375400d61a1baec8..bec6bd4bf73ef830ee50eec1e638024c099b3b91 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.cpp @@ -14,18 +14,21 @@ */ #include "single_ver_sync_engine.h" +#include "db_common.h" #include "single_ver_sync_task_context.h" #include "log_print.h" namespace DistributedDB { ISyncTaskContext *SingleVerSyncEngine::CreateSyncTaskContext() { - auto context = new (std::nothrow) SingleVerSyncTaskContext; + auto context = new (std::nothrow) SingleVerSyncTaskContext(); if (context == nullptr) { LOGE("[SingleVerSyncEngine][CreateSyncTaskContext] create failed, may be out of memory"); return nullptr; } + context->SetSyncRetry(GetSyncRetry()); context->EnableClearRemoteStaleData(needClearRemoteStaleData_); + context->SetSubscribeManager(subManager_); return context; } @@ -42,5 +45,66 @@ void SingleVerSyncEngine::EnableClearRemoteStaleData(bool enable) } } +int SingleVerSyncEngine::StartAutoSubscribeTimer() +{ + std::lock_guard lockGuard(timerLock_); + if (subscribeTimerId_ > 0) { + LOGI("[SingleSyncEngine] subscribeTimerId is already set"); + return -E_INTERNAL_ERROR; + } + TimerId timerId = 0; + TimerAction timeOutCallback = std::bind(&SingleVerSyncEngine::SubscribeTimeOut, this, std::placeholders::_1); + int errCode = RuntimeContext::GetInstance()->SetTimer(SUBSCRIBE_TRIGGER_TIME_OUT, timeOutCallback, nullptr, + timerId); + if (errCode != E_OK) { + return errCode; + } + subscribeTimerId_ = timerId; + LOGI("[SingleSyncEngine] start auto subscribe timerId=%d finished", timerId); + return errCode; +} + +void SingleVerSyncEngine::StopAutoSubscribeTimer() +{ + std::lock_guard lockGuard(timerLock_); + if (subscribeTimerId_ == 0) { + return; + } + LOGI("[SingleSyncEngine] stop auto subscribe timerId=%d finished", subscribeTimerId_); + RuntimeContext::GetInstance()->RemoveTimer(subscribeTimerId_); + subscribeTimerId_ = 0; +} + +int SingleVerSyncEngine::SubscribeTimeOut(TimerId id) +{ + if (!queryAutoSyncCallback_) { + return E_OK; + } + std::lock_guard lockGuard(timerLock_); + std::map> allSyncQueries; + GetAllUnFinishSubQueries(allSyncQueries); + LOGI("[SingleVerSyncEngine] SubscribeTimeOut,size=%d", allSyncQueries.size()); + if (allSyncQueries.size() == 0) { + LOGI("no need to trigger auto subscribe"); + return E_OK; + } + for (auto &item : allSyncQueries) { + for (auto &query : item.second) { + InternalSyncParma param; + GetSubscribeSyncParam(item.first, query, param); + queryAutoSyncCallback_(param); + } + } + return E_OK; +} + +void SingleVerSyncEngine::SetIsNeedResetAbilitySync(const std::string &deviceId, bool isNeedReset) +{ + ISyncTaskContext *context = GetSyncTaskContextAndInc(deviceId); + if (context != nullptr) { + context->SetIsNeedResetAbilitySync(isNeedReset); + RefObject::DecObjRef(context); + } +} DEFINE_OBJECT_TAG_FACILITIES(SingleVerSyncEngine); } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.h index c4d4ea04f45b89163d44d97726bb4ac40187b9d7..2fc8498a5847d5b8048d49c445ec3713ed857758 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_engine.h @@ -23,9 +23,19 @@ class SingleVerSyncEngine final : public SyncEngine { public: SingleVerSyncEngine() : needClearRemoteStaleData_(false) {}; - // If set true, remote stale data will be clear when remote db rebuiled. + // If set true, remote stale data will be clear when remote db rebuilt. void EnableClearRemoteStaleData(bool enable); + // used by SingleVerKVSyncer when db online + int StartAutoSubscribeTimer() override; + + // used by SingleVerKVSyncer when remote/local db closed + void StopAutoSubscribeTimer() override; + + int SubscribeTimeOut(TimerId id); + + void SetIsNeedResetAbilitySync(const std::string &deviceId, bool isNeedReset); + DISABLE_COPY_ASSIGN_MOVE(SingleVerSyncEngine); protected: ~SingleVerSyncEngine() override {}; @@ -35,8 +45,17 @@ protected: private: DECLARE_OBJECT_TAG(SingleVerSyncEngine); +#ifndef RUNNING_ON_TESTCASE + static constexpr int SUBSCRIBE_TRIGGER_TIME_OUT = 30 * 60 * 1000; // 30min +#else + static constexpr int SUBSCRIBE_TRIGGER_TIME_OUT = 5 * 60 * 1000; // 5min for test +#endif bool needClearRemoteStaleData_; + + // for subscribe timeout callback + std::mutex timerLock_; + TimerId subscribeTimerId_ = 0; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp index adc30e2d24605e763ff68aa7da42abf0d0b47e34..0dc5fe0007fddcf822c3fdc0157e10054a83036e 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.cpp @@ -24,9 +24,11 @@ #include "sync_operation.h" #include "message_transform.h" #include "sync_types.h" +#include "db_common.h" #include "runtime_context.h" #include "performance_analysis.h" #include "single_ver_sync_target.h" +#include "single_ver_data_sync.h" namespace DistributedDB { namespace { @@ -45,13 +47,14 @@ namespace { {TIME_SYNC, INNER_ERR_EVENT, INNER_ERR}, // In ABILITY_SYNC state, compare version num and schema - {ABILITY_SYNC, VERSION_NOT_SUPPORT_EVENT, INNER_ERR}, + {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, {ABILITY_SYNC, SWITCH_TO_PROCTOL_V1_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_DATA_SYNC}, {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, + {ABILITY_SYNC, CONTROL_CMD_EVENT, SYNC_CONTROL_CMD}, - // In START_INITIACTIVE_DATA_SYNC state, send a sync request, and send first packet of data sync + // In START_INITIACTIVE_DATA_SYNC state, send a sync request, and send first packt of data sync {START_INITIACTIVE_DATA_SYNC, SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, {START_INITIACTIVE_DATA_SYNC, RESPONSE_PUSH_REMAINDER_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_INITIACTIVE_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, @@ -68,7 +71,7 @@ namespace { {INACTIVE_PUSH_REMAINDER_DATA, RE_SEND_DATA_EVENT, INACTIVE_PUSH_REMAINDER_DATA}, {INACTIVE_PUSH_REMAINDER_DATA, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, - // In START_PASSIVE_DATA_SYNC state, do response pull request, and send first packet of data sync + // In START_PASSIVE_DATA_SYNC state, do response pull request, and send first packt of data sync {START_PASSIVE_DATA_SYNC, SEND_DATA_EVENT, PASSIVE_PUSH_REMAINDER_DATA}, {START_PASSIVE_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_DATA_SYNC}, {START_PASSIVE_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, @@ -92,6 +95,11 @@ namespace { {WAIT_FOR_RECEIVE_DATA_FINISH, INNER_ERR_EVENT, INNER_ERR}, {WAIT_FOR_RECEIVE_DATA_FINISH, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + {SYNC_CONTROL_CMD, SEND_FINISHED_EVENT, SYNC_TASK_FINISHED}, + {SYNC_CONTROL_CMD, TIME_OUT_EVENT, SYNC_TIME_OUT}, + {SYNC_CONTROL_CMD, INNER_ERR_EVENT, INNER_ERR}, + {SYNC_CONTROL_CMD, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + // In SYNC_TASK_FINISHED, {SYNC_TASK_FINISHED, ALL_TASK_FINISHED_EVENT, IDLE}, {SYNC_TASK_FINISHED, START_SYNC_EVENT, TIME_SYNC}, @@ -111,19 +119,20 @@ namespace { {TIME_SYNC, INNER_ERR_EVENT, INNER_ERR}, // In ABILITY_SYNC state, compare version num and schema - {ABILITY_SYNC, VERSION_NOT_SUPPORT_EVENT, INNER_ERR}, + {ABILITY_SYNC, VERSION_NOT_SUPPOR_EVENT, INNER_ERR}, {ABILITY_SYNC, ABILITY_SYNC_FINISHED_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, {ABILITY_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {ABILITY_SYNC, INNER_ERR_EVENT, INNER_ERR}, + {ABILITY_SYNC, CONTROL_CMD_EVENT, SYNC_CONTROL_CMD}, - // In START_INITIACTIVE_SLIDING_DATA_SYNC state, send a sync request, and send first packet of data sync + // In START_INITIACTIVE_SLIDING_DATA_SYNC state, send a sync request, and send first packt of data sync {START_INITIACTIVE_SLIDING_DATA_SYNC, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, {START_INITIACTIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, {START_INITIACTIVE_SLIDING_DATA_SYNC, INNER_ERR_EVENT, INNER_ERR}, {START_INITIACTIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, {START_INITIACTIVE_SLIDING_DATA_SYNC, RE_SEND_DATA_EVENT, START_INITIACTIVE_SLIDING_DATA_SYNC}, - // In START_PASSIVE_SLIDING_DATA_SYNC state, do response pull request, and send first packet of data sync + // In START_PASSIVE_SLIDING_DATA_SYNC state, do response pull request, and send first packt of data sync {START_PASSIVE_SLIDING_DATA_SYNC, SEND_FINISHED_EVENT, START_PASSIVE_SLIDING_DATA_SYNC}, {START_PASSIVE_SLIDING_DATA_SYNC, RESPONSE_TASK_FINISHED_EVENT, WAIT_FOR_RECEIVE_DATA_FINISH}, {START_PASSIVE_SLIDING_DATA_SYNC, TIME_OUT_EVENT, SYNC_TIME_OUT}, @@ -138,6 +147,11 @@ namespace { {WAIT_FOR_RECEIVE_DATA_FINISH, INNER_ERR_EVENT, INNER_ERR}, {WAIT_FOR_RECEIVE_DATA_FINISH, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + {SYNC_CONTROL_CMD, SEND_FINISHED_EVENT, SYNC_TASK_FINISHED}, + {SYNC_CONTROL_CMD, TIME_OUT_EVENT, SYNC_TIME_OUT}, + {SYNC_CONTROL_CMD, INNER_ERR_EVENT, INNER_ERR}, + {SYNC_CONTROL_CMD, NEED_ABILITY_SYNC_EVENT, ABILITY_SYNC}, + // In SYNC_TASK_FINISHED, {SYNC_TASK_FINISHED, ALL_TASK_FINISHED_EVENT, IDLE}, {SYNC_TASK_FINISHED, START_SYNC_EVENT, TIME_SYNC}, @@ -168,7 +182,7 @@ SingleVerSyncStateMachine::~SingleVerSyncStateMachine() Clear(); } -int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, IKvDBSyncInterface *syncInterface, +int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, ISyncInterface *syncInterface, std::shared_ptr &metaData, ICommunicator *communicator) { if ((context == nullptr) || (syncInterface == nullptr) || (metaData == nullptr) || (communicator == nullptr)) { @@ -193,7 +207,7 @@ int SingleVerSyncStateMachine::Initialize(ISyncTaskContext *context, IKvDBSyncIn if (errCode != E_OK) { goto ERROR_OUT; } - errCode = abilitySync_->Initialize(communicator, syncInterface, context->GetDeviceId()); + errCode = abilitySync_->Initialize(communicator, syncInterface, metaData, context->GetDeviceId()); if (errCode != E_OK) { goto ERROR_OUT; } @@ -237,7 +251,6 @@ int SingleVerSyncStateMachine::ReceiveMessageCallback(Message *inMsg) if (errCode != E_OK) { return errCode; } - switch (inMsg->GetMessageId()) { case TIME_SYNC_MESSAGE: errCode = TimeMarkSyncRecv(inMsg); @@ -246,8 +259,12 @@ int SingleVerSyncStateMachine::ReceiveMessageCallback(Message *inMsg) errCode = AbilitySyncRecv(inMsg); break; case DATA_SYNC_MESSAGE: + case QUERY_SYNC_MESSAGE: errCode = DataPktRecv(inMsg); break; + case CONTROL_SYNC_MESSAGE: + errCode = ControlPktRecv(inMsg); + break; default: errCode = -E_NOT_SUPPORT; } @@ -278,8 +295,8 @@ void SingleVerSyncStateMachine::SyncStepInner() if (iter != stateMapping_.end()) { event = static_cast(iter->second()); } else { - LOGE("[StateMachine][SyncStepInner] can not find state=%d,label=%s,dev=%s{private}", currentState_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGE("[StateMachine][SyncStepInner] can not find state=%d,label=%s,dev=%s", currentState_, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); break; } } while (event != Event::WAIT_ACK_EVENT && SwitchMachineState(event) == E_OK && currentState_ != IDLE); @@ -305,10 +322,10 @@ int SingleVerSyncStateMachine::StartSyncInner() void SingleVerSyncStateMachine::AbortInner() { - LOGE("[StateMachine][AbortInner] error occurred,abort,label=%s,dev=%s{private}", dataSync_->GetLabel().c_str(), - context_->GetDeviceId().c_str()); - if (context_->GetMode() == SyncOperation::PUSH_AND_PULL || context_->GetMode() == SyncOperation::PULL || - context_->IsKilled()) { + LOGE("[StateMachine][AbortInner] error occurred,abort,label=%s,dev=%s", dataSync_->GetLabel().c_str(), + STR_MASK(context_->GetDeviceId())); + int mode = SyncOperation::TransferSyncMode(context_->GetMode()); + if (mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL || context_->IsKilled()) { dataSyncWithSlidingWindow_->ReceiverClear(); } dataSyncWithSlidingWindow_->SenderClear(); @@ -335,17 +352,17 @@ int SingleVerSyncStateMachine::PrepareNextSyncTask() } if (currentState_ != State::IDLE && currentState_ != State::SYNC_TASK_FINISHED) { - LOGW("[StateMachine][PrepareNextSyncTask] PreSync may get an err, state=%d,dev=%s{private}", - currentState_, context_->GetDeviceId().c_str()); + LOGW("[StateMachine][PrepareNextSyncTask] PreSync may get an err, state=%d,dev=%s", + currentState_, STR_MASK(context_->GetDeviceId())); currentState_ = State::IDLE; } return E_OK; } -void SingleVerSyncStateMachine::SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) +void SingleVerSyncStateMachine::SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) { dataSync_->SendSaveDataNotifyPacket(context_, - std::min(context_->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT), sessionId, sequenceId); + std::min(context_->GetRemoteSoftwareVersion(), SOFTWARE_VERSION_CURRENT), sessionId, sequenceId, inMsgId); } void SingleVerSyncStateMachine::CommErrAbort() @@ -417,24 +434,26 @@ void SingleVerSyncStateMachine::InitStateMapping() std::bind(&SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow, this); stateMapping_[START_PASSIVE_SLIDING_DATA_SYNC] = std::bind(&SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow, this); + stateMapping_[SYNC_CONTROL_CMD] = std::bind(&SingleVerSyncStateMachine::DoInitiactiveControlSync, this); } Event SingleVerSyncStateMachine::DoInitiactiveDataSync() { int errCode = E_OK; - switch (context_->GetMode()) { - case SyncOperation::PUSH: - context_->SetOperationStatus(SyncOperation::RECV_FINISHED); + int mode = SyncOperation::TransferSyncMode(context_->GetMode()); + switch (mode) { + case SyncModeType::PUSH: + context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); errCode = dataSync_->PushStart(context_); break; - case SyncOperation::PULL: - context_->SetOperationStatus(SyncOperation::SEND_FINISHED); + case SyncModeType::PULL: + context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); errCode = dataSync_->PullRequestStart(context_); break; - case SyncOperation::PUSH_AND_PULL: + case SyncModeType::PUSH_AND_PULL: errCode = dataSync_->PushPullStart(context_); break; - case SyncOperation::RESPONSE_PULL: + case SyncModeType::RESPONSE_PULL: // In response pull mode, reminader data should send in // PASSIVE_PUSH_REMAINDER_DATA return Event::RESPONSE_PUSH_REMAINDER_EVENT; @@ -450,23 +469,26 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSync() Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() { - LOGD("[StateMachine][activeDataSync] mode=%d,label=%s,dev=%s{private}", context_->GetMode(), - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGD("[StateMachine][activeDataSync] mode=%d,label=%s,dev=%s", context_->GetMode(), + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); int errCode = E_OK; switch (context_->GetMode()) { - case SyncOperation::PUSH: - context_->SetOperationStatus(SyncOperation::RECV_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::PUSH, context_, dataSync_); + case SyncModeType::PUSH: + case SyncModeType::QUERY_PUSH: + context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); + errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); break; - case SyncOperation::PULL: - context_->SetOperationStatus(SyncOperation::SEND_FINISHED); - errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::PULL, context_, dataSync_); + case SyncModeType::PULL: + case SyncModeType::QUERY_PULL: + context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); + errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); break; - case SyncOperation::PUSH_AND_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::PUSH_AND_PULL, context_, dataSync_); + case SyncModeType::PUSH_AND_PULL: + case SyncModeType::QUERY_PUSH_PULL: + errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); break; - case SyncOperation::RESPONSE_PULL: - errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::RESPONSE_PULL, context_, dataSync_); + case SyncModeType::RESPONSE_PULL: + errCode = dataSyncWithSlidingWindow_->SenderStart(context_->GetMode(), context_, dataSync_); break; default: errCode = -E_NOT_SUPPORT; @@ -476,10 +498,15 @@ Event SingleVerSyncStateMachine::DoInitiactiveDataSyncWithSlidingWindow() return Event::WAIT_ACK_EVENT; } // once E_EKEYREVOKED error occurred, PUSH_AND_PULL mode should wait for ack to pull remote data. - if (context_->GetMode() == SyncOperation::PUSH_AND_PULL && errCode == -E_EKEYREVOKED) { + if (SyncOperation::TransferSyncMode(context_->GetMode()) == SyncModeType::PUSH_AND_PULL && + errCode == -E_EKEYREVOKED) { return Event::WAIT_ACK_EVENT; } - return TransformErrCodeToEvent(errCode); + + // when response task step dataSync again while request task is running, ignore the errCode + bool ignoreInnerErr = (context_->GetResponseSessionId() != 0 && context_->GetRequestSessionId() != 0); + Event event = TransformErrCodeToEvent(errCode); + return (ignoreInnerErr && event == INNER_ERR_EVENT) ? SEND_FINISHED_EVENT : event; } Event SingleVerSyncStateMachine::DoPassiveDataSync() @@ -509,23 +536,50 @@ Event SingleVerSyncStateMachine::DoPassiveDataSyncWithSlidingWindow() return RESPONSE_TASK_FINISHED_EVENT; } } - int errCode = dataSyncWithSlidingWindow_->SenderStart(SyncOperation::RESPONSE_PULL, context_, dataSync_); + int errCode = dataSyncWithSlidingWindow_->SenderStart(SyncModeType::RESPONSE_PULL, context_, dataSync_); + if (errCode != E_OK) { + LOGW("[SingleVerSyncStateMachine][DoPassiveDataSyncWithSlidingWindow] response pull send failed[%d]", errCode); + return RESPONSE_TASK_FINISHED_EVENT; + } + return Event::WAIT_ACK_EVENT; +} + +Event SingleVerSyncStateMachine::DoInitiactiveControlSync() +{ + LOGD("[StateMachine][activeControlSync] mode=%d,label=%s,dev=%s", context_->GetMode(), + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); + context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); + int errCode = dataSync_->ControlCmdStart(context_); if (errCode == E_OK) { return Event::WAIT_ACK_EVENT; } + context_->SetTaskErrCode(errCode); return TransformErrCodeToEvent(errCode); } +int SingleVerSyncStateMachine::HandleControlAckRecv(const Message *inMsg) +{ + std::lock_guard lock(stateMachineLock_); + if (IsNeedResetWatchdog(inMsg)) { + (void)ResetWatchDog(); + } + int errCode = dataSync_->ControlCmdAckRecv(context_, inMsg); + ControlAckRecvErrCodeHandle(errCode); + SwitchStateAndStep(TransformErrCodeToEvent(errCode)); + return E_OK; +} + Event SingleVerSyncStateMachine::DoInitiactivePushRemainderData() { int errCode; - switch (context_->GetMode()) { - case SyncOperation::PULL: - case SyncOperation::RESPONSE_PULL: + int mode = SyncOperation::TransferSyncMode(context_->GetMode()); + switch (mode) { + case SyncModeType::PULL: + case SyncModeType::RESPONSE_PULL: // In pull or response pul mode, don't need to do INACTIVE_PUSH return Event::SEND_FINISHED_EVENT; - case SyncOperation::PUSH: - case SyncOperation::PUSH_AND_PULL: + case SyncModeType::PUSH: + case SyncModeType::PUSH_AND_PULL: errCode = dataSync_->PushStart(context_); break; default: @@ -553,11 +607,11 @@ Event SingleVerSyncStateMachine::DoWaitForDataRecv() const if (context_->GetRspTargetQueueSize() != 0) { return START_PULL_RESPONSE_EVENT; } - if (context_->GetOperationStatus() == SyncOperation::FINISHED_ALL) { + if (context_->GetOperationStatus() == SyncOperation::OP_FINISHED_ALL) { return RECV_FINISHED_EVENT; } - if (context_->GetMode() == SyncOperation::PUSH_AND_PULL && - context_->GetOperationStatus() == SyncOperation::EKEYREVOKED_FAILURE && + if (SyncOperation::TransferSyncMode(context_->GetMode()) == SyncModeType::PUSH_AND_PULL && + context_->GetOperationStatus() == SyncOperation::OP_EKEYREVOKED_FAILURE && context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { return RECV_FINISHED_EVENT; } @@ -573,7 +627,7 @@ Event SingleVerSyncStateMachine::DoTimeSync() handler = std::bind(&SyncTaskContext::CommErrHandlerFunc, std::placeholders::_1, context_, context_->GetRequestSessionId()); } - int errCode = timeSync_->SyncStart(handler); + int errCode = timeSync_->SyncStart(handler, context_->GetRequestSessionId()); if (errCode == E_OK) { return Event::WAIT_ACK_EVENT; } @@ -594,18 +648,20 @@ Event SingleVerSyncStateMachine::DoAbilitySync() // Fistr version, not support AbilitySync if (remoteCommunicatorVersion == 0) { context_->SetRemoteSoftwareVersion(SOFTWARE_VERSION_EARLIEST); - LOGI("remote version is 0, switch to v1 proctol"); - return Event::SWITCH_TO_PROCTOL_V1_EVENT; + return GetEventAfterTimeSync(context_->GetMode(), true); + } + if (context_->GetIsNeedResetAbilitySync()) { + abilitySync_->SetAbilitySyncFinishedStatus(false); + context_->SetIsNeedResetAbilitySync(false); } - if (abilitySync_->GetAbilitySyncFinishedStatus()) { - return Event::ABILITY_SYNC_FINISHED_EVENT; + return GetEventAfterTimeSync(context_->GetMode(), false); } CommErrHandler handler = std::bind(&SyncTaskContext::CommErrHandlerFunc, std::placeholders::_1, context_, context_->GetRequestSessionId()); - LOGI("[StateMachine][AbilitySync] start abilitySync,label=%s,dev=%s{private}", dataSync_->GetLabel().c_str(), - context_->GetDeviceId().c_str()); + LOGI("[StateMachine][AbilitySync] start abilitySync,label=%s,dev=%s", dataSync_->GetLabel().c_str(), + STR_MASK(context_->GetDeviceId())); errCode = abilitySync_->SyncStart(context_->GetRequestSessionId(), context_->GetSequenceId(), remoteCommunicatorVersion, handler); if (errCode != E_OK) { @@ -615,6 +671,18 @@ Event SingleVerSyncStateMachine::DoAbilitySync() return Event::WAIT_ACK_EVENT; } +Event SingleVerSyncStateMachine::GetEventAfterTimeSync(int mode, bool isEarliestVersion) +{ + if (mode == SyncModeType::SUBSCRIBE_QUERY || mode == SyncModeType::UNSUBSCRIBE_QUERY) { + return Event::CONTROL_CMD_EVENT; + } + if (isEarliestVersion) { + LOGI("remote version is 0, switch to v1 proctol"); + return Event::SWITCH_TO_PROCTOL_V1_EVENT; + } + return Event::ABILITY_SYNC_FINISHED_EVENT; +} + Event SingleVerSyncStateMachine::DoSyncTaskFinished() { StopWatchDog(); @@ -630,7 +698,13 @@ Event SingleVerSyncStateMachine::DoSyncTaskFinished() Event SingleVerSyncStateMachine::DoTimeout() { RefObject::AutoLock lock(context_); - context_->Abort(SyncOperation::TIMEOUT); + if (context_->GetMode() == SyncModeType::SUBSCRIBE_QUERY) { + std::shared_ptr subManager = context_->GetSubscribeManager(); + if (subManager != nullptr) { + subManager->DeleteLocalSubscribeQuery(context_->GetDeviceId(), context_->GetQuery()); + } + } + context_->Abort(SyncOperation::OP_TIMEOUT); context_->Clear(); AbortInner(); return Event::ANY_EVENT; @@ -640,17 +714,16 @@ Event SingleVerSyncStateMachine::DoInnerErr() { RefObject::AutoLock lock(context_); if (!context_->IsCommNormal()) { - context_->Abort(SyncOperation::COMM_ABNORMAL); - } else if (context_->GetTaskErrCode() == -E_SCHEMA_MISMATCH) { - context_->Abort(SyncOperation::SCHEMA_INCOMPATIBLE); - } else if (context_->GetTaskErrCode() == -E_EKEYREVOKED) { - context_->Abort(SyncOperation::EKEYREVOKED_FAILURE); - } else if (context_->GetTaskErrCode() == -E_SECURITY_OPTION_CHECK_ERROR) { - context_->Abort(SyncOperation::SECURITY_OPTION_CHECK_FAILURE); - } else if (context_->GetTaskErrCode() == -E_BUSY) { - context_->Abort(SyncOperation::BUSY_FAILURE); + if (context_->GetMode() == SyncModeType::SUBSCRIBE_QUERY) { + std::shared_ptr subManager = context_->GetSubscribeManager(); + if (subManager != nullptr) { + subManager->DeleteLocalSubscribeQuery(context_->GetDeviceId(), context_->GetQuery()); + } + } + context_->Abort(SyncOperation::OP_COMM_ABNORMAL); } else { - context_->Abort(SyncOperation::FAILED); + int status = GetSyncOperationStatus(context_->GetTaskErrCode()); + context_->Abort(status); } context_->Clear(); AbortInner(); @@ -663,19 +736,19 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) return abilitySync_->RequestRecv(inMsg, context_); } - if (inMsg->GetMessageType() == TYPE_RESPONSE) { + if (inMsg->GetMessageType() == TYPE_RESPONSE && AbilityMsgSessionIdCheck(inMsg)) { int errCode = abilitySync_->AckRecv(inMsg, context_); std::lock_guard lock(stateMachineLock_); (void)ResetWatchDog(); - if (errCode != E_OK && errCode != E_FEEDBACK_UNKNOWN_MESSAGE) { + if (errCode != E_OK) { LOGE("[StateMachine][AbilitySyncRecv] handle ackRecv failed,errCode=%d", errCode); SwitchStateAndStep(TransformErrCodeToEvent(errCode)); } else if (context_->GetRemoteSoftwareVersion() <= SOFTWARE_VERSION_RELEASE_2_0) { abilitySync_->SetAbilitySyncFinishedStatus(true); - LOGI("[StateMachine][AbilitySyncRecv] ability Sync Finished,label=%s,dev=%s{private}", - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[StateMachine][AbilitySyncRecv] ability Sync Finished,label=%s,dev=%s", + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); currentRemoteVersionId_ = context_->GetRemoteSoftwareVersionId(); - SwitchStateAndStep(ABILITY_SYNC_FINISHED_EVENT); + JumpStatusAfterAbilitySync(context_->GetMode()); } return E_OK; } @@ -691,15 +764,17 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) SwitchStateAndStep(Event::INNER_ERR_EVENT); return E_OK; } - if (ackCode == AbilitySync::LAST_NOTIFY) { + if (ackCode == AbilitySync::LAST_NOTIFY && AbilityMsgSessionIdCheck(inMsg)) { abilitySync_->SetAbilitySyncFinishedStatus(true); - LOGI("[StateMachine][AbilitySyncRecv] ability sync finished,label=%s,dev=%s{private}", - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + // while recv last notify means ability sync finished,it is better to reset watchDog to avoid timeout. + LOGI("[StateMachine][AbilitySyncRecv] ability sync finished,label=%s,dev=%s", + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); currentRemoteVersionId_ = context_->GetRemoteSoftwareVersionId(); (static_cast(context_))->SetIsSchemaSync(true); std::lock_guard lock(stateMachineLock_); - SwitchStateAndStep(ABILITY_SYNC_FINISHED_EVENT); - } else { + (void)ResetWatchDog(); + JumpStatusAfterAbilitySync(context_->GetMode()); + } else if (ackCode != AbilitySync::LAST_NOTIFY) { abilitySync_->AckNotifyRecv(inMsg, context_); } return E_OK; @@ -711,7 +786,11 @@ int SingleVerSyncStateMachine::AbilitySyncRecv(const Message *inMsg) int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) { - StopFeedDogForSync(SyncDirectionFlag::RECEIVE); + PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); + if (performance != nullptr) { + performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); + } + DecRefCountOfFeedDogTimer(SyncDirectionFlag::RECEIVE); { std::lock_guard lockWatchDog(stateMachineLock_); if (IsNeedResetWatchdog(inMsg)) { @@ -721,9 +800,12 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) // RequestRecv will save data, it may cost a long time. // So we need to send save data notify to keep remote alive. - bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId()); + bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); WaterMark pullEndWaterkark = 0; - int errCode = dataSync_->RequestRecv(context_, inMsg, pullEndWaterkark); + int errCode = dataSync_->DataRequestRecv(context_, inMsg, pullEndWaterkark); + if (performance != nullptr) { + performance->StepTimeRecordEnd(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); + } if (isNeedStop) { StopSaveDataNotify(); } @@ -734,34 +816,9 @@ int SingleVerSyncStateMachine::HandleDataRequestRecv(const Message *inMsg) return errCode; } std::lock_guard lock(stateMachineLock_); - if (IsNeedErrCodeHandle(inMsg->GetSessionId())) { - switch (errCode) { - case E_OK: - break; - case -E_RECV_FINISHED: - context_->SetOperationStatus(SyncOperation::RECV_FINISHED); - SwitchStateAndStep(Event::RECV_FINISHED_EVENT); - break; - case -E_EKEYREVOKED: - PushPullDataRequestEvokeErrHandle(context_); - break; - case -E_BUSY: - context_->SetTaskErrCode(-E_BUSY); - SwitchStateAndStep(Event::INNER_ERR_EVENT); - break; - case -E_SECURITY_OPTION_CHECK_ERROR: - context_->SetTaskErrCode(-E_SECURITY_OPTION_CHECK_ERROR); - SwitchStateAndStep(Event::INNER_ERR_EVENT); - break; - case -E_NEED_ABILITY_SYNC: - return errCode; - default: - SwitchStateAndStep(Event::INNER_ERR_EVENT); - break; - } - } + DataRecvErrCodeHandle(inMsg->GetSessionId(), errCode); if (pullEndWaterkark > 0) { - AddPullResponseTarget(pullEndWaterkark, inMsg->GetSessionId()); + AddPullResponseTarget(inMsg, pullEndWaterkark); } return E_OK; } @@ -774,7 +831,8 @@ int SingleVerSyncStateMachine::PreHandleAckRecv(const Message *inMsg) return E_OK; } -void SingleVerSyncStateMachine::HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg) +void SingleVerSyncStateMachine::HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg, + bool ignoreInnerErr) { if (errCode == -E_RE_SEND_DATA) { // LOCAL_WATER_MARK_NOT_INIT dataSyncWithSlidingWindow_->SenderClear(); @@ -783,12 +841,13 @@ void SingleVerSyncStateMachine::HandleDataAckRecvWithSlidingWindow(int errCode, int ret = dataSyncWithSlidingWindow_->SenderAckRecv(inMsg); if (ret == -E_FINISHED) { SwitchStateAndStep(Event::SEND_FINISHED_EVENT); - } else if (ret != E_OK) { - SwitchStateAndStep(TransformErrCodeToEvent(ret)); + return; + } else if (ret == E_OK) { // do nothing and waiting for all ack receive + return; } - } else { - SwitchStateAndStep(TransformErrCodeToEvent(errCode)); + errCode = ret; } + ResponsePullError(errCode, ignoreInnerErr); } void SingleVerSyncStateMachine::NeedAbilitySyncHandle() @@ -808,8 +867,17 @@ void SingleVerSyncStateMachine::NeedAbilitySyncHandle() int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) { - StopFeedDogForSync(SyncDirectionFlag::SEND); + if (inMsg->GetMessageType() == TYPE_RESPONSE) { + DecRefCountOfFeedDogTimer(SyncDirectionFlag::SEND); + } std::lock_guard lock(stateMachineLock_); + // Unfortunately we use stateMachineLock_ in many sync process + // So a bad ack will check before the lock and wait + // And then another process is running, it will get the lock.After this process, the ack became invalid. + // If we don't check ack again, it will be delivered to dataSyncer. + if (!IsPacketValid(inMsg)) { + return -E_INVALID_ARGS; + } if (IsNeedResetWatchdog(inMsg)) { (void)ResetWatchDog(); } @@ -827,40 +895,25 @@ int SingleVerSyncStateMachine::HandleDataAckRecv(const Message *inMsg) // AckRecv will save meta data, it may cost a long time. if another thread is saving data // So we need to send save data notify to keep remote alive. // eg. remote do pull sync - bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId()); + bool isNeedStop = StartSaveDataNotify(inMsg->GetSessionId(), inMsg->GetSequenceId(), inMsg->GetMessageId()); errCode = dataSync_->AckRecv(context_, inMsg); if (isNeedStop) { StopSaveDataNotify(); } - - switch (errCode) { - case -E_NEED_ABILITY_SYNC: - NeedAbilitySyncHandle(); - break; - case -E_NO_DATA_SEND: - // when version is higher than 102, this step will be handle in slide windows function. - if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - context_->SetOperationStatus(SyncOperation::SEND_FINISHED); - } - break; - case -E_NOT_PERMIT: - context_->SetOperationStatus(SyncOperation::PERMISSION_CHECK_FAILED); - break; - case -E_EKEYREVOKED: - case -E_SECURITY_OPTION_CHECK_ERROR: - case -E_BUSY: - context_->SetTaskErrCode(errCode); - break; - case -E_SAVE_DATA_NOTIFY: - return errCode; - default: - break; + if (errCode == -E_NEED_ABILITY_SYNC || errCode == -E_RE_SEND_DATA) { + StopFeedDogForSync(SyncDirectionFlag::SEND); + } else if (errCode == -E_SAVE_DATA_NOTIFY) { + return errCode; } + // when this msg is from response task while request task is running, ignore the errCode + bool ignoreInnerErr = inMsg->GetSessionId() == context_->GetResponseSessionId() + && context_->GetRequestSessionId() != 0; + DataAckRecvErrCodeHandle(errCode, !ignoreInnerErr); if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { - SwitchStateAndStep(TransformErrCodeToEvent(errCode)); + ResponsePullError(errCode, ignoreInnerErr); return errCode; } - HandleDataAckRecvWithSlidingWindow(errCode, inMsg); + HandleDataAckRecvWithSlidingWindow(errCode, inMsg, ignoreInnerErr); return errCode; } @@ -869,19 +922,18 @@ int SingleVerSyncStateMachine::DataPktRecv(Message *inMsg) PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); int errCode; TimeOffset offset = 0; + uint32_t timeout = TIME_SYNC_WAIT_TIME; switch (inMsg->GetMessageType()) { case TYPE_REQUEST: + timeout = communicator_->GetTimeout(context_->GetDeviceId()); // If message is data sync request, we should check timeoffset. - errCode = timeSync_->GetTimeOffset(offset); + errCode = timeSync_->GetTimeOffset(offset, timeout); if (errCode != E_OK) { LOGE("[StateMachine][DataPktRecv] GetTimeOffset err! errCode=%d", errCode); return errCode; } context_->SetTimeOffset(offset); - if (performance != nullptr) { - performance->StepTimeRecordStart(PT_TEST_RECORDS::RECORD_DATA_REQUEST_RECV_TO_SEND_ACK); - } if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0) { // higher than 102 version may go to here, but it is ok not use slwr,after abilitysync would go slwr. errCode = HandleDataRequestRecv(inMsg); @@ -904,16 +956,58 @@ int SingleVerSyncStateMachine::DataPktRecv(Message *inMsg) return errCode; } +int SingleVerSyncStateMachine::ControlPktRecv(Message *inMsg) +{ + int errCode = E_OK; + switch (inMsg->GetMessageType()) { + case TYPE_REQUEST: + errCode = dataSync_->ControlCmdRequestRecv(context_, inMsg); + break; + case TYPE_RESPONSE: + errCode = HandleControlAckRecv(inMsg); + break; + default: + errCode = -E_INVALID_ARGS; + break; + } + return errCode; +} + void SingleVerSyncStateMachine::StepToTimeout() { std::lock_guard lock(stateMachineLock_); SwitchStateAndStep(Event::TIME_OUT_EVENT); } +int SingleVerSyncStateMachine::GetSyncOperationStatus(int errCode) const +{ + static const std::map statusMap = { + { -E_SCHEMA_MISMATCH, SyncOperation::OP_SCHEMA_INCOMPATIBLE }, + { -E_EKEYREVOKED, SyncOperation::OP_EKEYREVOKED_FAILURE }, + { -E_SECURITY_OPTION_CHECK_ERROR, SyncOperation::OP_SECURITY_OPTION_CHECK_FAILURE }, + { -E_BUSY, SyncOperation::OP_BUSY_FAILURE }, + { -E_NOT_PERMIT, SyncOperation::OP_PERMISSION_CHECK_FAILED }, + { -E_TIMEOUT, SyncOperation::OP_TIMEOUT }, + { -E_INVALID_QUERY_FORMAT, SyncOperation::OP_QUERY_FORMAT_FAILURE }, + { -E_INVALID_QUERY_FIELD, SyncOperation::OP_QUERY_FIELD_FAILURE }, + { -E_FEEDBACK_UNKNOWN_MESSAGE, SyncOperation::OP_NOT_SUPPORT }, + { -E_FEEDBACK_COMMUNICATOR_NOT_FOUND, SyncOperation::OP_COMM_ABNORMAL }, + { -E_NOT_SUPPORT, SyncOperation::OP_NOT_SUPPORT }, + { -E_INTERCEPT_DATA_FAIL, SyncOperation::OP_INTERCEPT_DATA_FAIL }, + { -E_MAX_LIMITS, SyncOperation::OP_MAX_LIMITS }, + { -E_NOT_REGISTER, SyncOperation::OP_NOT_SUPPORT }, + }; + auto iter = statusMap.find(errCode); + if (iter != statusMap.end()) { + return iter->second; + } + return SyncOperation::OP_FAILED; +} + int SingleVerSyncStateMachine::TimeMarkSyncRecv(const Message *inMsg) { - LOGD("[StateMachine][TimeMarkSyncRecv] type=%d,label=%s,dev=%s{private}", inMsg->GetMessageType(), - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGD("[StateMachine][TimeMarkSyncRecv] type=%d,label=%s,dev=%s", inMsg->GetMessageType(), + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); { std::lock_guard lock(stateMachineLock_); (void)ResetWatchDog(); @@ -921,9 +1015,13 @@ int SingleVerSyncStateMachine::TimeMarkSyncRecv(const Message *inMsg) if (inMsg->GetMessageType() == TYPE_REQUEST) { return timeSync_->RequestRecv(inMsg); } else if (inMsg->GetMessageType() == TYPE_RESPONSE) { - int errCode = timeSync_->AckRecv(inMsg); + int errCode = timeSync_->AckRecv(inMsg, context_->GetRequestSessionId()); if (errCode != E_OK) { LOGE("[StateMachine][TimeMarkSyncRecv] AckRecv failed errCode=%d", errCode); + if (inMsg->GetSessionId() != 0 && inMsg->GetSessionId() == context_->GetRequestSessionId()) { + context_->SetTaskErrCode(-E_FEEDBACK_COMMUNICATOR_NOT_FOUND); + CommErrAbort(); + } return errCode; } std::lock_guard lock(stateMachineLock_); @@ -950,26 +1048,41 @@ bool SingleVerSyncStateMachine::IsPacketValid(const Message *inMsg) const return false; } - if ((inMsg->GetMessageId() < TIME_SYNC_MESSAGE) || (inMsg->GetMessageId() > ABILITY_SYNC_MESSAGE)) { + if ((inMsg->GetMessageId() < TIME_SYNC_MESSAGE) || (inMsg->GetMessageId() >= UNKNOW_MESSAGE)) { LOGE("[StateMachine][IsPacketValid] Message is invalid, id=%d", inMsg->GetMessageId()); return false; } + // filter invalid ack at first + bool isResponseType = (inMsg->GetMessageType() == TYPE_RESPONSE); + if (isResponseType && (inMsg->GetMessageId() == CONTROL_SYNC_MESSAGE) && + (inMsg->GetSessionId() != context_->GetRequestSessionId())) { + LOGE("[StateMachine][IsPacketValid] Control Message is invalid, label=%s, dev=%s", + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); + return false; + } + if (isResponseType && (inMsg->GetMessageId() != TIME_SYNC_MESSAGE) && + (inMsg->GetSessionId() != context_->GetRequestSessionId()) && + (inMsg->GetSessionId() != context_->GetResponseSessionId())) { + LOGE("[StateMachine][IsPacketValid] Data Message is invalid, label=%s, dev=%s", + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); + return false; + } - if (inMsg->GetMessageId() == TIME_SYNC_MESSAGE || inMsg->GetMessageId() == ABILITY_SYNC_MESSAGE) { + // timeSync and abilitySync don't need to check sequenceId + if (inMsg->GetMessageId() == TIME_SYNC_MESSAGE || inMsg->GetMessageId() == ABILITY_SYNC_MESSAGE || + inMsg->GetMessageId() == CONTROL_SYNC_MESSAGE) { return true; } + + // sliding window don't need to check sequenceId here, just return if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0) { return true; } - if (inMsg->GetMessageType() == TYPE_RESPONSE) { - if ((inMsg->GetSequenceId() != context_->GetSequenceId()) || - ((inMsg->GetSessionId() != context_->GetRequestSessionId()) && - (inMsg->GetSessionId() != context_->GetResponseSessionId()))) { - LOGE("[StateMachine][IsPacketValid] Message is invalid,inMsg SequenceId=%d,seqId=%d,syncId=%d", - inMsg->GetSequenceId(), context_->GetSequenceId(), context_->GetSyncId()); - return false; - } + if (isResponseType && (inMsg->GetSequenceId() != context_->GetSequenceId())) { + LOGE("[StateMachine][IsPacketValid] Message is invalid,inMsg SequenceId=%d,seqId=%d,syncId=%d", + inMsg->GetSequenceId(), context_->GetSequenceId(), context_->GetSyncId()); + return false; } return true; @@ -981,13 +1094,15 @@ void SingleVerSyncStateMachine::PreStartPullResponse() context_->PopResponseTarget(target); context_->SetEndMark(target.GetEndWaterMark()); context_->SetResponseSessionId(target.GetResponseSessionId()); - context_->SetMode(SyncOperation::RESPONSE_PULL); + context_->SetMode(SyncModeType::RESPONSE_PULL); context_->ReSetSequenceId(); + context_->SetQuerySync(target.IsQuerySync()); + context_->SetQuery(target.GetQuery()); } bool SingleVerSyncStateMachine::IsRightDataResponsePkt(const Message *inMsg) const { - if (inMsg->GetMessageId() != DATA_SYNC_MESSAGE) { + if (inMsg->GetMessageId() != DATA_SYNC_MESSAGE || inMsg->GetMessageId() != QUERY_SYNC_MESSAGE) { return false; } @@ -995,7 +1110,7 @@ bool SingleVerSyncStateMachine::IsRightDataResponsePkt(const Message *inMsg) con return false; } - if ((context_->GetMode() != SyncOperation::INVALID) && (inMsg->GetMessageType() == TYPE_RESPONSE)) { + if ((context_->GetMode() != SyncModeType::INVALID_MODE) && (inMsg->GetMessageType() == TYPE_RESPONSE)) { return true; } @@ -1028,19 +1143,34 @@ int SingleVerSyncStateMachine::MessageCallbackPre(const Message *inMsg) return E_OK; } -void SingleVerSyncStateMachine::AddPullResponseTarget(WaterMark pullEndWatermark, uint32_t sessionId) +void SingleVerSyncStateMachine::AddPullResponseTarget(const Message *inMsg, WaterMark pullEndWatermark) { + int messageType = inMsg->GetMessageId(); + uint32_t sessionId = inMsg->GetSessionId(); if (pullEndWatermark == 0) { LOGE("[StateMachine][AddPullResponseTarget] pullEndWatermark is 0!"); return; } + if (context_->GetResponseSessionId() == sessionId || context_->FindResponseSyncTarget(sessionId)) { + LOGI("[StateMachine][AddPullResponseTarget] task is already running"); + return; + } + const DataRequestPacket *packet = inMsg->GetObject(); + if (packet == nullptr) { + LOGE("[AddPullResponseTarget] get packet object failed"); + return; + } SingleVerSyncTarget *targetTmp = new (std::nothrow) SingleVerSyncTarget; if (targetTmp == nullptr) { LOGE("[StateMachine][AddPullResponseTarget] add failed, may oom"); return; } targetTmp->SetTaskType(ISyncTarget::RESPONSE); - targetTmp->SetMode(SyncOperation::RESPONSE_PULL); + if (messageType == QUERY_SYNC_MESSAGE) { + targetTmp->SetQuery(packet->GetQuery()); + targetTmp->SetQuerySync(true); + } + targetTmp->SetMode(SyncModeType::RESPONSE_PULL); targetTmp->SetEndWaterMark(pullEndWatermark); targetTmp->SetResponseSessionId(sessionId); if (context_->AddSyncTarget(targetTmp) != E_OK) { @@ -1057,8 +1187,8 @@ Event SingleVerSyncStateMachine::TransformErrCodeToEvent(int errCode) switch (errCode) { case -E_TIMEOUT: return TransforTimeOutErrCodeToEvent(); - case -VERSION_NOT_SUPPORT_EVENT: - return Event::VERSION_NOT_SUPPORT_EVENT; + case -VERSION_NOT_SUPPOR_EVENT: + return Event::VERSION_NOT_SUPPOR_EVENT; case -E_SEND_DATA: return Event::SEND_DATA_EVENT; case -E_NO_DATA_SEND: @@ -1101,10 +1231,11 @@ bool SingleVerSyncStateMachine::IsNeedResetWatchdog(const Message *inMsg) const Event SingleVerSyncStateMachine::TransforTimeOutErrCodeToEvent() { - if (syncContext_->IsAutoSync() && (syncContext_->GetRetryTime() < RETRY_TIME)) { + if (syncContext_->IsSyncTaskNeedRetry() && (syncContext_->GetRetryTime() < syncContext_->GetSyncRetryTimes())) { return Event::WAIT_TIME_OUT_EVENT; + } else { + return Event::TIME_OUT_EVENT; } - return Event::TIME_OUT_EVENT; } void SingleVerSyncStateMachine::SetSlidingWindowSenderErr(bool isErr) @@ -1114,8 +1245,8 @@ void SingleVerSyncStateMachine::SetSlidingWindowSenderErr(bool isErr) bool SingleVerSyncStateMachine::IsNeedErrCodeHandle(uint32_t sessionId) const { - // version_102 omit to set sessionId so version_3 should skip to compare sessionid. - if (sessionId == context_->GetRequestSessionId() || context_->GetRequestSessionId() == 0 || + // omit to set sessionId so version_3 should skip to compare sessionid. + if (sessionId == context_->GetRequestSessionId() || context_->GetRemoteSoftwareVersion() == SOFTWARE_VERSION_RELEASE_2_0) { return true; } else { @@ -1123,18 +1254,150 @@ bool SingleVerSyncStateMachine::IsNeedErrCodeHandle(uint32_t sessionId) const } } -void SingleVerSyncStateMachine::PushPullDataRequestEvokeErrHandle(SingleVerSyncTaskContext *context) +void SingleVerSyncStateMachine::PushPullDataRequestEvokeErrHandle() { // the pushpull sync task should wait for send finished after remote dev get data occur E_EKEYREVOKED error. - if (context->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && - context->GetMode() == SyncOperation::PUSH_AND_PULL) { + if (context_->GetRemoteSoftwareVersion() > SOFTWARE_VERSION_RELEASE_2_0 && + SyncOperation::TransferSyncMode(context_->GetMode()) == SyncModeType::PUSH_AND_PULL) { LOGI("data request errCode = %d, wait for send finished", -E_EKEYREVOKED); - context->SetTaskErrCode(-E_EKEYREVOKED); - context->SetOperationStatus(SyncOperation::RECV_FINISHED); + context_->SetTaskErrCode(-E_EKEYREVOKED); + context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); SwitchStateAndStep(Event::RECV_FINISHED_EVENT); } else { - context->SetTaskErrCode(-E_EKEYREVOKED); + context_->SetTaskErrCode(-E_EKEYREVOKED); SwitchStateAndStep(Event::INNER_ERR_EVENT); } } + +void SingleVerSyncStateMachine::DataRecvErrCodeHandle(uint32_t sessionId, int errCode) +{ + if (IsNeedErrCodeHandle(sessionId)) { + switch (errCode) { + case E_OK: + break; + case -E_RECV_FINISHED: + context_->SetOperationStatus(SyncOperation::OP_RECV_FINISHED); + SwitchStateAndStep(Event::RECV_FINISHED_EVENT); + break; + case -E_EKEYREVOKED: + PushPullDataRequestEvokeErrHandle(); + break; + case -E_BUSY: + case -E_SECURITY_OPTION_CHECK_ERROR: + case -E_INVALID_QUERY_FORMAT: + case -E_INVALID_QUERY_FIELD: + case -E_INTERCEPT_DATA_FAIL: + context_->SetTaskErrCode(errCode); + SwitchStateAndStep(Event::INNER_ERR_EVENT); + break; + default: + SwitchStateAndStep(Event::INNER_ERR_EVENT); + break; + } + } +} + +bool SingleVerSyncStateMachine::AbilityMsgSessionIdCheck(const Message *inMsg) +{ + if (inMsg != nullptr && inMsg->GetSessionId() == context_->GetRequestSessionId()) { + return true; + } + LOGE("[AbilitySync] session check failed,dev=%s", STR_MASK(context_->GetDeviceId())); + return false; +} + +SyncType SingleVerSyncStateMachine::GetSyncType(uint32_t messageId) const +{ + if (messageId == QUERY_SYNC_MESSAGE) { + return SyncType::QUERY_SYNC_TYPE; + } + return SyncType::MANUAL_FULL_SYNC_TYPE; +} + +void SingleVerSyncStateMachine::DataAckRecvErrCodeHandle(int errCode, bool handleError) +{ + switch (errCode) { + case -E_NEED_ABILITY_SYNC: + NeedAbilitySyncHandle(); + break; + case -E_NO_DATA_SEND: + // when version is higher than 102, this step will be handle in slide windows function. + if (context_->GetRemoteSoftwareVersion() < SOFTWARE_VERSION_RELEASE_3_0 && handleError) { + context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); + } + break; + case -E_NOT_PERMIT: + if (handleError) { + context_->SetOperationStatus(SyncOperation::OP_PERMISSION_CHECK_FAILED); + } + break; + case -E_EKEYREVOKED: + case -E_SECURITY_OPTION_CHECK_ERROR: + case -E_BUSY: + case -E_INVALID_QUERY_FORMAT: + case -E_INVALID_QUERY_FIELD: + case -E_FEEDBACK_UNKNOWN_MESSAGE: + case -E_FEEDBACK_COMMUNICATOR_NOT_FOUND: + case -E_NOT_SUPPORT: + case -E_INTERCEPT_DATA_FAIL: + case -E_NOT_REGISTER: + case -E_MAX_LIMITS: + if (handleError) { + context_->SetTaskErrCode(errCode); + } + break; + default: + break; + } +} + +bool SingleVerSyncStateMachine::IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query) +{ + return dataSync_->IsNeedTriggerQueryAutoSync(inMsg, query); +} + +void SingleVerSyncStateMachine::JumpStatusAfterAbilitySync(int mode) +{ + if ((mode == SyncModeType::SUBSCRIBE_QUERY) || (mode == SyncModeType::UNSUBSCRIBE_QUERY)) { + SwitchStateAndStep(CONTROL_CMD_EVENT); + } else { + SwitchStateAndStep(ABILITY_SYNC_FINISHED_EVENT); + } +} + +void SingleVerSyncStateMachine::ControlAckRecvErrCodeHandle(int errCode) +{ + switch (errCode) { + case -E_NEED_ABILITY_SYNC: + NeedAbilitySyncHandle(); + break; + case -E_NO_DATA_SEND: + context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); + break; + case -E_NOT_PERMIT: + context_->SetOperationStatus(SyncOperation::OP_PERMISSION_CHECK_FAILED); + break; + // other errCode use default + default: + context_->SetTaskErrCode(errCode); + break; + } +} + +void SingleVerSyncStateMachine::GetLocalWaterMark(const DeviceID &deviceId, uint64_t &outValue) +{ + metadata_->GetLocalWaterMark(deviceId, outValue); +} + +int SingleVerSyncStateMachine::GetSendQueryWaterMark(const std::string &queryId, const DeviceID &deviceId, + bool isAutoLift, uint64_t &outValue) +{ + return metadata_->GetSendQueryWaterMark(queryId, deviceId, outValue, isAutoLift); +} + +void SingleVerSyncStateMachine::ResponsePullError(int errCode, bool ignoreInnerErr) +{ + Event event = TransformErrCodeToEvent(errCode); + SwitchStateAndStep((ignoreInnerErr && event == INNER_ERR_EVENT) ? RESPONSE_TASK_FINISHED_EVENT : event); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h index 44d5733d791d2abf42a9b88fde0454650f1148a2..5fbb258f12703e76440453fed7007a0254b6622d 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_state_machine.h @@ -19,17 +19,18 @@ #include #include -#include "sync_state_machine.h" -#include "sync_target.h" -#include "semaphore_utils.h" +#include "ability_sync.h" #include "message.h" +#include "meta_data.h" +#include "semaphore_utils.h" +#include "single_ver_data_sync.h" +#include "single_ver_data_sync_with_sliding_window.h" #include "single_ver_sync_task_context.h" +#include "sync_state_machine.h" +#include "sync_target.h" + #include "time_sync.h" #include "time_helper.h" -#include "single_ver_data_sync.h" -#include "meta_data.h" -#include "ability_sync.h" -#include "single_ver_data_sync_with_sliding_window.h" namespace DistributedDB { namespace { @@ -37,23 +38,24 @@ namespace { IDLE = 0, TIME_SYNC, ABILITY_SYNC, - START_INITIACTIVE_DATA_SYNC, // used for do sync started by local device - START_PASSIVE_DATA_SYNC, // used for do sync response remote device + START_INITIACTIVE_DATA_SYNC, // used to do sync started by local device + START_PASSIVE_DATA_SYNC, // used to do sync response remote device INACTIVE_PUSH_REMAINDER_DATA, // push remainded data if initactive sync data more than on frame PASSIVE_PUSH_REMAINDER_DATA, // push remainded data if passive sync data more than on frame WAIT_FOR_RECEIVE_DATA_FINISH, // all data send finished, wait for data revice if has pull request - SYNC_TASK_FINISHED, // current sync task finished, try to schedule next sync task + SYNC_TASK_FINISHED, // current sync task finihsed, try to schedule next sync task SYNC_TIME_OUT, INNER_ERR, - START_INITIACTIVE_SLIDING_DATA_SYNC, // used for do sync started by local device, use sliding window - START_PASSIVE_SLIDING_DATA_SYNC // used for do pull response, use sliding window + START_INITIACTIVE_SLIDING_DATA_SYNC, // used to do sync started by local device, use sliding window + START_PASSIVE_SLIDING_DATA_SYNC, // used to do pull response, use sliding window + SYNC_CONTROL_CMD // used to send control cmd. }; enum Event { START_SYNC_EVENT = 1, TIME_SYNC_FINISHED_EVENT, ABILITY_SYNC_FINISHED_EVENT, - VERSION_NOT_SUPPORT_EVENT, + VERSION_NOT_SUPPOR_EVENT, SWITCH_TO_PROCTOL_V1_EVENT, SEND_DATA_EVENT, SEND_FINISHED_EVENT, @@ -68,6 +70,7 @@ namespace { INNER_ERR_EVENT, WAIT_TIME_OUT_EVENT, RE_SEND_DATA_EVENT, + CONTROL_CMD_EVENT, ANY_EVENT }; } @@ -80,7 +83,7 @@ public: ~SingleVerSyncStateMachine() override; // Init the SingleVerSyncStateMachine - int Initialize(ISyncTaskContext *context, IKvDBSyncInterface *syncInterface, std::shared_ptr &metadata, + int Initialize(ISyncTaskContext *context, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) override; // send Message to the StateMachine @@ -95,7 +98,16 @@ public: bool IsNeedErrCodeHandle(uint32_t sessionId) const; - void PushPullDataRequestEvokeErrHandle(SingleVerSyncTaskContext *context); + void PushPullDataRequestEvokeErrHandle(); + + void DataRecvErrCodeHandle(uint32_t sessionId, int errCode); + + bool IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query) override; + + void GetLocalWaterMark(const DeviceID &deviceId, uint64_t &outValue); + + int GetSendQueryWaterMark(const std::string &queryId, const DeviceID &deviceId, bool isAutoLift, + uint64_t &outValue); protected: // Step the SingleVerSyncStateMachine @@ -122,7 +134,7 @@ protected: int PrepareNextSyncTask() override; // Called by StartSaveDataNotifyTimer, used to send a save data notify packet - void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) override; + void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) override; private: // Used to init sync state machine switchbables @@ -154,7 +166,7 @@ private: // Waiting for pull data revice finish, if coming a pull request, should goto START_PASSIVE_DATA_SYNC state Event DoWaitForDataRecv() const; - // Sync task finished, should do some data clear and exec next task. + // Sync task finihsed, should do some data clear and exec next task. Event DoSyncTaskFinished(); // Do something when sync timeout. @@ -167,19 +179,31 @@ private: Event DoPassiveDataSyncWithSlidingWindow(); + Event DoInitiactiveControlSync(); + + Event GetEventAfterTimeSync(int mode, bool isEarliestVersion); + + int HandleControlAckRecv(const Message *inMsg); + + int GetSyncOperationStatus(int errCode) const; + int TimeMarkSyncRecv(const Message *inMsg); int AbilitySyncRecv(const Message *inMsg); int DataPktRecv(Message *inMsg); + int ControlPktRecv(Message *inMsg); + void NeedAbilitySyncHandle(); int HandleDataAckRecv(const Message *inMsg); int PreHandleAckRecv(const Message *inMsg); - void HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg); + void HandleDataAckRecvWithSlidingWindow(int errCode, const Message *inMsg, bool ignoreInnerErr); + + void ResponsePullError(int errCode, bool ignoreInnerErr); void Clear(); @@ -193,7 +217,7 @@ private: int MessageCallbackPre(const Message *inMsg); - void AddPullResponseTarget(WaterMark pullEndWatermark, uint32_t sessionId); + void AddPullResponseTarget(const Message *inMsg, WaterMark pullEndWatermark); Event TransformErrCodeToEvent(int errCode); @@ -201,6 +225,16 @@ private: Event TransforTimeOutErrCodeToEvent(); + bool AbilityMsgSessionIdCheck(const Message *inMsg); + + SyncType GetSyncType(uint32_t messageId) const; + + void DataAckRecvErrCodeHandle(int errCode, bool handleError); + + void JumpStatusAfterAbilitySync(int mode); + + void ControlAckRecvErrCodeHandle(int errCode); + DISABLE_COPY_ASSIGN_MOVE(SingleVerSyncStateMachine); static std::mutex stateSwitchTableLock_; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.cpp index 40c328d4a17c621b3f9fa8150b5330c28e93ac9f..2771ec16416d668de2d6861720751caeb18ff7ad 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.cpp @@ -23,7 +23,8 @@ namespace DistributedDB { SingleVerSyncTarget::SingleVerSyncTarget() - : endWaterMark_(0) + : endWaterMark_(0), + query_(QuerySyncObject()) { } @@ -31,6 +32,15 @@ SingleVerSyncTarget::~SingleVerSyncTarget() { } +void SingleVerSyncTarget::SetSyncOperation(SyncOperation *operation) +{ + SyncTarget::SetSyncOperation(operation); + if ((operation != nullptr) && !operation->IsKilled()) { + query_ = operation->GetQuery(); + isQuerySync_ = operation->IsQuerySync(); + } +} + void SingleVerSyncTarget::SetEndWaterMark(WaterMark waterMark) { endWaterMark_ = waterMark; @@ -50,4 +60,24 @@ uint32_t SingleVerSyncTarget::GetResponseSessionId() const { return responseSessionId_; } + +void SingleVerSyncTarget::SetQuery(const QuerySyncObject &query) +{ + query_ = query; +} + +QuerySyncObject SingleVerSyncTarget::GetQuery() const +{ + return query_; +} + +void SingleVerSyncTarget::SetQuerySync(bool isQuerySync) +{ + isQuerySync_ = isQuerySync; +} + +bool SingleVerSyncTarget::IsQuerySync() const +{ + return isQuerySync_; +} } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.h index 6d1fbde3956974000b0b2ea1bb64934e442a1590..d0c405c6b8f59ec8ff7b6a5088ae935edcb8efcf 100644 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_target.h @@ -17,6 +17,7 @@ #define SINGLE_VER_SYNC_TARGET_H #include "db_types.h" +#include "query_sync_object.h" #include "sync_target.h" namespace DistributedDB { @@ -25,19 +26,29 @@ public: SingleVerSyncTarget(); ~SingleVerSyncTarget() override; + // Set a syncOperation + void SetSyncOperation(SyncOperation *operation) override; + // Set the end water mark of this task void SetEndWaterMark(WaterMark waterMark); // Get the end water mark of this task WaterMark GetEndWaterMark() const; + // For pull response sync void SetResponseSessionId(uint32_t responseSessionId); + uint32_t GetResponseSessionId() const override; - uint32_t GetResponseSessionId() const; - + // For query sync + void SetQuery(const QuerySyncObject &query); + QuerySyncObject GetQuery() const; + void SetQuerySync(bool isQuerySync); + bool IsQuerySync() const; private: WaterMark endWaterMark_; uint32_t responseSessionId_ = 0; + QuerySyncObject query_; + bool isQuerySync_ = false; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp index 97c8554110f4daad31dbf0a4cedc12f8367b4970..d07b32898fb788b571d5e6b9138141dab3f6e8db 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.cpp @@ -16,12 +16,13 @@ #include "single_ver_sync_task_context.h" #include - +#include "db_common.h" #include "db_errno.h" #include "log_print.h" #include "isyncer.h" #include "single_ver_sync_state_machine.h" #include "single_ver_sync_target.h" +#include "sync_types.h" namespace DistributedDB { SingleVerSyncTaskContext::SingleVerSyncTaskContext() @@ -34,10 +35,11 @@ SingleVerSyncTaskContext::SingleVerSyncTaskContext() SingleVerSyncTaskContext::~SingleVerSyncTaskContext() { token_ = nullptr; + subManager_ = nullptr; } int SingleVerSyncTaskContext::Initialize(const std::string &deviceId, - IKvDBSyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) + ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) { if (deviceId.empty() || syncInterface == nullptr || metadata == nullptr || communicator == nullptr) { @@ -91,15 +93,23 @@ int SingleVerSyncTaskContext::AddSyncOperation(SyncOperation *operation) // If auto sync, just update the end watermark if (operation->IsAutoSync()) { std::lock_guard lock(targetQueueLock_); - auto iter = std::find_if(requestTargetQueue_.begin(), requestTargetQueue_.end(), [](const ISyncTarget *target) { + bool isQuerySync = operation->IsQuerySync(); + std::string queryId = operation->GetQueryId(); + auto iter = std::find_if(requestTargetQueue_.begin(), requestTargetQueue_.end(), + [isQuerySync, queryId](const ISyncTarget *target) { if (target == nullptr) { return false; } + if (isQuerySync) { + SyncOperation *tmpOperation = nullptr; + target->GetSyncOperation(tmpOperation); + return (tmpOperation != nullptr && tmpOperation->GetQueryId() == queryId) && target->IsAutoSync(); + } return target->IsAutoSync(); }); if (iter != requestTargetQueue_.end()) { static_cast(*iter)->SetEndWaterMark(timeHelper_->GetTime()); - operation->SetStatus(deviceId_, SyncOperation::FINISHED_ALL); + operation->SetStatus(deviceId_, SyncOperation::OP_FINISHED_ALL); return E_OK; } } @@ -176,15 +186,17 @@ uint32_t SingleVerSyncTaskContext::GetResponseSessionId() const return responseSessionId_; } -void SingleVerSyncTaskContext::CopyTargetData(const ISyncTarget *target) +void SingleVerSyncTaskContext::CopyTargetData(const ISyncTarget *target, const TaskParam &taskParam) { const SingleVerSyncTarget *targetTmp = static_cast(target); - SyncTaskContext::CopyTargetData(target); + SyncTaskContext::CopyTargetData(target, taskParam); mode_ = targetTmp->GetMode(); endMark_ = targetTmp->GetEndWaterMark(); - if (mode_ == SyncOperation::RESPONSE_PULL) { + if (mode_ == SyncModeType::RESPONSE_PULL) { responseSessionId_ = targetTmp->GetResponseSessionId(); } + query_ = targetTmp->GetQuery(); + isQuerySync_ = targetTmp->IsQuerySync(); } void SingleVerSyncTaskContext::Clear() @@ -192,12 +204,14 @@ void SingleVerSyncTaskContext::Clear() retryTime_ = 0; ClearSyncOperation(); SyncTaskContext::Clear(); - SetMode(SyncOperation::INVALID); + SetMode(SyncModeType::INVALID_MODE); syncId_ = 0; isAutoSync_ = false; - SetOperationStatus(SyncOperation::WAITING); + SetOperationStatus(SyncOperation::OP_WAITING); SetEndMark(0); SetResponseSessionId(0); + query_ = QuerySyncObject(); + isQuerySync_ = false; } void SingleVerSyncTaskContext::Abort(int status) @@ -206,7 +220,7 @@ void SingleVerSyncTaskContext::Abort(int status) std::lock_guard lock(operationLock_); if (syncOperation_ != nullptr) { syncOperation_->SetStatus(deviceId_, status); - if ((status >= SyncOperation::FINISHED_ALL)) { + if ((status >= SyncOperation::OP_FINISHED_ALL)) { UnlockObj(); if (syncOperation_->CheckIsAllFinished()) { syncOperation_->Finished(); @@ -220,6 +234,59 @@ void SingleVerSyncTaskContext::Abort(int status) Clear(); } +void SingleVerSyncTaskContext::ClearAllSyncTask() +{ + // clear request queue sync task and responsequeue first. + std::list targetQueue; + { + std::lock_guard lock(targetQueueLock_); + LOGI("[SingleVerSyncTaskContext] request taskcount=%d，responsecount=%d", requestTargetQueue_.size(), + responseTargetQueue_.size()); + while (!requestTargetQueue_.empty()) { + ISyncTarget *tmpTarget = nullptr; + tmpTarget = requestTargetQueue_.front(); + requestTargetQueue_.pop_front(); + targetQueue.push_back(tmpTarget); + } + while (!responseTargetQueue_.empty()) { + ISyncTarget *tmpTarget = nullptr; + tmpTarget = responseTargetQueue_.front(); + responseTargetQueue_.pop_front(); + delete tmpTarget; + tmpTarget = nullptr; + } + } + while (!targetQueue.empty()) { + ISyncTarget *target = nullptr; + target = targetQueue.front(); + targetQueue.pop_front(); + SyncOperation *tmpOperation = nullptr; + target->GetSyncOperation(tmpOperation); + if (tmpOperation == nullptr) { + LOGE("[ClearAllSyncTask] tmpOperation is nullptr"); + continue; // not exit this scene + } + LOGI("[SingleVerSyncTaskContext] killing syncId=%d,dev=%s", tmpOperation->GetSyncId(), STR_MASK(deviceId_)); + if (target->IsAutoSync()) { + tmpOperation->SetStatus(deviceId_, SyncOperation::OP_FINISHED_ALL); + } else { + tmpOperation->SetStatus(deviceId_, SyncOperation::OP_COMM_ABNORMAL); + } + if (tmpOperation->CheckIsAllFinished()) { + tmpOperation->Finished(); + } + delete target; + target = nullptr; + } + if (GetTaskExecStatus() == SyncTaskContext::RUNNING) { + // clear syncing task. + isCommNormal_ = false; + stateMachine_->CommErrAbort(); + } + // reset last push status for sync merge + ResetLastPushTaskStatus(); +} + void SingleVerSyncTaskContext::EnableClearRemoteStaleData(bool enable) { needClearRemoteStaleData_ = enable; @@ -240,21 +307,16 @@ void SingleVerSyncTaskContext::StopFeedDogForSync(SyncDirectionFlag flag) stateMachine_->StopFeedDogForSync(flag); } -void SingleVerSyncTaskContext::SetSequenceStartAndEndTimeStamp(TimeStamp start, TimeStamp end) +void SingleVerSyncTaskContext::SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange) { - sequenceStartTimeStamp_ = start; - sequenceEndTimeStamp_ = end; + dataTimeRange_ = dataTimeRange; } -TimeStamp SingleVerSyncTaskContext::GetSequenceStartTimeStamp() const +SyncTimeRange SingleVerSyncTaskContext::GetDataTimeRange() const { - return sequenceStartTimeStamp_; + return dataTimeRange_; } -TimeStamp SingleVerSyncTaskContext::GetSequenceEndTimeStamp() const -{ - return sequenceEndTimeStamp_; -} void SingleVerSyncTaskContext::SetSessionEndTimeStamp(TimeStamp end) { @@ -341,12 +403,190 @@ bool SingleVerSyncTaskContext::GetIsSchemaSync() const bool SingleVerSyncTaskContext::IsSkipTimeoutError(int errCode) const { - if (errCode == -E_TIMEOUT && IsAutoSync() && (GetRetryTime() < ISyncStateMachine::RETRY_TIME)) { + if (errCode == -E_TIMEOUT && IsSyncTaskNeedRetry() && (GetRetryTime() < GetSyncRetryTimes())) { LOGE("[SingleVerSyncTaskContext] send message timeout error occurred"); return true; } else { return false; } } + +bool SingleVerSyncTaskContext::FindResponseSyncTarget(uint32_t responseSessionId) const +{ + std::lock_guard lock(targetQueueLock_); + auto iter = std::find_if(responseTargetQueue_.begin(), responseTargetQueue_.end(), + [responseSessionId](const ISyncTarget *target) { + return target->GetResponseSessionId() == responseSessionId; + }); + if (iter == responseTargetQueue_.end()) { + return false; + } + return true; +} + +void SingleVerSyncTaskContext::SetQuery(const QuerySyncObject &query) +{ + query_ = query; +} + +const QuerySyncObject &SingleVerSyncTaskContext::GetQuery() const +{ + return query_; +} + +void SingleVerSyncTaskContext::SetQuerySync(bool isQuerySync) +{ + isQuerySync_ = isQuerySync; +} + +bool SingleVerSyncTaskContext::IsQuerySync() const +{ + return isQuerySync_; +} + +std::set SingleVerSyncTaskContext::GetRemoteCompressAlgo() const +{ + std::set compressAlgoSet; + for (const auto &algo : COMPRESSALGOMAP) { + if (remoteDbAbility_.GetAbilityItem(algo.second) == SUPPORT_MARK) { + compressAlgoSet.insert(static_cast(algo.first)); + } + } + return compressAlgoSet; +} + +std::string SingleVerSyncTaskContext::GetRemoteCompressAlgoStr() const +{ + static std::map algoMap = {{CompressAlgorithm::ZLIB, "zlib"}}; + std::set remoteCompressAlgoSet = GetRemoteCompressAlgo(); + if (remoteCompressAlgoSet.size() == 0) { + return "none"; + } + std::string currentAlgoStr; + for (const auto &algo : remoteCompressAlgoSet) { + auto iter = algoMap.find(algo); + if (iter != algoMap.end()) { + currentAlgoStr += algoMap[algo] + ","; + } + } + return currentAlgoStr.substr(0, currentAlgoStr.length() - 1); +} + +void SingleVerSyncTaskContext::SetDbAbility(DbAbility &remoteDbAbility) +{ + remoteDbAbility_ = remoteDbAbility; + LOGI("[SingleVerSyncTaskContext] set dev=%s compressAlgo=%s, IsSupAllPredicateQuery=%u, IsSupSubscribeQuery=%u", + STR_MASK(GetDeviceId()), GetRemoteCompressAlgoStr().c_str(), remoteDbAbility.GetAbilityItem(ALLPREDICATEQUERY), + remoteDbAbility.GetAbilityItem(SUBSCRIBEQUERY)); +} + +CompressAlgorithm SingleVerSyncTaskContext::ChooseCompressAlgo() const +{ + std::set remoteAlgo = GetRemoteCompressAlgo(); + if (remoteAlgo.size() == 0) { + return CompressAlgorithm::NONE; + } + std::set localAlgorithmSet; + (void)(static_cast(syncInterface_))->GetCompressionAlgo(localAlgorithmSet); + std::set algoIntersection; + set_intersection(remoteAlgo.begin(), remoteAlgo.end(), localAlgorithmSet.begin(), localAlgorithmSet.end(), + inserter(algoIntersection, algoIntersection.begin())); + if (algoIntersection.size() == 0) { + return CompressAlgorithm::NONE; + } + return *(algoIntersection.begin()); +} + +const DbAbility& SingleVerSyncTaskContext::GetRemoteDbAbility() const +{ + return remoteDbAbility_; +} + +void SingleVerSyncTaskContext::SetSubscribeManager(std::shared_ptr &subManager) +{ + subManager_ = subManager; +} + +std::shared_ptr SingleVerSyncTaskContext::GetSubscribeManager() const +{ + return subManager_; +} DEFINE_OBJECT_TAG_FACILITIES(SingleVerSyncTaskContext) + +bool SingleVerSyncTaskContext::IsCurrentSyncTaskCanBeSkipped() const +{ + if (mode_ == SyncModeType::PUSH) { + if (lastFullSyncTaskStatus_ != SyncOperation::OP_FINISHED_ALL) { + return false; + } + } else if (mode_ == SyncModeType::QUERY_PUSH) { + auto it = lastQuerySyncTaskStatusMap_.find(syncOperation_->GetQueryId()); + if (it == lastQuerySyncTaskStatusMap_.end()) { + // no last query_push and push + if (lastFullSyncTaskStatus_ != SyncOperation::OP_FINISHED_ALL) { + LOGD("no prev query push or successful prev push"); + return false; + } + } else { + if (it->second != SyncOperation::OP_FINISHED_ALL) { + LOGD("last query push status = %d.", it->second); + return false; + } + } + } else { + return false; + } + + TimeStamp maxTimeStampInDb; + syncInterface_->GetMaxTimeStamp(maxTimeStampInDb); + uint64_t localWaterMark = 0; + int errCode = GetCorrectedSendWaterMarkForCurrentTask(localWaterMark); + if (errCode != E_OK) { + LOGE("GetLocalWaterMark in state machine failed: %d", errCode); + return false; + } + if (localWaterMark > maxTimeStampInDb) { + LOGD("skip current push task, deviceId_ = %s, localWaterMark = %llu, maxTimeStampInDb = %llu", + STR_MASK(deviceId_), localWaterMark, maxTimeStampInDb); + return true; + } + return false; +} + +void SingleVerSyncTaskContext::SaveLastPushTaskExecStatus(int finalStatus) +{ + if (IsTargetQueueEmpty()) { + LOGD("sync que is empty, reset last push status"); + ResetLastPushTaskStatus(); + return; + } + if (mode_ == SyncModeType::PUSH || mode_ == SyncModeType::PUSH_AND_PULL || mode_ == SyncModeType::RESPONSE_PULL) { + lastFullSyncTaskStatus_ = finalStatus; + } else if (mode_ == SyncModeType::QUERY_PUSH || mode_ == SyncModeType::QUERY_PUSH_PULL) { + lastQuerySyncTaskStatusMap_[syncOperation_->GetQueryId()] = finalStatus; + } +} + +int SingleVerSyncTaskContext::GetCorrectedSendWaterMarkForCurrentTask(uint64_t &waterMark) const +{ + if (syncOperation_->IsQuerySync()) { + LOGD("Is QuerySync"); + int errCode = static_cast(stateMachine_)->GetSendQueryWaterMark( + syncOperation_->GetQueryId(), deviceId_, + lastFullSyncTaskStatus_ == SyncOperation::OP_FINISHED_ALL, waterMark); + if (errCode != E_OK) { + return errCode; + } + } else { + LOGD("Not QuerySync"); + static_cast(stateMachine_)->GetLocalWaterMark(deviceId_, waterMark); + } + return E_OK; +} + +void SingleVerSyncTaskContext::ResetLastPushTaskStatus() +{ + lastFullSyncTaskStatus_ = SyncOperation::OP_WAITING; + lastQuerySyncTaskStatusMap_.clear(); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h index 5f2f6013777a188b4df053db4f0ebf3b4ee92a07..221d65e9953ccf3a0c202230d1307eaf058bae12 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_sync_task_context.h @@ -19,27 +19,35 @@ #include #include #include +#include +#include "db_ability.h" +#include "query_sync_object.h" +#include "single_ver_kvdb_sync_interface.h" +#include "single_ver_sync_target.h" +#include "subscribe_manager.h" #include "sync_target.h" #include "sync_task_context.h" #include "time_helper.h" -#include "single_ver_sync_target.h" + namespace DistributedDB { -class SingleVerSyncTaskContext final : public SyncTaskContext { +class SingleVerSyncTaskContext : public SyncTaskContext { public: - SingleVerSyncTaskContext(); + explicit SingleVerSyncTaskContext(); DISABLE_COPY_ASSIGN_MOVE(SingleVerSyncTaskContext); // Init SingleVerSyncTaskContext - int Initialize(const std::string &deviceId, IKvDBSyncInterface *syncInterface, std::shared_ptr &metadata, + int Initialize(const std::string &deviceId, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) override; // Add a sync task target with the operation to the queue int AddSyncOperation(SyncOperation *operation) override; + bool IsCurrentSyncTaskCanBeSkipped() const override; + // Set the end water mark of this task void SetEndMark(WaterMark endMark); @@ -66,6 +74,8 @@ public: void Abort(int status) override; + void ClearAllSyncTask() override; + // If set true, remote stale data will be clear when remote db rebuiled. void EnableClearRemoteStaleData(bool enable); @@ -78,16 +88,12 @@ public: // stop timer to ResetWatchDog when sync data one (key,value) size bigger than mtu void StopFeedDogForSync(SyncDirectionFlag flag); - // if sended by sliding window, get the start timeStamp of data in a sequence - TimeStamp GetSequenceStartTimeStamp() const; + SyncTimeRange GetDataTimeRange() const; - // if sended by sliding window, get the end timeStamp of data in a sequence - TimeStamp GetSequenceEndTimeStamp() const; - - int HandleDataRequestRecv(const Message *msg); + virtual int HandleDataRequestRecv(const Message *msg); // if sended by sliding window, set the start and and timeStamp of data in a sequence - void SetSequenceStartAndEndTimeStamp(TimeStamp start, TimeStamp end); + void SetSequenceStartAndEndTimeStamp(SyncTimeRange dataTimeRange); // if sended by sliding window, set the last data timeStamp in a sync session void SetSessionEndTimeStamp(TimeStamp end); @@ -125,9 +131,31 @@ public: bool GetIsSchemaSync() const; bool IsSkipTimeoutError(int errCode) const; + + bool FindResponseSyncTarget(uint32_t responseSessionId) const; + + // For query sync + void SetQuery(const QuerySyncObject &query); + const QuerySyncObject &GetQuery() const; + void SetQuerySync(bool isQuerySync); + bool IsQuerySync() const; + std::set GetRemoteCompressAlgo() const; + std::string GetRemoteCompressAlgoStr() const; + void SetDbAbility(DbAbility &remoteDbAbility); + CompressAlgorithm ChooseCompressAlgo() const; + const DbAbility& GetRemoteDbAbility() const; + + void SetSubscribeManager(std::shared_ptr &subManager); + std::shared_ptr GetSubscribeManager() const; + + void SaveLastPushTaskExecStatus(int finalStatus) override; + void ResetLastPushTaskStatus() override; protected: ~SingleVerSyncTaskContext() override; - void CopyTargetData(const ISyncTarget *target) override; + void CopyTargetData(const ISyncTarget *target, const TaskParam &taskParam) override; + +private: + int GetCorrectedSendWaterMarkForCurrentTask(uint64_t &waterMark) const; private: constexpr static int64_t REDUNDACE_WATER_MARK = 1 * 1000LL * 1000LL * 10LL; // 1s @@ -145,14 +173,28 @@ private: SyncStrategy syncStrategy_; bool isSchemaSync_ = false; - // in a sync session, if sended by sliding window, the min timeStamp of data in a sequence - TimeStamp sequenceStartTimeStamp_ = 0; - // in a sync session, if sended by sliding window, the max timeStamp of data in a sequence - TimeStamp sequenceEndTimeStamp_ = 0; + // normal data or delete data start timestamp and end timestamp,recorded for slws resend. + SyncTimeRange dataTimeRange_; // in a sync session, the last data timeStamp TimeStamp sessionEndTimeStamp_ = 0; + // is receive waterMark err, peerWaterMark bigger than remote localWaterMark bool isReceiveWaterMarkErr_ = false; + + // For querySync + QuerySyncObject query_; + bool isQuerySync_ = false; + + // For db ability + DbAbility remoteDbAbility_; + + // For subscribe manager + std::shared_ptr subManager_; + + // for merge sync task + int lastFullSyncTaskStatus_ = SyncOperation::Status::OP_WAITING; + // + std::unordered_map lastQuerySyncTaskStatusMap_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp index 4b87eb81fc40fc27946ee522219abf34b5d52a6b..cbe12a6d8fe5d8902866b4c1d86203d897261ce4 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.cpp @@ -12,124 +12,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include "single_ver_syncer.h" -#include -#include -#include - -#include "ikvdb_sync_interface.h" -#include "meta_data.h" -#include "log_print.h" -#include "sqlite_single_ver_natural_store.h" +#include "db_common.h" #include "single_ver_sync_engine.h" namespace DistributedDB { -SingleVerSyncer::SingleVerSyncer() - : autoSyncEnable_(false) -{ -} - -SingleVerSyncer::~SingleVerSyncer() -{ -} - -void SingleVerSyncer::EnableAutoSync(bool enable) -{ - LOGI("[Syncer] EnableAutoSync enable = %d, Label=%s", enable, label_.c_str()); - if (autoSyncEnable_ == enable) { - return; - } - - autoSyncEnable_ = enable; - if (!enable) { - return; - } - - if (!initialized_) { - LOGE("[Syncer] Syncer has not Init"); - return; - } - - std::vector devices; - GetOnlineDevices(devices); - if (devices.empty()) { - LOGI("[Syncer] EnableAutoSync no online devices"); - return; - } - int syncId = Sync(devices, SyncOperation::AUTO_PUSH, nullptr, nullptr, false); - if (syncId < MIN_VALID_SYNC_ID) { - LOGE("[Syncer] sync start by EnableAutoSync failed err %d", syncId); - } -} - -int SingleVerSyncer::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) -{ - if (metadata_ == nullptr) { - return -E_NOT_INIT; - } - return metadata_->EraseDeviceWaterMark(deviceId, isNeedHash); -} - -// Local data changed callback -void SingleVerSyncer::LocalDataChanged(int notifyEvent) +void SingleVerSyncer::RemoteDeviceOffline(const std::string &device) { - if (!initialized_) { - LOGE("[Syncer] Syncer has not Init"); - return; - } - - if (!autoSyncEnable_) { - LOGD("[Syncer] autoSync no enable"); - return; - } - - if (notifyEvent != SQLITE_GENERAL_FINISH_MIGRATE_EVENT && - notifyEvent != SQLITE_GENERAL_NS_PUT_EVENT) { - LOGD("[Syncer] ignore event:%d", notifyEvent); - return; - } - - std::vector devices; - GetOnlineDevices(devices); - if (devices.empty()) { - LOGI("[Syncer] LocalDataChanged no online devices, Label=%s", label_.c_str()); - return; - } - - int syncId = Sync(devices, SyncOperation::AUTO_PUSH, nullptr, nullptr, false); - if (syncId < MIN_VALID_SYNC_ID) { - LOGE("[Syncer] sync start by RemoteDataChanged failed err %d", syncId); - } - return; -} - -// Remote data changed callback -void SingleVerSyncer::RemoteDataChanged(const std::string &device) -{ - LOGI("[SingleVerSyncer] device online dev %s{private}", device.c_str()); - if (autoSyncEnable_) { - RefObject::IncObjRef(syncEngine_); - int retCode = RuntimeContext::GetInstance()->ScheduleTask([this, device] { - std::vector devices; - devices.push_back(device); - int syncId = Sync(devices, SyncOperation::AUTO_PUSH, nullptr, nullptr, false); - if (syncId < MIN_VALID_SYNC_ID) { - LOGE("[SingleVerSyncer] sync start by RemoteDataChanged failed err %d", syncId); - } - RefObject::DecObjRef(syncEngine_); - }); - if (retCode != E_OK) { - LOGE("[AutoLaunch] RemoteDataChanged triggler sync retCode:%d", retCode); - RefObject::DecObjRef(syncEngine_); - } - } -} - -ISyncEngine *SingleVerSyncer::CreateSyncEngine() -{ - return new (std::nothrow) SingleVerSyncEngine(); + LOGI("[SingleVerRelationalSyncer] device offline dev %s", STR_MASK(device)); + RefObject::IncObjRef(syncEngine_); + static_cast(syncEngine_)->OfflineHandleByDevice(device); + RefObject::DecObjRef(syncEngine_); } int SingleVerSyncer::SetStaleDataWipePolicy(WipePolicy policy) @@ -156,4 +50,9 @@ int SingleVerSyncer::SetStaleDataWipePolicy(WipePolicy policy) } return errCode; } -} // namespace DistributedDB + +ISyncEngine *SingleVerSyncer::CreateSyncEngine() +{ + return new (std::nothrow) SingleVerSyncEngine(); +} +} diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.h b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.h index 1a2176cd8373fa16ddaa192be59e0abcf6462447..b88b6c5204bfcbfa459f8b45ab75e9936da2fb2a 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/single_ver_syncer.h @@ -12,29 +12,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #ifndef SYNCER_H #define SYNCER_H #include "generic_syncer.h" namespace DistributedDB { -class SingleVerSyncer final : public GenericSyncer { +class SingleVerSyncer : public GenericSyncer { public: - SingleVerSyncer(); - ~SingleVerSyncer() override; - - // Enable auto sync function - void EnableAutoSync(bool enable) override; - - // delete specified device's watermark - int EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) override; - - // Local data changed callback - void LocalDataChanged(int notifyEvent) override; - - // Remote data changed callback - void RemoteDataChanged(const std::string &device) override; + void RemoteDeviceOffline(const std::string &device) override; // Set stale data wipe policy int SetStaleDataWipePolicy(WipePolicy policy) override; @@ -42,10 +28,6 @@ public: protected: // Create a sync engine, if has memory error, will return nullptr. ISyncEngine *CreateSyncEngine() override; - -private: - bool autoSyncEnable_; }; -} // namespace DistributedDB - -#endif // SYNCER_H +} +#endif // SYNCER_H diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp index 35185ff11a9ca7479d8be09909a03700f1e1e95b..3cb5c8a725d059d6120fcbdd68882b2c1e1b3679 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.cpp @@ -15,6 +15,7 @@ #include "sliding_window_receiver.h" #include "sync_task_context.h" +#include "db_common.h" namespace DistributedDB { SlidingWindowReceiver::~SlidingWindowReceiver() @@ -69,11 +70,10 @@ int SlidingWindowReceiver::PutMsg(Message *inMsg) uint32_t sessionId = inMsg->GetSessionId(); uint32_t sequenceId = inMsg->GetSequenceId(); bool isLastSequence = packet->IsLastSequence(); - uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value std::unique_lock lock(lock_); if (workingId_ != 0 && sessionId_ != 0) { LOGI("[PutMsg] task is running, wait for workdingId=%u end,seId=%u", workingId_, sequenceId); - workingTaskcv_.wait(lock); + workingTaskcv_.wait(lock, [this] { return workingId_ == 0; }); } if (sessionId_ != sessionId) { ResetInfo(); @@ -86,15 +86,23 @@ int SlidingWindowReceiver::PutMsg(Message *inMsg) if (sequenceId <= hasFinishedMaxId_) { LOGI("[slwr] seId=%u,FinishedMId_=%u,label=%s", sequenceId, hasFinishedMaxId_, dataSync_->GetLabel().c_str()); lock.unlock(); - dataSync_->SendAck(context_, sessionId_, sequenceId, packetId); + dataSync_->SendDataAck(context_, inMsg, E_OK, 0); return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; } - int errCode = ErrHandle(sequenceId); + int errCode = ErrHandle(sequenceId, packet->GetPacketId()); if (errCode != E_OK) { return errCode; } if (messageMap_.count(sequenceId) > 0) { - LOGI("[slwr] PutMsg sequenceId already in map"); + LOGI("[slwr] PutMsg sequenceId already in map label=%s,deviceId=%s", + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); + // it cannot be null here + const auto *cachePacket = messageMap_[sequenceId]->GetObject(); + if (cachePacket->GetPacketId() > packet->GetPacketId()) { + LOGI("[slwr] PutMsg receive old packet %u, new packet is %u", + packet->GetPacketId(), cachePacket->GetPacketId()); + return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; + } delete messageMap_[sequenceId]; messageMap_[sequenceId] = nullptr; } @@ -107,18 +115,29 @@ void SlidingWindowReceiver::DealMsg() { while (true) { Message *msg = nullptr; + uint64_t curPacketId = 0; { std::lock_guard lock(lock_); if (workingId_ != 0 || messageMap_.count(hasFinishedMaxId_ + 1) == 0) { - LOGI("[slwr] DealMsg do nothing workingId_=%u,hasFinishedMaxId_=%u,label=%s,deviceId=%s{private}", - workingId_, hasFinishedMaxId_, dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[slwr] DealMsg do nothing workingId_=%u,hasFinishedMaxId_=%u,label=%s,deviceId=%s", + workingId_, hasFinishedMaxId_, dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); return; } - workingId_ = hasFinishedMaxId_ + 1; - msg = messageMap_[workingId_]; - messageMap_.erase(workingId_); - LOGI("[slwr] DealMsg workingId_=%u,label=%s,deviceId=%s{private}", workingId_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + uint32_t curWorkingId = hasFinishedMaxId_ + 1; + msg = messageMap_[curWorkingId]; + messageMap_.erase(curWorkingId); + const auto *packet = msg->GetObject(); + curPacketId = packet->GetPacketId(); + if (curPacketId != 0 && curPacketId < hasFinishedPacketId_) { + delete msg; + msg = nullptr; + LOGI("[slwr] DealMsg ignore msg, packetId:%u,label=%s,deviceId=%s", + curPacketId, dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); + continue; + } + workingId_ = curWorkingId; + LOGI("[slwr] DealMsg workingId_=%u,label=%s,deviceId=%s", workingId_, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); } int errCode = context_->HandleDataRequestRecv(msg); delete msg; @@ -134,8 +153,9 @@ void SlidingWindowReceiver::DealMsg() isWaterMarkErrHappened_ = true; } else { hasFinishedMaxId_++; - LOGI("[slwr] DealMsg ok hasFinishedMaxId_=%u,label=%s,deviceId=%s{private}", hasFinishedMaxId_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + hasFinishedPacketId_ = curPacketId; + LOGI("[slwr] DealMsg ok hasFinishedMaxId_=%u,label=%s,deviceId=%s", hasFinishedMaxId_, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); } context_->SetReceiveWaterMarkErr(false); workingTaskcv_.notify_all(); @@ -159,18 +179,17 @@ int SlidingWindowReceiver::TimeOut(TimerId timerId) void SlidingWindowReceiver::StartTimer() { - LOGD("[slwr] StartTimer"); std::lock_guard lock(lock_); TimerId timerId = 0; RefObject::IncObjRef(context_); TimerAction timeOutCallback = std::bind(&SlidingWindowReceiver::TimeOut, this, std::placeholders::_1); int errCode = RuntimeContext::GetInstance()->SetTimer(IDLE_TIME_OUT, timeOutCallback, [this]() { - int errCode = RuntimeContext::GetInstance()->ScheduleTask([this]() { + int ret = RuntimeContext::GetInstance()->ScheduleTask([this]() { RefObject::DecObjRef(context_); }); - if (errCode != E_OK) { - LOGE("[slwr] timer finalizer ScheduleTask, errCode=%d", errCode); + if (ret != E_OK) { + LOGE("[slwr] timer finalizer ScheduleTask, errCode=%d", ret); } }, timerId); if (errCode != E_OK) { @@ -178,7 +197,9 @@ void SlidingWindowReceiver::StartTimer() LOGE("[slwr] timer ScheduleTask, errCode=%d", errCode); return; } + StopTimer(timerId_); timerId_ = timerId; + LOGD("[slwr] StartTimer timerId[%llu] success", timerId_); } void SlidingWindowReceiver::StopTimer() @@ -186,13 +207,19 @@ void SlidingWindowReceiver::StopTimer() TimerId timerId; { std::lock_guard lock(lock_); - LOGD("[slwr] StopTimer,remove Timer id[%llu]", timerId_); timerId = timerId_; if (timerId_ == 0) { return; } timerId_ = 0; } + StopTimer(timerId); +} + +void SlidingWindowReceiver::StopTimer(TimerId timerId) +{ + if (timerId == 0) return; + LOGD("[slwr] StopTimer,remove Timer id[%llu]", timerId); RuntimeContext::GetInstance()->RemoveTimer(timerId); } @@ -210,14 +237,18 @@ void SlidingWindowReceiver::ResetInfo() { ClearMap(); hasFinishedMaxId_ = 0; + hasFinishedPacketId_ = 0; workingId_ = 0; endId_ = 0; + isWaterMarkErrHappened_ = false; } -int SlidingWindowReceiver::ErrHandle(uint32_t sequenceId) +int SlidingWindowReceiver::ErrHandle(uint32_t sequenceId, uint64_t packetId) { - if (sequenceId == workingId_ || (endId_ != 0 && sequenceId > endId_)) { - LOGI("[slwr] PutMsg sequenceId:%u, endId_:%u, workingId_:%u!", sequenceId, endId_, workingId_); + if (sequenceId == workingId_ || (endId_ != 0 && sequenceId > endId_) + || (packetId > 0 && packetId < hasFinishedPacketId_)) { + LOGI("[slwr] PutMsg sequenceId:%u, endId_:%u, workingId_:%u, packetId:%u!", + sequenceId, endId_, workingId_, packetId); return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; } // if waterMark err when DealMsg(), the waterMark of msg in messageMap_ is also err, so we clear messageMap_ @@ -230,6 +261,8 @@ int SlidingWindowReceiver::ErrHandle(uint32_t sequenceId) if (sequenceId == 1) { isWaterMarkErrHappened_ = false; } else { + LOGI("[slwr] PutMsg With waterMark Error sequenceId:%u, endId_:%u, workingId_:%u, packetId:%u!", + sequenceId, endId_, workingId_, packetId); return -E_SLIDING_WINDOW_RECEIVER_INVALID_MSG; // drop invalid packet } } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h index af882c1480d905b5fab66808f0eb8a612bfbb0a0..461ac6465ce5a26cb3e30a7b62cdb0b6c1c71a83 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_receiver.h @@ -19,8 +19,8 @@ #include #include "message.h" -#include "single_ver_sync_task_context.h" #include "single_ver_data_sync.h" +#include "single_ver_sync_task_context.h" namespace DistributedDB { class SlidingWindowReceiver { @@ -38,9 +38,10 @@ private: int TimeOut(TimerId timerId); void StartTimer(); void StopTimer(); + static void StopTimer(TimerId timerId); void ClearMap(); void ResetInfo(); - int ErrHandle(uint32_t sequenceId); + int ErrHandle(uint32_t sequenceId, uint64_t packetId); void SetEndField(bool isLastSequence, uint32_t sequenceId); std::mutex lock_; @@ -51,9 +52,10 @@ private: std::map messageMap_; // 0 is has finished nothing; e.g. 3 is sequenceId 1 2 3 has finished, sequenceId 4 has not finished. uint32_t hasFinishedMaxId_ = 0; + uint64_t hasFinishedPacketId_ = 0; // 0 is idle uint32_t workingId_ = 0; - // 0 is has not received the end pakcet now. + // 0 is has not received the end packet now. uint32_t endId_ = 0; bool isWaterMarkErrHappened_ = false; // timeout for idle wait packet diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp index 5bbb5e383d9dadcb85d4b6ccf1c43fe12c125abf..32df1ca7619976d9d156fa71191e36383496193e 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.cpp @@ -14,7 +14,7 @@ */ #include "sliding_window_sender.h" - +#include "db_common.h" namespace DistributedDB { SlidingWindowSender::~SlidingWindowSender() { @@ -37,16 +37,17 @@ int SlidingWindowSender::ParamCheck(int32_t mode, const SingleVerSyncTaskContext return E_OK; } -void SlidingWindowSender::Init(int32_t mode, SingleVerSyncTaskContext *context, +void SlidingWindowSender::Init(int32_t inMode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync) { isErr_ = false; - mode_ = mode; + mode_ = inMode; context_->SetSessionEndTimeStamp(0); context_->ReSetSequenceId(); maxSequenceIdhasSent_ = 0; windowSize_ = MAX_WINDOW_SIZE; - if (mode == SyncOperation::PUSH || mode == SyncOperation::PUSH_AND_PULL || mode == SyncOperation::PULL) { + int mode = SyncOperation::TransferSyncMode(inMode); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL || mode == SyncModeType::PULL) { sessionId_ = context->GetRequestSessionId(); } else { sessionId_ = context->GetResponseSessionId(); @@ -71,8 +72,6 @@ int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *conte if (errCode != E_OK) { return errCode; } - LOGI("[slws] SendStart,mode=%d,label=%s,device=%s{private}", mode_, dataSync->GetLabel().c_str(), - context->GetDeviceId().c_str()); errCode = dataSync->CheckPermitSendData(mode, context); if (errCode != E_OK) { return errCode; @@ -82,11 +81,14 @@ int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *conte return ReSend(); } Init(mode, context, dataSync); - if (mode == SyncOperation::PUSH) { + LOGI("[slws] SendStart,mode=%d,label=%s,device=%s", mode_, dataSync->GetLabel().c_str(), + STR_MASK(context->GetDeviceId())); + int tmpMode = SyncOperation::TransferSyncMode(mode); + if (tmpMode == SyncModeType::PUSH) { errCode = dataSync->PushStart(context); - } else if (mode == SyncOperation::PUSH_AND_PULL) { + } else if (tmpMode == SyncModeType::PUSH_AND_PULL) { errCode = dataSync->PushPullStart(context); - } else if (mode == SyncOperation::PULL) { + } else if (tmpMode == SyncModeType::PULL) { errCode = dataSync->PullRequestStart(context); } else { errCode = dataSync->PullResponseStart(context); @@ -102,7 +104,7 @@ int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *conte return errCode; } errCode = UpdateInfo(); - if (mode == SyncOperation::PUSH_AND_PULL && context_->GetTaskErrCode() == -E_EKEYREVOKED) { + if (tmpMode == SyncModeType::PUSH_AND_PULL && context_->GetTaskErrCode() == -E_EKEYREVOKED) { LOGE("wait for recv finished for push and pull mode"); return -E_EKEYREVOKED; } @@ -115,15 +117,19 @@ int SlidingWindowSender::SendStart(int32_t mode, SingleVerSyncTaskContext *conte int SlidingWindowSender::UpdateInfo() { ReSendInfo reSendInfo; - reSendInfo.start = context_->GetSequenceStartTimeStamp(); - reSendInfo.end = context_->GetSequenceEndTimeStamp(); + SyncTimeRange reSendDataTimeRange = context_->GetDataTimeRange(); + reSendInfo.start = reSendDataTimeRange.beginTime; + reSendInfo.end = reSendDataTimeRange.endTime; + reSendInfo.deleteBeginTime = reSendDataTimeRange.deleteBeginTime; + reSendInfo.deleteEndTime = reSendDataTimeRange.deleteEndTime; reSendInfo.packetId = context_->GetPacketId(); maxSequenceIdhasSent_++; reSendMap_[maxSequenceIdhasSent_] = reSendInfo; windowSize_--; - LOGI("[slws] mode=%d,start=%llu,end=%llu,seqId=%d,packetId=%llu,window_size=%d,label=%s,device=%s{private}", mode_, - reSendInfo.start, reSendInfo.end, maxSequenceIdhasSent_, reSendInfo.packetId, windowSize_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[slws] mode=%d,start=%llu,end=%llu,deleteStart=%llu,deleteEnd=%llu,seqId=%d,packetId=%llu,window_size=%d," + "label=%s,device=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, maxSequenceIdhasSent_, reSendInfo.packetId, windowSize_, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); ContinueToken token; context_->GetContinueToken(token); if (token == nullptr) { @@ -147,12 +153,13 @@ int SlidingWindowSender::InnerSend() } int errCode; context_->IncSequenceId(); - if (mode_ == SyncOperation::PUSH || mode_ == SyncOperation::PUSH_AND_PULL) { + int mode = SyncOperation::TransferSyncMode(mode_); + if (mode == SyncModeType::PUSH || mode == SyncModeType::PUSH_AND_PULL) { errCode = dataSync_->PushStart(context_); } else { errCode = dataSync_->PullResponseStart(context_); } - if (mode_ == SyncOperation::PUSH_AND_PULL && errCode == -E_EKEYREVOKED) { + if ((mode == SyncModeType::PUSH_AND_PULL) && errCode == -E_EKEYREVOKED) { LOGE("errCode = %d, wait for recv finished for push and pull mode", errCode); isAllDataHasSent_ = true; return -E_EKEYREVOKED; @@ -181,7 +188,7 @@ int SlidingWindowSender::PreHandleAckRecv(const Message *message) LOGE("[slws] AckRecv message nullptr"); return -E_INVALID_ARGS; } - if (message->GetMessageType() == TYPE_NOTIFY) { + if (message->GetMessageType() == TYPE_NOTIFY || message->IsFeedbackError()) { return E_OK; } const DataAckPacket *packet = message->GetObject(); @@ -214,8 +221,8 @@ int SlidingWindowSender::AckRecv(const Message *message) uint64_t packetId = packet->GetPacketId(); // above 102 version data request reserve[0] store packetId value uint32_t sessionId = message->GetSessionId(); uint32_t sequenceId = message->GetSequenceId(); - LOGI("[slws] AckRecv sequecneId=%d,packetId=%llu,label=%s,dev=%s{private}", sequenceId, packetId, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); + LOGI("[slws] AckRecv sequecneId=%d,packetId=%llu,label=%s,dev=%s", sequenceId, packetId, + dataSync_->GetLabel().c_str(), STR_MASK(context_->GetDeviceId())); std::lock_guard lock(lock_); if (sessionId != sessionId_) { @@ -233,9 +240,9 @@ int SlidingWindowSender::AckRecv(const Message *message) if (!isAllDataHasSent_) { return InnerSend(); } else if (reSendMap_.size() == 0) { - context_->SetOperationStatus(SyncOperation::SEND_FINISHED); - LOGI("[slws] AckRecv all finished,label=%s,dev=%s{private}", dataSync_->GetLabel().c_str(), - context_->GetDeviceId().c_str()); + context_->SetOperationStatus(SyncOperation::OP_SEND_FINISHED); + LOGI("[slws] AckRecv all finished,label=%s,dev=%s", dataSync_->GetLabel().c_str(), + STR_MASK(context_->GetDeviceId())); InnerClear(); return -E_FINISHED; } @@ -254,10 +261,12 @@ int SlidingWindowSender::ReSend() const } uint32_t sequenceId = reSendMap_.begin()->first; ReSendInfo reSendInfo = reSendMap_.begin()->second; - LOGI("[slws] ReSend mode=%d,start=%llu,end=%llu,seqId=%d,packetId=%llu,windowsize=%d,label=%s,deviceId=%s{private}", - mode_, reSendInfo.start, reSendInfo.end, sequenceId, reSendInfo.packetId, windowSize_, - dataSync_->GetLabel().c_str(), context_->GetDeviceId().c_str()); - DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, reSendInfo.packetId}; + LOGI("[slws] ReSend mode=%d,start=%llu,end=%llu,delStart=%llu,delEnd=%llu,seqId=%d,packetId=%llu,windowsize=%d," + "label=%s,deviceId=%s", mode_, reSendInfo.start, reSendInfo.end, reSendInfo.deleteBeginTime, + reSendInfo.deleteEndTime, sequenceId, reSendInfo.packetId, windowSize_, dataSync_->GetLabel().c_str(), + STR_MASK(context_->GetDeviceId())); + DataSyncReSendInfo dataReSendInfo = {sessionId_, sequenceId, reSendInfo.start, reSendInfo.end, + reSendInfo.deleteBeginTime, reSendInfo.deleteEndTime, reSendInfo.packetId}; return dataSync_->ReSend(context_, dataReSendInfo); } diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h index 74727ac4eb26c98303ebd7be7c35bd395f2d4078..652a20a1812bdd8e3d60e82c2b512d0c6ffad387 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sliding_window_sender.h @@ -18,13 +18,15 @@ #include #include "message.h" -#include "single_ver_sync_task_context.h" #include "single_ver_data_sync.h" +#include "single_ver_sync_task_context.h" namespace DistributedDB { struct ReSendInfo { TimeStamp start = 0; TimeStamp end = 0; + TimeStamp deleteBeginTime = 0; + TimeStamp deleteEndTime = 0; // packetId is used for matched ackpacket packetId which saved in ackPacket.reserve // if equaled, means need to handle the ack, or drop. it is always increased uint64_t packetId = 0; @@ -46,14 +48,14 @@ public: private: int ParamCheck(int32_t mode, const SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); - void Init(int32_t mode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); + void Init(int32_t inMode, SingleVerSyncTaskContext *context, std::shared_ptr &dataSync); int UpdateInfo(); int InnerSend(); void InnerClear(); int ReSend() const; // when timeout, used for reSend data // initial max window size - static const int MAX_WINDOW_SIZE = 5; + static const int MAX_WINDOW_SIZE = 3; std::mutex lock_; // 0 is default invalid sessionId. uint32_t sessionId_ = 0; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/subscribe_manager.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/subscribe_manager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..23463325bf00b427f2890931d4f7e23693075261 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/subscribe_manager.cpp @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "subscribe_manager.h" +#include "db_common.h" +#include "sync_types.h" + +namespace DistributedDB { +void SubscribeManager::ClearRemoteSubscribeQuery(const std::string &device) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + ClearSubscribeQuery(device, remoteSubscribedMap_, remoteSubscribedTotalMap_); +} + +void SubscribeManager::ClearAllRemoteQuery() +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + remoteSubscribedMap_.clear(); + remoteSubscribedTotalMap_.clear(); +} + +void SubscribeManager::ClearLocalSubscribeQuery(const std::string &device) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + unFinishedLocalAutoSubMap_.erase(device); + ClearSubscribeQuery(device, localSubscribeMap_, localSubscribeTotalMap_); +} + +int SubscribeManager::ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + int errCode = ReserveSubscribeQuery(device, query, remoteSubscribedMap_, remoteSubscribedTotalMap_); + LOGI("[SubscribeManager] dev=%s,queryId=%s remote reserve err=%d", STR_MASK(device), STR_MASK(query.GetIdentify()), + errCode); + return errCode; +} + +int SubscribeManager::ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + std::string queryId = query.GetIdentify(); + int errCode = ActiveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); + LOGI("[SubscribeManager] dev=%s,queryId=%s remote active err=%d", STR_MASK(device), STR_MASK(queryId), errCode); + return errCode; +} + +int SubscribeManager::ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + int errCode = ReserveSubscribeQuery(device, query, localSubscribeMap_, localSubscribeTotalMap_); + LOGI("[SubscribeManager] dev=%s,queryId=%s local reserve err=%d", STR_MASK(device), STR_MASK(query.GetIdentify()), + errCode); + return errCode; +} + +int SubscribeManager::ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + std::string queryId = query.GetIdentify(); + int errCode = ActiveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); + LOGI("[SubscribeManager] dev=%s,queryId=%s local active err=%d", STR_MASK(device), STR_MASK(queryId), errCode); + if (errCode != E_OK) { + return errCode; + } + if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && + unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { + unFinishedLocalAutoSubMap_[device].erase(queryId); + } + return errCode; +} + +void SubscribeManager::DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + std::string queryId = query.GetIdentify(); + DeleteSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); +} + +void SubscribeManager::DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + std::string queryId = query.GetIdentify(); + DeleteSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); +} + +void SubscribeManager::PutLocalUnFiniedSubQueries(const std::string &device, + std::vector &subscribeQueries) +{ + LOGI("[SubscribeManager] put local unfinished subscribe queries, nums=%d", subscribeQueries.size()); + std::unique_lock lockGuard(localSubscribeMapLock_); + if (subscribeQueries.size() == 0) { + unFinishedLocalAutoSubMap_.erase(device); + return; + } + unFinishedLocalAutoSubMap_[device].clear(); + auto iter = unFinishedLocalAutoSubMap_.find(device); + for (const auto &query : subscribeQueries) { + iter->second.insert(query.GetIdentify()); + } +} + +void SubscribeManager::GetAllUnFinishSubQueries( + std::map> &allSyncQueries) const +{ + std::shared_lock lock(localSubscribeMapLock_); + for (auto &item : unFinishedLocalAutoSubMap_) { + if (item.second.size() == 0) { + continue; + } + allSyncQueries[item.first] = {}; + auto iter = allSyncQueries.find(item.first); + for (const auto &queryId : item.second) { + auto iterTmp = localSubscribeTotalMap_.find(queryId); + if (iterTmp == localSubscribeTotalMap_.end()) { + LOGI("[SubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryId)); + continue; + } + iter->second.push_back(iterTmp->second.first); + } + } +} + +void SubscribeManager::RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(remoteSubscribedMapLock_); + std::string queryId = query.GetIdentify(); + RemoveSubscribeQuery(device, queryId, remoteSubscribedMap_, remoteSubscribedTotalMap_); +} + +void SubscribeManager::RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query) +{ + std::unique_lock lockGuard(localSubscribeMapLock_); + std::string queryId = query.GetIdentify(); + RemoveSubscribeQuery(device, queryId, localSubscribeMap_, localSubscribeTotalMap_); + if (unFinishedLocalAutoSubMap_.find(device) != unFinishedLocalAutoSubMap_.end() && + unFinishedLocalAutoSubMap_[device].find(queryId) != unFinishedLocalAutoSubMap_[device].end()) { + unFinishedLocalAutoSubMap_[device].erase(queryId); + LOGI("[SubscribeManager] dev=%s,queryId=%s delete from UnFinishedMap", STR_MASK(device), STR_MASK(queryId)); + if (unFinishedLocalAutoSubMap_[device].size() == 0) { + LOGI("[SubscribeManager] dev=%s delete from unFinish map", STR_MASK(device)); + unFinishedLocalAutoSubMap_.erase(device); + } + } +} + +void SubscribeManager::GetLocalSubscribeQueries(const std::string &device, + std::vector &subscribeQueries) const +{ + std::shared_lock lock(localSubscribeMapLock_); + GetSubscribeQueries(device, localSubscribeMap_, localSubscribeTotalMap_, subscribeQueries); +} + +void SubscribeManager::GetRemoteSubscribeQueries(const std::string &device, + std::vector &subscribeQueries) const +{ + std::shared_lock lockGuard(remoteSubscribedMapLock_); + GetSubscribeQueries(device, remoteSubscribedMap_, remoteSubscribedTotalMap_, subscribeQueries); +} + +bool SubscribeManager::IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const +{ + std::shared_lock lockGuard(remoteSubscribedMapLock_); + auto iter = remoteSubscribedMap_.find(device); + if (iter == remoteSubscribedMap_.end()) { + LOGD("[SubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); + return false; + } + std::string queryId = query.GetIdentify(); + auto subIter = iter->second.find(queryId); + if (subIter == iter->second.end()) { + LOGE("[SubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryId)); + return false; + } + return true; +} + +void SubscribeManager::GetRemoteSubscribeQueryIds(const std::string &device, + std::vector &subscribeQueryIds) const +{ + std::shared_lock lockGuard(remoteSubscribedMapLock_); + auto iter = remoteSubscribedMap_.find(device); + if (iter == remoteSubscribedMap_.end()) { + LOGI("[SubscribeManager] dev=%s not in remoteSubscribedMap", STR_MASK(device)); + return; + } + for (const auto &queryInfo : iter->second) { + if (remoteSubscribedTotalMap_.find(queryInfo.first) == remoteSubscribedTotalMap_.end()) { + LOGE("[SubscribeManager] queryId=%s not in RemoteTotalMap", STR_MASK(queryInfo.first)); + continue; + } + subscribeQueryIds.push_back(queryInfo.first); + } +} + +int SubscribeManager::LocalSubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const +{ + std::shared_lock lock(localSubscribeMapLock_); + int devNum = localSubscribeMap_.size(); + for (const auto &device : devices) { + if (localSubscribeMap_.find(device) != localSubscribeMap_.end()) { + continue; + } + devNum++; + if (devNum > MAX_DEVICES_NUM) { + LOGE("[SubscribeManager] local subscribe devices is over limit"); + return -E_MAX_LIMITS; + } + } + std::string queryId = query.GetIdentify(); + auto allIter = localSubscribeTotalMap_.find(queryId); + if (allIter == localSubscribeTotalMap_.end() && localSubscribeTotalMap_.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { + LOGE("[SubscribeManager] all local subscribe sums is over limit"); + return -E_MAX_LIMITS; + } + return E_OK; +} + +void SubscribeManager::ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap) +{ + if (subscribeMap.find(device) == subscribeMap.end()) { + LOGI("[SubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); + return; + } + for (const auto &queryInfo : subscribeMap[device]) { + if (subscribedTotalMap.find(queryInfo.first) != subscribedTotalMap.end()) { + if (subscribedTotalMap[queryInfo.first].second > 0) { + subscribedTotalMap[queryInfo.first].second--; + } + if (subscribedTotalMap[queryInfo.first].second == 0) { + LOGI("[SubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryInfo.first)); + subscribedTotalMap.erase(queryInfo.first); + } + } + } + subscribeMap.erase(device); + LOGI("[SubscribeManager] clear dev=%s remote subscribe queies finished", STR_MASK(device)); +} + +int SubscribeManager::ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, + SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) +{ + std::string queryId = query.GetIdentify(); + auto iter = subscribeMap.find(device); + auto allIter = subscribedTotalMap.find(queryId); + // limit check + if (allIter == subscribedTotalMap.end() && subscribedTotalMap.size() >= MAX_SUBSCRIBE_NUM_PER_DB) { + LOGE("[SubscribeManager] all subscribe sums is over limit"); + return -E_MAX_LIMITS; + } + if (iter == subscribeMap.end() && subscribeMap.size() >= MAX_DEVICES_NUM) { + LOGE("[SubscribeManager] subscribe devices is over limit"); + return -E_MAX_LIMITS; + } + if (iter != subscribeMap.end() && iter->second.find(queryId) == iter->second.end() && + iter->second.size() >= MAX_SUBSCRIBE_NUM_PER_DEV) { + LOGE("[SubscribeManager] subscribe sums is over limit"); + return -E_MAX_LIMITS; + } + if (iter != subscribeMap.end() && iter->second.find(queryId) != iter->second.end() && + iter->second[queryId] == SubscribeStatus::ACTIVE) { + LOGE("[SubscribeManager] dev=%s,queryId=%s already active in map", STR_MASK(device), STR_MASK(queryId)); + return E_OK; + } + + if (iter == subscribeMap.end()) { + subscribeMap[device] = std::map {}; + } + bool isNeedInc = false; + if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { + subscribeMap[device][queryId] = SubscribeStatus::NOT_ACTIVE; + isNeedInc = true; + } + if (allIter == subscribedTotalMap.end()) { + subscribedTotalMap[queryId] = {query, 1}; + } else if (isNeedInc) { + subscribedTotalMap[queryId].second++; + } + return E_OK; +} + +int SubscribeManager::ActiveSubscribeQuery(const std::string &device, const std::string &queryId, + SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) +{ + if (subscribedTotalMap.find(queryId) == subscribedTotalMap.end()) { + LOGE("[SubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); + return -E_INTERNAL_ERROR; + } + if (subscribeMap.find(device) == subscribeMap.end()) { + LOGE("[SubscribeManager] can not find dev=%s in localSubscribeMap", STR_MASK(device)); + return -E_INTERNAL_ERROR; + } + if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { + LOGE("[SubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); + return -E_INTERNAL_ERROR; + } + subscribeMap[device][queryId] = SubscribeStatus::ACTIVE; + return E_OK; +} + +void SubscribeManager::DeleteSubscribeQuery(const std::string &device, const std::string &queryId, + SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) +{ + if (subscribeMap.find(device) == subscribeMap.end()) { + LOGE("[SubscribeManager] can not find dev=%s in map", STR_MASK(device)); + return; + } + if (subscribeMap[device].find(queryId) == subscribeMap[device].end()) { + LOGE("[SubscribeManager] can not find dev=%s,queryId=%s in map", STR_MASK(device), STR_MASK(queryId)); + return; + } + SubscribeStatus queryStatus = subscribeMap[device][queryId]; + // not permit to delete the query when something wrong this time,because it is subscribed successfully last time + if (queryStatus == SubscribeStatus::ACTIVE) { + LOGE("[SubscribeManager] dev=%s,queryId=%s is active, no need to del", STR_MASK(device), STR_MASK(queryId)); + return; + } + subscribeMap[device].erase(queryId); + auto iter = subscribedTotalMap.find(queryId); + if (iter == subscribedTotalMap.end()) { + LOGE("[SubscribeManager] can not find queryId=%s in SubscribeTotalMap", STR_MASK(queryId)); + return; + } + iter->second.second--; + if (iter->second.second <= 0) { + LOGI("[SubscribeManager] del queryId=%s from SubscribeTotalMap", STR_MASK(queryId)); + subscribedTotalMap.erase(queryId); + } + LOGI("[SubscribeManager] dev=%s,queryId=%s remove from SubscribeMap success", STR_MASK(device), STR_MASK(queryId)); +} + +void SubscribeManager::RemoveSubscribeQuery(const std::string &device, const std::string &queryId, + SubscribeMap &subscribeMap, SubscribedTotalMap &subscribedTotalMap) +{ + auto iter = subscribeMap.find(device); + if (iter == subscribeMap.end()) { + LOGE("[SubscribeManager] dev=%s not in SubscribedMap", STR_MASK(device)); + return; + } + if (iter->second.find(queryId) == subscribeMap[device].end()) { + LOGI("[SubscribeManager] dev=%s,queryId=%s not in SubscribedMap", STR_MASK(device), STR_MASK(queryId)); + return; + } + iter->second.erase(queryId); + auto allIter = subscribedTotalMap.find(queryId); + if (allIter == subscribedTotalMap.end()) { + LOGI("[SubscribeManager] queryId=%s not in TotalMap", STR_MASK(queryId)); + return; + } + allIter->second.second--; + if (allIter->second.second <= 0) { + subscribedTotalMap.erase(queryId); + LOGI("[SubscribeManager] queryId=%s delete from TotalMap", STR_MASK(queryId)); + } + LOGI("[SubscribeManager] dev=%s,queryId=%s remove from SubscribedMap success", STR_MASK(device), STR_MASK(queryId)); +} + +void SubscribeManager::GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, + const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const +{ + auto iter = subscribeMap.find(device); + if (iter == subscribeMap.end()) { + LOGD("[SubscribeManager] dev=%s not in localSubscribeMap", STR_MASK(device)); + return; + } + for (const auto &queryInfo : iter->second) { + auto iterTmp = subscribedTotalMap.find(queryInfo.first); + if (iterTmp == subscribedTotalMap.end()) { + LOGE("[SubscribeManager] queryId=%s not in localTotalMap", STR_MASK(queryInfo.first)); + continue; + } + subscribeQueries.push_back(iterTmp->second.first); + } +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/subscribe_manager.h b/services/distributeddataservice/libs/distributeddb/syncer/src/subscribe_manager.h new file mode 100644 index 0000000000000000000000000000000000000000..6b3bb1d387cf7b3dd450e144b2020ca8138fe00f --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/subscribe_manager.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SINGLE_VER_SUBSCRIBE_MANAGER_H +#define SINGLE_VER_SUBSCRIBE_MANAGER_H + +#include +#include +#include "query_sync_object.h" + +namespace DistributedDB { +enum class SubscribeStatus { + NOT_ACTIVE = 0, + ACTIVE = 1, +}; + +using SubscribeMap = std::map>; +using SubscribedTotalMap = std::map>; + +class SubscribeManager { +public: + SubscribeManager() = default; + ~SubscribeManager() {}; + + DISABLE_COPY_ASSIGN_MOVE(SubscribeManager); + + // clear remoteSubscribeMap_[device] list when remote db is closed or dev offline. + void ClearRemoteSubscribeQuery(const std::string &device); + + // clear localSubscribeMap_[device] list when device is offline. + void ClearLocalSubscribeQuery(const std::string &device); + + void ClearAllRemoteQuery(); + + // add query when receive subscribe command + int ReserveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // active query to ACTIVE when send ack ok + int ActiveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // reserve query when user call SubscribeRemoteQuery, status set to NOT_ACTIVE + int ReserveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // active query to ACTIVE when receive ack ok + int ActiveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // delete local subscribe query when recv wrong errCode, only not_active status allowed to del + void DeleteLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // delete remote subscribe query when send msg failed, only not_active status allowed to del + void DeleteRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // put subscribe queries into unfinished map when remote db online + void PutLocalUnFiniedSubQueries(const std::string &device, std::vector &subscribeQueries); + + // get all device unFinished subscribe queries which triggered by auto subscribe and need retry subscribe + void GetAllUnFinishSubQueries(std::map> &allSyncQueries) const; + + // remove query when receive unsubscribe command + void RemoveRemoteSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // remove query when user call UnSubscribeRemoteQuery + void RemoveLocalSubscribeQuery(const std::string &device, const QuerySyncObject &query); + + // get device active subscribeQueries from localSubscribeMap_ + void GetLocalSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; + + // get device remote queryId from remoteSubscribedMap_ while data change + void GetRemoteSubscribeQueryIds(const std::string &device, std::vector &subscribeQueryIds) const; + // get device remote subscribeQueries from remoteSubscribedMap_ while data change + void GetRemoteSubscribeQueries(const std::string &device, std::vector &subscribeQueries) const; + + bool IsRemoteContainSubscribe(const std::string &device, const QuerySyncObject &query) const; + + int LocalSubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const; +private: + void ClearSubscribeQuery(const std::string &device, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + int ReserveSubscribeQuery(const std::string &device, const QuerySyncObject &query, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + int ActiveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + void DeleteSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + void RemoveSubscribeQuery(const std::string &device, const std::string &queryId, SubscribeMap &subscribeMap, + SubscribedTotalMap &subscribedTotalMap); + + void GetSubscribeQueries(const std::string &device, const SubscribeMap &subscribeMap, + const SubscribedTotalMap &subscribedTotalMap, std::vector &subscribeQueries) const; + + mutable std::shared_mutex localSubscribeMapLock_; + // subscribe sponsor, key: device, value: pair map + // status 0: active, 1: not active + SubscribeMap localSubscribeMap_; + + // used retry subscribe in db open scene, key: device value: set + std::map> unFinishedLocalAutoSubMap_; + + // subscribe sponsor total query info, key:queryId, value: + // while use_num is 0, delete item from the map + SubscribedTotalMap localSubscribeTotalMap_; + + mutable std::shared_mutex remoteSubscribedMapLock_; + // subscribed, key: device, value: pair map + // status 0: active, 1: not active + SubscribeMap remoteSubscribedMap_; + + // subscribed total query info, key:queryId, value: + // while use_num is 0, delete item from the map + SubscribedTotalMap remoteSubscribedTotalMap_; +}; +} // namespace DistributedDB + +#endif // SINGLE_VER_SUBSCRIBE_MANAGER_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp index 658fa94ba4c139cb21aae644057618c455473bcb..abd479b8708d59b4704877faa6616e8df2a88a04 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.cpp @@ -15,25 +15,27 @@ #include "sync_engine.h" -#include -#include #include +#include +#include -#include "isync_state_machine.h" +#include "ability_sync.h" +#include "db_common.h" #include "db_errno.h" -#include "log_print.h" +#include "device_manager.h" #include "hash.h" +#include "isync_state_machine.h" +#include "log_print.h" #include "runtime_context.h" +#include "single_ver_serialize_manager.h" +#include "subscribe_manager.h" #include "time_sync.h" + #ifndef OMIT_MULTI_VER #include "commit_history_sync.h" #include "multi_ver_data_sync.h" #include "value_slice_sync.h" #endif -#include "single_ver_data_sync.h" -#include "ability_sync.h" -#include "device_manager.h" -#include "db_common.h" namespace DistributedDB { int SyncEngine::queueCacheSize_ = 0; @@ -47,74 +49,63 @@ SyncEngine::SyncEngine() deviceManager_(nullptr), metadata_(nullptr), timeChangedListener_(nullptr), - execTaskCount_(0) + execTaskCount_(0), + isSyncRetry_(false), + communicatorProxy_(nullptr), + isActive_(false) { } SyncEngine::~SyncEngine() { LOGD("[SyncEngine] ~SyncEngine!"); - if (syncInterface_ != nullptr) { - syncInterface_->DecRefCount(); - syncInterface_ = nullptr; - } - if (deviceManager_ != nullptr) { - delete deviceManager_; - deviceManager_ = nullptr; - } - if (timeChangedListener_ != nullptr) { - timeChangedListener_->Drop(true); - timeChangedListener_ = nullptr; - } - communicator_ = nullptr; - metadata_ = nullptr; + ClearInnerResource(); + subManager_ = nullptr; LOGD("[SyncEngine] ~SyncEngine ok!"); } -int SyncEngine::Initialize(IKvDBSyncInterface *syncInterface, std::shared_ptr &metadata, - const std::function &onRemoteDataChanged) +int SyncEngine::Initialize(ISyncInterface *syncInterface, std::shared_ptr &metadata, + const std::function &onRemoteDataChanged, const std::function &offlineChanged, + const std::function &queryAutoSyncCallback) { if ((syncInterface == nullptr) || (metadata == nullptr)) { return -E_INVALID_ARGS; } + int errCode = StartAutoSubscribeTimer(); + if (errCode != OK) { + return errCode; + } syncInterface_ = syncInterface; - int errCode = InitComunicator(syncInterface); + errCode = InitComunicator(syncInterface); if (errCode != E_OK) { LOGE("[SyncEngine] Init Communicator failed"); // There need to set nullptr. other wise, syncInterface will be // DecRef in th destroy-method. + StopAutoSubscribeTimer(); syncInterface_ = nullptr; return errCode; } onRemoteDataChanged_ = onRemoteDataChanged; - errCode = InitDeviceManager(onRemoteDataChanged); + offlineChanged_ = offlineChanged; + queryAutoSyncCallback_ = queryAutoSyncCallback; + errCode = InitDeviceManager(onRemoteDataChanged, offlineChanged); if (errCode != E_OK) { // reset ptr if initialize device manager failed syncInterface_ = nullptr; + StopAutoSubscribeTimer(); return errCode; } - + if (subManager_ == nullptr) { + subManager_ = std::make_shared(); + } metadata_ = metadata; - timeChangedListener_ = RuntimeContext::GetInstance()->RegisterTimeChangedLister( - [this](void *changedOffset) { - if (changedOffset == nullptr) { - return; - } - TimeOffset changedTimeOffset = *(reinterpret_cast(changedOffset)) * - static_cast(TimeHelper::TO_100_NS); - TimeOffset orgOffset = this->metadata_->GetLocalTimeOffset() - changedTimeOffset; - TimeStamp currentSysTime = TimeHelper::GetSysCurrentTime(); - TimeStamp maxItemTime = 0; - this->syncInterface_->GetMaxTimeStamp(maxItemTime); - if ((currentSysTime + orgOffset) <= maxItemTime) { - orgOffset = maxItemTime - currentSysTime + TimeHelper::MS_TO_100_NS; // 1ms - } - this->metadata_->SaveLocalTimeOffset(orgOffset); - }, errCode); - if (timeChangedListener_ == nullptr) { - LOGE("[SyncEngine] Init RegisterTimeChangedLister failed"); + errCode = InitTimeChangedListener(); + if (errCode != E_OK) { + syncInterface_ = nullptr; + StopAutoSubscribeTimer(); return errCode; } + isActive_ = true; LOGI("[SyncEngine] Engine init ok"); return E_OK; } @@ -122,38 +113,25 @@ int SyncEngine::Initialize(IKvDBSyncInterface *syncInterface, std::shared_ptrDrop(true); - timeChangedListener_ = nullptr; - } - if (communicator_ != nullptr) { - communicator_->RegOnMessageCallback(nullptr, nullptr); - communicator_->RegOnConnectCallback(nullptr, nullptr); - communicator_->RegOnSendableCallback(nullptr, nullptr); - } + isActive_ = false; + UnRegCommunicatorsCallback(); + StopAutoSubscribeTimer(); + // Clear SyncContexts { std::unique_lock lock(contextMapLock_); for (auto &iter : syncTaskContextMap_) { ISyncTaskContext *tempContext = iter.second; - iter.second = nullptr; lock.unlock(); RefObject::KillAndDecObjRef(tempContext); tempContext = nullptr; lock.lock(); + iter.second = nullptr; } - } - if (communicator_ != nullptr) { - ICommunicatorAggregator *communicatorAggregator = nullptr; - int errCode = RuntimeContext::GetInstance()->GetCommunicatorAggregator(communicatorAggregator); - if (communicatorAggregator == nullptr) { - LOGF("[SyncEngine] ICommunicatorAggregator get failed when fialize SyncEngine err %d", errCode); - return errCode; - } - communicatorAggregator->ReleaseCommunicator(communicator_); - communicator_ = nullptr; + syncTaskContextMap_.clear(); } + ReleaseCommunicators(); std::lock_guard msgLock(queueLock_); while (!msgQueue_.empty()) { Message *inMsg = msgQueue_.front(); @@ -163,7 +141,10 @@ int SyncEngine::Close() delete inMsg; } } - + // close db, rekey or import scene, need clear all remote query info + // local query info will destroy with syncEngine destruct + subManager_->ClearAllRemoteQuery(); + ClearInnerResource(); LOGI("[SyncEngine] SyncEngine closed!"); return E_OK; } @@ -171,20 +152,20 @@ int SyncEngine::Close() int SyncEngine::AddSyncOperation(SyncOperation *operation) { if (operation == nullptr) { + LOGE("[SyncEngine] operation is nullptr"); return -E_INVALID_ARGS; } std::vector devices = operation->GetDevices(); for (const auto &deviceId : devices) { - int checkErrCode = RunPermissionCheck(deviceId, operation->GetMode()); - if (checkErrCode == E_OK) { - operation->SetStatus(deviceId, SyncOperation::WAITING); - int errCode = AddSyncOperForContext(deviceId, operation); - if (errCode != E_OK) { - operation->SetStatus(deviceId, SyncOperation::FAILED); - } - } else { - operation->SetStatus(deviceId, SyncOperation::PERMISSION_CHECK_FAILED); + if (deviceId.size() == 0) { + operation->SetStatus(deviceId, SyncOperation::OP_INVALID_ARGS); + continue; + } + operation->SetStatus(deviceId, SyncOperation::OP_WAITING); + int errCode = AddSyncOperForContext(deviceId, operation); + if (errCode != E_OK) { + operation->SetStatus(deviceId, SyncOperation::OP_FAILED); } } return E_OK; @@ -233,7 +214,8 @@ void SyncEngine::GetOnlineDevices(std::vector &devices) const } } -int SyncEngine::InitDeviceManager(const std::function &onRemoteDataChanged) +int SyncEngine::InitDeviceManager(const std::function &onRemoteDataChanged, + const std::function &offlineChanged) { deviceManager_ = new (std::nothrow) DeviceManager(); if (deviceManager_ == nullptr) { @@ -241,7 +223,7 @@ int SyncEngine::InitDeviceManager(const std::function &onRemo return -E_OUT_OF_MEMORY; } - int errCode = deviceManager_->Initialize(communicator_, onRemoteDataChanged); + int errCode = deviceManager_->Initialize(communicatorProxy_, onRemoteDataChanged, offlineChanged); if (errCode != E_OK) { LOGE("[SyncEngine] deviceManager init failed! err %d", errCode); delete deviceManager_; @@ -251,7 +233,7 @@ int SyncEngine::InitDeviceManager(const std::function &onRemo return E_OK; } -int SyncEngine::InitComunicator(const IKvDBSyncInterface *syncInterface) +int SyncEngine::InitComunicator(const ISyncInterface *syncInterface) { ICommunicatorAggregator *communicatorAggregator = nullptr; int errCode = RuntimeContext::GetInstance()->GetCommunicatorAggregator(communicatorAggregator); @@ -259,6 +241,7 @@ int SyncEngine::InitComunicator(const IKvDBSyncInterface *syncInterface) LOGE("[SyncEngine] Get ICommunicatorAggregator error when init the sync engine err = %d", errCode); return errCode; } + std::vector label = syncInterface->GetIdentifier(); communicator_ = communicatorAggregator->AllocCommunicator(label, errCode); if (communicator_ == nullptr) { @@ -267,7 +250,7 @@ int SyncEngine::InitComunicator(const IKvDBSyncInterface *syncInterface) } errCode = communicator_->RegOnMessageCallback( - std::bind(&SyncEngine::MessageReceiveCallback, this, std::placeholders::_1, std::placeholders::_2), + std::bind(&SyncEngine::MessageReciveCallback, this, std::placeholders::_1, std::placeholders::_2), []() {}); if (errCode != E_OK) { LOGE("[SyncEngine] SyncRequestCallback register failed! err = %d", errCode); @@ -275,6 +258,15 @@ int SyncEngine::InitComunicator(const IKvDBSyncInterface *syncInterface) communicator_ = nullptr; return errCode; } + + communicatorProxy_ = new (std::nothrow) CommunicatorProxy(); + if (communicatorProxy_ == nullptr) { + communicatorAggregator->ReleaseCommunicator(communicator_); + communicator_ = nullptr; + return -E_OUT_OF_MEMORY; + } + + communicatorProxy_->SetMainCommunicator(communicator_); label.resize(3); // only show 3 Bytes enough label_ = DBCommon::VectorToHexString(label); LOGD("[SyncEngine] RegOnConnectCallback"); @@ -308,33 +300,47 @@ int SyncEngine::AddSyncOperForContext(const std::string &deviceId, SyncOperation return errCode; } -void SyncEngine::MessageReceiveCallbackTask(ISyncTaskContext *context, const ICommunicator *communicator, +void SyncEngine::MessageReciveCallbackTask(ISyncTaskContext *context, const ICommunicator *communicator, Message *inMsg) { std::string deviceId = context->GetDeviceId(); - if (inMsg == nullptr) { - LOGE("[SyncEngine] MessageReceiveCallback inMsg is null!"); - goto MSG_CALLBACK_OUT; - } - // deal remote local data changed message - if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { - if (onRemoteDataChanged_ && deviceManager_->IsDeviceOnline(deviceId)) { - onRemoteDataChanged_(deviceId); - } else { - LOGE("[SyncEngine] onRemoteDataChanged is null!"); - } - } else { // others message, it should be dealt by context + if (inMsg->GetMessageId() != LOCAL_DATA_CHANGED) { int errCode = context->ReceiveMessageCallback(inMsg); if (errCode == -E_NOT_NEED_DELETE_MSG) { goto MSG_CALLBACK_OUT_NOT_DEL; } + // add auto sync here while recv subscribe request + QuerySyncObject syncObject; + if (errCode == E_OK && context->IsNeedTriggerQueryAutoSync(inMsg, syncObject)) { + InternalSyncParma param; + GetQueryAutoSyncParam(deviceId, syncObject, param); + queryAutoSyncCallback_(param); + } } -MSG_CALLBACK_OUT: delete inMsg; inMsg = nullptr; MSG_CALLBACK_OUT_NOT_DEL: + ScheduleTaskOut(context, communicator); +} + +void SyncEngine::RemoteDataChangedTask(ISyncTaskContext *context, const ICommunicator *communicator, Message *inMsg) +{ + do { + std::string deviceId = context->GetDeviceId(); + if (onRemoteDataChanged_ && deviceManager_->IsDeviceOnline(deviceId)) { + onRemoteDataChanged_(deviceId); + } else { + LOGE("[SyncEngine] onRemoteDataChanged is null!"); + } + } while (false); + delete inMsg; + ScheduleTaskOut(context, communicator); +} + +void SyncEngine::ScheduleTaskOut(ISyncTaskContext *context, const ICommunicator *communicator) +{ (void)DealMsgUtilQueueEmpty(); { std::lock_guard lock(queueLock_); @@ -342,8 +348,6 @@ MSG_CALLBACK_OUT_NOT_DEL: } RefObject::DecObjRef(communicator); RefObject::DecObjRef(context); - communicator = nullptr; - context = nullptr; } int SyncEngine::DealMsgUtilQueueEmpty() @@ -406,16 +410,27 @@ ISyncTaskContext *SyncEngine::GetConextForMsg(const std::string &targetDev, int int SyncEngine::ScheduleDealMsg(ISyncTaskContext *context, Message *inMsg) { - RefObject::IncObjRef(communicator_); + if (inMsg == nullptr) { + LOGE("[SyncEngine] MessageReciveCallback inMsg is null!"); + return E_OK; + } + RefObject::IncObjRef(communicatorProxy_); { std::lock_guard incLock(queueLock_); execTaskCount_++; } - int errCode = RuntimeContext::GetInstance()->ScheduleTask(std::bind(&SyncEngine::MessageReceiveCallbackTask, - this, context, communicator_, inMsg)); + int errCode = E_OK; + // deal remote local data changed message + if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { + RemoteDataChangedTask(context, communicatorProxy_, inMsg); + } else { + errCode = RuntimeContext::GetInstance()->ScheduleTask(std::bind(&SyncEngine::MessageReciveCallbackTask, + this, context, communicatorProxy_, inMsg)); + } + if (errCode != E_OK) { - LOGE("[SyncEngine] MessageReceiveCallbackTask Schedule failed err %d", errCode); - RefObject::DecObjRef(communicator_); + LOGE("[SyncEngine] MessageReciveCallbackTask Schedule failed err %d", errCode); + RefObject::DecObjRef(communicatorProxy_); { std::lock_guard decLock(queueLock_); execTaskCount_--; @@ -424,27 +439,33 @@ int SyncEngine::ScheduleDealMsg(ISyncTaskContext *context, Message *inMsg) return errCode; } -void SyncEngine::MessageReceiveCallback(const std::string &targetDev, Message *inMsg) +void SyncEngine::MessageReciveCallback(const std::string &targetDev, Message *inMsg) { - int errCode = MessageReceiveCallbackInner(targetDev, inMsg); + int errCode = MessageReciveCallbackInner(targetDev, inMsg); if (errCode != E_OK) { delete inMsg; inMsg = nullptr; - LOGE("[SyncEngine] MessageReceiveCallback failed!"); + LOGE("[SyncEngine] MessageReciveCallback failed!"); } } -int SyncEngine::MessageReceiveCallbackInner(const std::string &targetDev, Message *inMsg) +int SyncEngine::MessageReciveCallbackInner(const std::string &targetDev, Message *inMsg) { if (targetDev.empty() || inMsg == nullptr) { - LOGE("[SyncEngine][MessageReceiveCallback] from a invalid device or inMsg is null "); + LOGE("[SyncEngine][MessageReciveCallback] from a invalid device or inMsg is null "); return -E_INVALID_ARGS; } - - int msgSize = GetMsgSize(inMsg); - if (msgSize <= 0) { - LOGE("[SyncEngine] GetMsgSize makes a mistake"); - return -E_NOT_SUPPORT; + if (!isActive_) { + LOGE("[SyncEngine] engine is closing, ignore msg"); + return -E_BUSY; + } + int msgSize = 0; + if (!IsSkipCalculateLen(inMsg)) { + msgSize = GetMsgSize(inMsg); + if (msgSize <= 0) { + LOGE("[SyncEngine] GetMsgSize makes a mistake"); + return -E_NOT_SUPPORT; + } } { @@ -467,7 +488,7 @@ int SyncEngine::MessageReceiveCallbackInner(const std::string &targetDev, Messag return errCode; } - LOGD("[SyncEngine] MessageReceiveCallback MSG ID = %d", inMsg->GetMessageId()); + LOGD("[SyncEngine] MessageReciveCallback MSG ID = %d", inMsg->GetMessageId()); return ScheduleDealMsg(nextContext, inMsg); } @@ -497,7 +518,9 @@ int SyncEngine::GetMsgSize(const Message *inMsg) const case ABILITY_SYNC_MESSAGE: return AbilitySync::CalculateLen(inMsg); case DATA_SYNC_MESSAGE: - return SingleVerDataSync::CalculateLen(inMsg); + case QUERY_SYNC_MESSAGE: + case CONTROL_SYNC_MESSAGE: + return SingleVerSerializeManager::CalculateLen(inMsg); #ifndef OMIT_MULTI_VER case COMMIT_HISTORY_SYNC_MESSAGE: return CommitHistorySync::CalculateLen(inMsg); @@ -524,6 +547,23 @@ ISyncTaskContext *SyncEngine::FindSyncTaskContext(const std::string &deviceId) return nullptr; } +ISyncTaskContext *SyncEngine::GetSyncTaskContextAndInc(const std::string &deviceId) +{ + ISyncTaskContext *context = nullptr; + std::lock_guard lock(contextMapLock_); + context = FindSyncTaskContext(deviceId); + if (context == nullptr) { + LOGI("[SyncEngine] dev=%s, context is null, no need to clear sync operation", STR_MASK(deviceId)); + return nullptr; + } + if (context->IsKilled()) { + LOGI("[SyncEngine] context is killing"); + return nullptr; + } + RefObject::IncObjRef(context); + return context; +} + ISyncTaskContext *SyncEngine::GetSyncTaskContext(const std::string &deviceId, int &errCode) { ISyncTaskContext *context = CreateSyncTaskContext(); @@ -532,9 +572,9 @@ ISyncTaskContext *SyncEngine::GetSyncTaskContext(const std::string &deviceId, in LOGE("[SyncEngine] SyncTaskContext alloc failed, may be no memory avliad!"); return nullptr; } - errCode = context->Initialize(deviceId, syncInterface_, metadata_, communicator_); + errCode = context->Initialize(deviceId, syncInterface_, metadata_, communicatorProxy_); if (errCode != E_OK) { - LOGE("[SyncEngine] context init failed err %d, dev %s{private}", errCode, deviceId.c_str()); + LOGE("[SyncEngine] context init failed err %d, dev %s", errCode, STR_MASK(deviceId)); RefObject::DecObjRef(context); context = nullptr; return nullptr; @@ -543,7 +583,7 @@ ISyncTaskContext *SyncEngine::GetSyncTaskContext(const std::string &deviceId, in // IncRef for SyncEngine to make sure SyncEngine is valid when context access RefObject::IncObjRef(this); context->OnLastRef([this, deviceId]() { - LOGD("[SyncEngine] SyncTaskContext for id %s{private} finalized", deviceId.c_str()); + LOGD("[SyncEngine] SyncTaskContext for id %s finalized", STR_MASK(deviceId)); RefObject::DecObjRef(this); }); context->RegOnSyncTask(std::bind(&SyncEngine::ExecSyncTask, this, context)); @@ -564,12 +604,19 @@ int SyncEngine::ExecSyncTask(ISyncTaskContext *context) context->SetTaskExecStatus(ISyncTaskContext::RUNNING); if (!context->IsTargetQueueEmpty()) { context->MoveToNextTarget(); + int checkErrCode = RunPermissionCheck(context->GetDeviceId(), + GetPermissionCheckFlag(context->IsAutoSync(), context->GetMode())); + if (checkErrCode != E_OK) { + context->SetOperationStatus(SyncOperation::OP_PERMISSION_CHECK_FAILED); + context->SetTaskExecStatus(ISyncTaskContext::FINISHED); + return checkErrCode; + } context->UnlockObj(); int errCode = context->StartStateMachine(); context->LockObj(); if (errCode != E_OK) { LOGE("[SyncEngine] machine StartSync failed"); - context->SetOperationStatus(SyncOperation::FAILED); + context->SetOperationStatus(SyncOperation::OP_FAILED); return errCode; } } else { @@ -601,38 +648,50 @@ void SyncEngine::SetMaxQueueCacheSize(int value) int SyncEngine::GetLocalIdentity(std::string &outTarget) const { - if (communicator_ == nullptr) { - LOGE("[SyncEngine] communicator_ is nullptr, return!"); + if (communicatorProxy_ == nullptr) { + LOGE("[SyncEngine] communicatorProxy_ is nullptr, return!"); return -E_NOT_INIT; } std::string deviceId; - int errCode = communicator_->GetLocalIdentity(deviceId); + int errCode = communicatorProxy_->GetLocalIdentity(deviceId); if (errCode != E_OK) { - LOGE("[SyncEngine] communicator_ GetLocalIdentity fail errCode:%d", errCode); + LOGE("[SyncEngine] communicatorProxy_ GetLocalIdentity fail errCode:%d", errCode); return errCode; } outTarget = DBCommon::TransferHashString(deviceId); return E_OK; } -int SyncEngine::RunPermissionCheck(const std::string &deviceId, int mode) const +uint8_t SyncEngine::GetPermissionCheckFlag(bool isAutoSync, int syncMode) { uint8_t flag = 0; - if (mode == SyncOperation::PUSH) { + int mode = SyncOperation::TransferSyncMode(syncMode); + if (mode == SyncModeType::PUSH || mode == SyncModeType::RESPONSE_PULL) { flag = CHECK_FLAG_SEND; - } else if (mode == SyncOperation::PULL) { + } else if (mode == SyncModeType::PULL) { flag = CHECK_FLAG_RECEIVE; - } else if (mode == SyncOperation::PUSH_AND_PULL) { + } else if (mode == SyncModeType::PUSH_AND_PULL) { flag = CHECK_FLAG_SEND | CHECK_FLAG_RECEIVE; } + if (isAutoSync) { + flag = flag | CHECK_FLAG_AUTOSYNC; + } + if (mode != SyncModeType::RESPONSE_PULL) { + // it means this sync is started by local + flag = flag | CHECK_FLAG_SPONSOR; + } + return flag; +} +int SyncEngine::RunPermissionCheck(const std::string &deviceId, uint8_t flag) const +{ std::string appId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::APP_ID, ""); std::string userId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::USER_ID, ""); std::string storeId = syncInterface_->GetDbProperties().GetStringProp(KvDBProperties::STORE_ID, ""); int errCode = RuntimeContext::GetInstance()->RunPermissionCheck(userId, appId, storeId, deviceId, flag); if (errCode != E_OK) { - LOGE("[SyncEngine] RunPermissionCheck not pass errCode:%d, flag:%d, %s{private} Label=%s", - errCode, flag, deviceId.c_str(), label_.c_str()); + LOGE("[SyncEngine] RunPermissionCheck not pass errCode:%d, flag:%d, %s Label=%s", + errCode, flag, STR_MASK(deviceId), label_.c_str()); } return errCode; } @@ -641,4 +700,273 @@ std::string SyncEngine::GetLabel() const { return label_; } + +bool SyncEngine::GetSyncRetry() const +{ + return isSyncRetry_; +} + +void SyncEngine::SetSyncRetry(bool isRetry) +{ + if (isSyncRetry_ == isRetry) { + LOGI("sync retry is equal, syncTry=%d, no need to set.", isRetry); + return; + } + isSyncRetry_ = isRetry; + std::lock_guard lock(contextMapLock_); + for (auto &iter : syncTaskContextMap_) { + ISyncTaskContext *context = iter.second; + if (context != nullptr) { + context->SetSyncRetry(isRetry); + } + } +} + +int SyncEngine::SetEqualIdentifier(const std::string &identifier, const std::vector &targets) +{ + ICommunicator *communicator = nullptr; + { + std::lock_guard lock(equalCommunicatorsLock_); + if (equalCommunicators_.count(identifier) != 0) { + communicator = equalCommunicators_[identifier]; + } else { + int errCode = E_OK; + communicator = AllocCommunicator(identifier, errCode); + if (communicator == nullptr) { + return errCode; + } + equalCommunicators_[identifier] = communicator; + } + } + LOGI("[SyncEngine] set equal identifier %s, original %s", + DBCommon::TransferStringToHex(identifier).c_str(), label_.c_str()); + communicatorProxy_->SetEqualCommunicator(communicator, targets); + communicator->Activate(); + return E_OK; +} + +void SyncEngine::OfflineHandleByDevice(const std::string &deviceId) +{ + if (communicatorProxy_ == nullptr) { + return; + } + // db closed or device is offline + // clear remote subscribe and trigger + std::vector remoteQueryId; + subManager_->GetRemoteSubscribeQueryIds(deviceId, remoteQueryId); + subManager_->ClearRemoteSubscribeQuery(deviceId); + static_cast(syncInterface_)->RemoveSubscribe(remoteQueryId); + // get context and Inc context if context is not nullprt + ISyncTaskContext *context = GetSyncTaskContextAndInc(deviceId); + if (context != nullptr) { + context->SetIsNeedResetAbilitySync(true); + } + if (communicatorProxy_->IsDeviceOnline(deviceId)) { + LOGI("[SyncEngine] target dev=%s is online, no need to clear task.", STR_MASK(deviceId)); + RefObject::DecObjRef(context); + return; + } + // means device is offline, clear local subscribe + subManager_->ClearLocalSubscribeQuery(deviceId); + // clear sync task + if (context != nullptr) { + context->ClearAllSyncTask(); + RefObject::DecObjRef(context); + } +} + +void SyncEngine::GetLocalSubscribeQueries(const std::string &device, std::vector &subscribeQueries) +{ + subManager_->GetLocalSubscribeQueries(device, subscribeQueries); +} + +void SyncEngine::GetRemoteSubscribeQueryIds(const std::string &device, std::vector &subscribeQueryIds) +{ + subManager_->GetRemoteSubscribeQueryIds(device, subscribeQueryIds); +} + +void SyncEngine::GetRemoteSubscribeQueries(const std::string &device, std::vector &subscribeQueries) +{ + subManager_->GetRemoteSubscribeQueries(device, subscribeQueries); +} + +void SyncEngine::PutUnfiniedSubQueries(const std::string &device, std::vector &subscribeQueries) +{ + subManager_->PutLocalUnFiniedSubQueries(device, subscribeQueries); +} + +void SyncEngine::GetAllUnFinishSubQueries(std::map> &allSyncQueries) +{ + subManager_->GetAllUnFinishSubQueries(allSyncQueries); +} + +ICommunicator *SyncEngine::AllocCommunicator(const std::string &identifier, int &errCode) +{ + ICommunicatorAggregator *communicatorAggregator = nullptr; + errCode = RuntimeContext::GetInstance()->GetCommunicatorAggregator(communicatorAggregator); + if (communicatorAggregator == nullptr) { + LOGE("[SyncEngine] Get ICommunicatorAggregator error when SetEqualIdentifier err = %d", errCode); + return nullptr; + } + std::vector identifierVect(identifier.begin(), identifier.end()); + auto communicator = communicatorAggregator->AllocCommunicator(identifierVect, errCode); + if (communicator == nullptr) { + LOGE("[SyncEngine] AllocCommunicator error when SetEqualIdentifier! err = %d", errCode); + return communicator; + } + + errCode = communicator->RegOnMessageCallback( + std::bind(&SyncEngine::MessageReciveCallback, this, std::placeholders::_1, std::placeholders::_2), + []() {}); + if (errCode != E_OK) { + LOGE("[SyncEngine] SyncRequestCallback register failed in SetEqualIdentifier! err = %d", errCode); + communicatorAggregator->ReleaseCommunicator(communicator); + return nullptr; + } + + errCode = communicator->RegOnConnectCallback( + std::bind(&DeviceManager::OnDeviceConnectCallback, deviceManager_, + std::placeholders::_1, std::placeholders::_2), nullptr); + if (errCode != E_OK) { + LOGE("[SyncEngine][RegConnCB] register failed in SetEqualIdentifier! err %d", errCode); + communicator->RegOnMessageCallback(nullptr, nullptr); + communicatorAggregator->ReleaseCommunicator(communicator); + return nullptr; + } + + return communicator; +} + +void SyncEngine::UnRegCommunicatorsCallback() +{ + if (communicator_ != nullptr) { + communicator_->RegOnMessageCallback(nullptr, nullptr); + communicator_->RegOnConnectCallback(nullptr, nullptr); + communicator_->RegOnSendableCallback(nullptr, nullptr); + } + + std::lock_guard lock(equalCommunicatorsLock_); + for (const auto &iter : equalCommunicators_) { + iter.second->RegOnMessageCallback(nullptr, nullptr); + iter.second->RegOnConnectCallback(nullptr, nullptr); + iter.second->RegOnSendableCallback(nullptr, nullptr); + } +} + +void SyncEngine::ReleaseCommunicators() +{ + RefObject::KillAndDecObjRef(communicatorProxy_); + communicatorProxy_ = nullptr; + ICommunicatorAggregator *communicatorAggregator = nullptr; + int errCode = RuntimeContext::GetInstance()->GetCommunicatorAggregator(communicatorAggregator); + if (communicatorAggregator == nullptr) { + LOGF("[SyncEngine] ICommunicatorAggregator get failed when fialize SyncEngine err %d", errCode); + return; + } + + if (communicator_ != nullptr) { + communicatorAggregator->ReleaseCommunicator(communicator_); + communicator_ = nullptr; + } + + std::lock_guard lock(equalCommunicatorsLock_); + for (auto &iter : equalCommunicators_) { + communicatorAggregator->ReleaseCommunicator(iter.second); + } + equalCommunicators_.clear(); +} + +bool SyncEngine::IsSkipCalculateLen(const Message *inMsg) +{ + if (inMsg->IsFeedbackError()) { + LOGE("[SyncEngine] Feedback Message with errorNo=%u.", inMsg->GetErrorNo()); + return true; + } + return false; +} + +void SyncEngine::GetSubscribeSyncParam(const std::string &device, const QuerySyncObject &query, + InternalSyncParma &outParam) +{ + outParam.devices = { device }; + outParam.mode = SyncModeType::AUTO_SUBSCRIBE_QUERY; + outParam.isQuerySync = true; + outParam.syncQuery = query; +} + +void SyncEngine::GetQueryAutoSyncParam(const std::string &device, const QuerySyncObject &query, + InternalSyncParma &outParam) +{ + outParam.devices = { device }; + outParam.mode = SyncModeType::AUTO_PUSH; + outParam.isQuerySync = true; + outParam.syncQuery = query; +} + +int SyncEngine::StartAutoSubscribeTimer() +{ + return E_OK; +} + +void SyncEngine::StopAutoSubscribeTimer() +{ +} + +int SyncEngine::InitTimeChangedListener() +{ + int errCode = E_OK; + timeChangedListener_ = RuntimeContext::GetInstance()->RegisterTimeChangedLister( + [this](void *changedOffset) { + if (changedOffset == nullptr) { + return; + } + TimeOffset changedTimeOffset = *(reinterpret_cast(changedOffset)) * + static_cast(TimeHelper::TO_100_NS); + TimeOffset orgOffset = this->metadata_->GetLocalTimeOffset() - changedTimeOffset; + TimeStamp currentSysTime = TimeHelper::GetSysCurrentTime(); + TimeStamp maxItemTime = 0; + this->syncInterface_->GetMaxTimeStamp(maxItemTime); + if ((currentSysTime + orgOffset) <= maxItemTime) { + orgOffset = maxItemTime - currentSysTime + TimeHelper::MS_TO_100_NS; // 1ms + } + this->metadata_->SaveLocalTimeOffset(orgOffset); + }, errCode); + if (timeChangedListener_ == nullptr) { + LOGE("[SyncEngine] Init RegisterTimeChangedLister failed"); + return errCode; + } + return E_OK; +} + +int SyncEngine::SubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const +{ + return subManager_->LocalSubscribeLimitCheck(devices, query); +} + + +void SyncEngine::ClearInnerResource() +{ + if (timeChangedListener_ != nullptr) { + timeChangedListener_->Drop(true); + timeChangedListener_ = nullptr; + } + if (syncInterface_ != nullptr) { + syncInterface_->DecRefCount(); + syncInterface_ = nullptr; + } + if (deviceManager_ != nullptr) { + delete deviceManager_; + deviceManager_ = nullptr; + } + communicator_ = nullptr; + metadata_ = nullptr; + onRemoteDataChanged_ = nullptr; + offlineChanged_ = nullptr; + queryAutoSyncCallback_ = nullptr; +} + +bool SyncEngine::IsEngineActive() const +{ + return isActive_; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h index 859607f49bfdcb11e1d237497aa0d0c62450d169..7456da44c9ce0af3e30ce065ff4036e26f985bcd 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_engine.h @@ -20,10 +20,12 @@ #include #include +#include "communicator_proxy.h" +#include "device_manager.h" #include "isync_engine.h" #include "isync_task_context.h" +#include "subscribe_manager.h" #include "task_pool.h" -#include "device_manager.h" namespace DistributedDB { constexpr uint16_t NEW_SEND_TASK = 1; @@ -31,11 +33,13 @@ constexpr uint16_t NEW_SEND_TASK = 1; class SyncEngine : public ISyncEngine { public: SyncEngine(); - virtual ~SyncEngine(); + ~SyncEngine() override; // Do some init things - int Initialize(IKvDBSyncInterface *syncInterface, std::shared_ptr &metadata, - const std::function &onRemoteDataChanged) override; + int Initialize(ISyncInterface *syncInterface, std::shared_ptr &metadata, + const std::function &onRemoteDataChanged, + const std::function &offlineChanged, + const std::function &queryAutoSyncCallback) override; // Do some things, when db close. int Close() override; @@ -73,38 +77,79 @@ public: std::string GetLabel() const override; + bool GetSyncRetry() const; + void SetSyncRetry(bool isRetry) override; + + // Set an equal identifier for this database, After this called, send msg to the target will use this identifier + int SetEqualIdentifier(const std::string &identifier, const std::vector &targets) override; + + void OfflineHandleByDevice(const std::string &deviceId); + + void GetLocalSubscribeQueries(const std::string &device, std::vector &subscribeQueries); + + // subscribeQueries item is queryId + void GetRemoteSubscribeQueryIds(const std::string &device, std::vector &subscribeQueryIds); + + void GetRemoteSubscribeQueries(const std::string &device, std::vector &subscribeQueries); + + void PutUnfiniedSubQueries(const std::string &device, std::vector &subscribeQueries); + + void GetAllUnFinishSubQueries(std::map> &allSyncQueries); + + // used by SingleVerSyncer when db online + int StartAutoSubscribeTimer() override; + + // used by SingleVerSyncer when remote/local db closed + void StopAutoSubscribeTimer() override; + + int SubscribeLimitCheck(const std::vector &devices, QuerySyncObject &query) const override; + + bool IsEngineActive() const override; + protected: // Create a context virtual ISyncTaskContext *CreateSyncTaskContext() = 0; // Find SyncTaskContext from the map ISyncTaskContext *FindSyncTaskContext(const std::string &deviceId); + ISyncTaskContext *GetSyncTaskContextAndInc(const std::string &deviceId); + void GetQueryAutoSyncParam(const std::string &device, const QuerySyncObject &query, InternalSyncParma &outParam); + void GetSubscribeSyncParam(const std::string &device, const QuerySyncObject &query, InternalSyncParma &outParam); // Used to store all send sync task infos (such as pull sync response, and push sync request) std::map syncTaskContextMap_; std::mutex contextMapLock_; + std::shared_ptr subManager_; + std::function queryAutoSyncCallback_; private: // Init DeviceManager set callback - int InitDeviceManager(const std::function &onRemoteDataChanged); + int InitDeviceManager(const std::function &onRemoteDataChanged, + const std::function &offlineChanged); + + int InitTimeChangedListener(); ISyncTaskContext *GetSyncTaskContext(const std::string &deviceId, int &errCode); // Init Comunicator, register callbacks - int InitComunicator(const IKvDBSyncInterface *syncInterface); + int InitComunicator(const ISyncInterface *syncInterface); // Add the sync task info to the map. int AddSyncOperForContext(const std::string &deviceId, SyncOperation *operation); // Sync Request CallbackTask run at a sub thread. - void MessageReceiveCallbackTask(ISyncTaskContext *context, const ICommunicator *communicator, Message *inMsg); + void MessageReciveCallbackTask(ISyncTaskContext *context, const ICommunicator *communicator, Message *inMsg); - // wrapper of MessageReceiveCallbackTask - void MessageReceiveCallback(const std::string &targetDev, Message *inMsg); + void RemoteDataChangedTask(ISyncTaskContext *context, const ICommunicator *communicator, Message *inMsg); + + void ScheduleTaskOut(ISyncTaskContext *context, const ICommunicator *communicator); + + // wrapper of MessageReciveCallbackTask + void MessageReciveCallback(const std::string &targetDev, Message *inMsg); // Sync Request Callback - int MessageReceiveCallbackInner(const std::string &targetDev, Message *inMsg); + int MessageReciveCallbackInner(const std::string &targetDev, Message *inMsg); // Exec the given SyncTarget. and callback onComplete. int ExecSyncTask(ISyncTaskContext *context); @@ -126,17 +171,34 @@ private: ISyncTaskContext *GetConextForMsg(const std::string &targetDev, int &errCode); - int RunPermissionCheck(const std::string &deviceId, int mode) const; + int RunPermissionCheck(const std::string &deviceId, uint8_t flag) const; + + ICommunicator *AllocCommunicator(const std::string &identifier, int &errCode); + + void UnRegCommunicatorsCallback(); + + void ReleaseCommunicators(); + + bool IsSkipCalculateLen(const Message *inMsg); + + void ClearInnerResource(); + + static uint8_t GetPermissionCheckFlag(bool isAutoSync, int syncMode); - IKvDBSyncInterface *syncInterface_; + ISyncInterface *syncInterface_; ICommunicator *communicator_; DeviceManager *deviceManager_; std::function onRemoteDataChanged_; + std::function offlineChanged_; std::shared_ptr metadata_; std::deque msgQueue_; NotificationChain::Listener *timeChangedListener_; - unsigned int execTaskCount_; + uint32_t execTaskCount_; std::string label_; + bool isSyncRetry_; + CommunicatorProxy *communicatorProxy_; + std::mutex equalCommunicatorsLock_; + std::map equalCommunicators_; static int queueCacheSize_; static int maxQueueCacheSize_; @@ -144,6 +206,7 @@ private: static const unsigned int MAX_EXEC_NUM = 7; // Set the maximum of threads as 6 < 7 static constexpr int DEFAULT_CACHE_SIZE = 160 * 1024 * 1024; // Initial the default cache size of queue as 160MB static std::mutex queueLock_; + std::atomic isActive_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp index e5c6996fd43dde6a096803f6f3d716ff7b4c6c93..bdafc83ef993c7028b753148f9500de7a0f44f24 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.cpp @@ -14,7 +14,6 @@ */ #include "sync_operation.h" - #include "db_errno.h" #include "log_print.h" #include "performance_analysis.h" @@ -29,7 +28,10 @@ SyncOperation::SyncOperation(uint32_t syncId, const std::vector &de isBlockSync_(isBlockSync), isAutoSync_(false), isFinished_(false), - semaphore_(nullptr) + semaphore_(nullptr), + query_(QuerySyncObject()), + isQuerySync_(false), + isAutoSubscribe_(false) { } @@ -44,16 +46,19 @@ int SyncOperation::Initialize() LOGD("[SyncOperation] Init SyncOperation id:%d.", syncId_); AutoLock lockGuard(this); for (const std::string &deviceId : devices_) { - statuses_.insert(std::pair(deviceId, WAITING)); + statuses_.insert(std::pair(deviceId, OP_WAITING)); } + if (mode_ == AUTO_PUSH) { mode_ = PUSH; isAutoSync_ = true; } else if (mode_ == AUTO_PULL) { mode_ = PULL; isAutoSync_ = true; + } else if (mode_ == AUTO_SUBSCRIBE_QUERY) { + mode_ = SUBSCRIBE_QUERY; + isAutoSubscribe_ = true; } - if (isBlockSync_) { semaphore_ = std::make_unique(0); } @@ -86,7 +91,7 @@ void SyncOperation::SetStatus(const std::string &deviceId, int status) auto iter = statuses_.find(deviceId); if (iter != statuses_.end()) { - if (iter->second >= FINISHED_ALL) { + if (iter->second >= OP_FINISHED_ALL) { return; } iter->second = status; @@ -116,12 +121,14 @@ int SyncOperation::GetMode() const void SyncOperation::Finished() { + std::map tmpStatus; { AutoLock lockGuard(this); if (IsKilled() || isFinished_) { return; } isFinished_ = true; + tmpStatus = statuses_; } PerformanceAnalysis *performance = PerformanceAnalysis::GetInstance(); if (performance != nullptr) { @@ -129,7 +136,19 @@ void SyncOperation::Finished() } if (userCallback_) { LOGI("[SyncOperation] Sync %d finished call onComplete.", syncId_); - userCallback_(statuses_); + if (IsBlockSync()) { + userCallback_(tmpStatus); + } else { + RefObject::IncObjRef(this); + int errCode = RuntimeContext::GetInstance()->ScheduleQueuedTask(identifier_, [this, tmpStatus] { + userCallback_(tmpStatus); + RefObject::DecObjRef(this); + }); + if (errCode != E_OK) { + LOGE("[Finished] SyncOperation Finished userCallback_ retCode:%d", errCode); + RefObject::DecObjRef(this); + } + } } if (onFinished_) { LOGD("[SyncOperation] Sync %d finished call onFinished.", syncId_); @@ -168,11 +187,16 @@ bool SyncOperation::IsBlockSync() const return isBlockSync_; } +bool SyncOperation::IsAutoControlCmd() const +{ + return isAutoSubscribe_; +} + bool SyncOperation::CheckIsAllFinished() const { AutoLock lockGuard(this); for (const auto &iter : statuses_) { - if (iter.second < FINISHED_ALL) { + if (iter.second < OP_FINISHED_ALL) { return false; } } @@ -187,5 +211,63 @@ void SyncOperation::Finalize() } } +void SyncOperation::SetQuery(const QuerySyncObject &query) +{ + query_ = query; + isQuerySync_ = true; + if (mode_ != SyncModeType::SUBSCRIBE_QUERY && mode_ != SyncModeType::UNSUBSCRIBE_QUERY) { + mode_ += QUERY_SYNC_MODE_BASE; + } +} + +QuerySyncObject SyncOperation::GetQuery() const +{ + return query_; +} + +bool SyncOperation::IsQuerySync() const +{ + return isQuerySync_; +} + +void SyncOperation::SetIdentifier(const std::vector &identifier) +{ + identifier_.assign(identifier.begin(), identifier.end()); +} + +SyncType SyncOperation::GetSyncType(int mode) +{ + static const std::map syncTypeMap = { + {SyncModeType::PUSH, SyncType::MANUAL_FULL_SYNC_TYPE}, + {SyncModeType::PULL, SyncType::MANUAL_FULL_SYNC_TYPE}, + {SyncModeType::PUSH_AND_PULL, SyncType::MANUAL_FULL_SYNC_TYPE}, + {SyncModeType::RESPONSE_PULL, SyncType::MANUAL_FULL_SYNC_TYPE}, + {SyncModeType::AUTO_PULL, SyncType::AUTO_SYNC_TYPE}, + {SyncModeType::AUTO_PUSH, SyncType::AUTO_SYNC_TYPE}, + {SyncModeType::QUERY_PUSH, SyncType::QUERY_SYNC_TYPE}, + {SyncModeType::QUERY_PULL, SyncType::QUERY_SYNC_TYPE}, + {SyncModeType::QUERY_PUSH_PULL, SyncType::QUERY_SYNC_TYPE}, + }; + auto iter = syncTypeMap.find(mode); + if (iter != syncTypeMap.end()) { + return iter->second; + } + return SyncType::INVALID_SYNC_TYPE; +} + +int SyncOperation::TransferSyncMode(int mode) +{ + // AUTO_PUSH and AUTO_PULL mode is used before sync, RESPONSE_PULL is regarded as push or query push mode. + // so for the three mode, it is no need to transfered. + if (mode >= SyncModeType::QUERY_PUSH && mode <= SyncModeType::QUERY_PUSH_PULL) { + return (mode - QUERY_SYNC_MODE_BASE); + } + return mode; +} + +std::string SyncOperation::GetQueryId() const +{ + return query_.GetIdentify(); +} DEFINE_OBJECT_TAG_FACILITIES(SyncOperation) } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/sync_operation.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h old mode 100755 new mode 100644 similarity index 70% rename from services/distributeddataservice/libs/distributeddb/syncer/include/sync_operation.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h index 913252f4f6393a1f5549b328924563a1bf3874d2..547d17a99a96a75cb03de341f1b465ad8169903b --- a/services/distributeddataservice/libs/distributeddb/syncer/include/sync_operation.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_operation.h @@ -23,38 +23,36 @@ #include #include "ikvdb_sync_interface.h" -#include "ref_object.h" -#include "semaphore_utils.h" #include "notification_chain.h" +#include "query_sync_object.h" +#include "ref_object.h" #include "runtime_context.h" +#include "semaphore_utils.h" +#include "sync_types.h" namespace DistributedDB { class SyncOperation : public RefObject { public: enum Status { - WAITING = 0, - SYNCING, - SEND_FINISHED, - RECV_FINISHED, - FINISHED_ALL, // status >= FINISHED_ALL is final status. - TIMEOUT, - PERMISSION_CHECK_FAILED, - COMM_ABNORMAL, - SECURITY_OPTION_CHECK_FAILURE, // remote device's SecurityOption not equal to local - EKEYREVOKED_FAILURE, // EKEYREVOKED error - BUSY_FAILURE, - SCHEMA_INCOMPATIBLE, - FAILED - }; - - enum SyncMode { - PUSH, - PULL, - PUSH_AND_PULL, - AUTO_PUSH, - AUTO_PULL, - RESPONSE_PULL, - INVALID + OP_WAITING = 0, + OP_SYNCING, + OP_SEND_FINISHED, + OP_RECV_FINISHED, + OP_FINISHED_ALL, // status >= OP_FINISHED_ALL is final status. + OP_FAILED, + OP_TIMEOUT, + OP_PERMISSION_CHECK_FAILED, + OP_COMM_ABNORMAL, + OP_SECURITY_OPTION_CHECK_FAILURE, // remote device's SecurityOption not equal to local + OP_EKEYREVOKED_FAILURE, // EKEYREVOKED error + OP_BUSY_FAILURE, + OP_SCHEMA_INCOMPATIBLE, + OP_QUERY_FORMAT_FAILURE, + OP_QUERY_FIELD_FAILURE, + OP_NOT_SUPPORT, + OP_INTERCEPT_DATA_FAIL, + OP_MAX_LIMITS, + OP_INVALID_ARGS }; using UserCallback = std::function)>; @@ -78,6 +76,9 @@ public: // Set the sync status, running or finished void SetStatus(const std::string &deviceId, int status); + // Set the identifier, used in SyncOperation::Finished + void SetIdentifier(const std::vector &identifier); + // Get the sync status, running or finished int GetStatus(const std::string &deviceId) const; @@ -105,9 +106,20 @@ public: // Return if this sync is block sync bool IsBlockSync() const; + // Return if this sync is AUTO_SUBSCRIBE_QUERY + bool IsAutoControlCmd() const; + // Check if All devices sync finished. bool CheckIsAllFinished() const; + // For query sync + void SetQuery(const QuerySyncObject &query); + QuerySyncObject GetQuery() const; + bool IsQuerySync() const; + std::string GetQueryId() const; + static SyncType GetSyncType(int mode); + static int TransferSyncMode(int mode); + protected: virtual ~SyncOperation(); @@ -117,6 +129,9 @@ private: // called by destruction void Finalize(); + // Transfer sync mode from interface to inner + void TransferQuerySyncMode(); + // The device list const std::vector devices_; @@ -149,6 +164,14 @@ private: // Used for block sync std::unique_ptr semaphore_; + + QuerySyncObject query_; + bool isQuerySync_; + + bool isAutoSubscribe_; + + // record identifier used to call ScheduleQueuedTask in SyncOperation::Finished + std::string identifier_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp index 911f698d242f61ba7e1a108b222e6928181a72a9..413d37543a0d6bbc094d8c1d9ea774abc34e8e48 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.cpp @@ -15,7 +15,6 @@ #include "sync_state_machine.h" -#include #include #include "log_print.h" @@ -47,7 +46,7 @@ SyncStateMachine::~SyncStateMachine() } } -int SyncStateMachine::Initialize(ISyncTaskContext *context, IKvDBSyncInterface *syncInterface, +int SyncStateMachine::Initialize(ISyncTaskContext *context, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) { if ((context == nullptr) || (syncInterface == nullptr) || (metadata == nullptr) || (communicator == nullptr)) { @@ -85,7 +84,7 @@ int SyncStateMachine::TimeoutCallback(TimerId timerId) } int retryTime = syncContext_->GetRetryTime(); - if (retryTime >= RETRY_TIME || !syncContext_->IsAutoSync()) { + if (retryTime >= syncContext_->GetSyncRetryTimes() || !syncContext_->IsSyncTaskNeedRetry()) { LOGI("[SyncStateMachine][Timeout] TimeoutCallback retryTime:%d", retryTime); syncContext_->UnlockObj(); StepToTimeout(); @@ -99,9 +98,7 @@ int SyncStateMachine::TimeoutCallback(TimerId timerId) syncContext_->IncSequenceId(); } syncContext_->SetRetryStatus(SyncTaskContext::NEED_RETRY); - int timeoutTime = syncContext_->GetTimeoutTime(); - // set the new timeout value with 2 raised to the power of retryTime. - timeoutTime = timeoutTime * static_cast(pow(2, retryTime)); + int timeoutTime = syncContext_->GetSyncRetryTimeout(retryTime); syncContext_->ModifyTimer(timeoutTime); LOGI("[SyncStateMachine][Timeout] Schedule task, timeoutTime = %d, retryTime = %d", timeoutTime, retryTime); SyncStep(); @@ -148,7 +145,7 @@ int SyncStateMachine::SwitchMachineState(uint8_t event) const std::map &table = (*tableIter).switchTable; auto eventToStateIter = table.find(currentState_); if (eventToStateIter == table.end()) { - LOGE("[SyncStateMachine][SwitchState] ver:%d, Can't find EventToState with currentSate %u", + LOGE("[SyncStateMachine][SwitchState] tableVer:%d, Can't find EventToState with currentSate %u", (*tableIter).version, currentState_); SetCurStateErrStatus(); return E_OK; @@ -157,13 +154,13 @@ int SyncStateMachine::SwitchMachineState(uint8_t event) const EventToState &eventToState = eventToStateIter->second; auto stateIter = eventToState.find(event); if (stateIter == eventToState.end()) { - LOGD("[SyncStateMachine][SwitchState] ver:%d, Can't find event %u int currentSate %u ignore", + LOGD("[SyncStateMachine][SwitchState] tableVer:%d, Can't find event %u int currentSate %u ignore", (*tableIter).version, event, currentState_); return -E_NOT_FOUND; } currentState_ = stateIter->second; - LOGD("[SyncStateMachine][SwitchState] ver:%d, from state %u move to state %u with event %u dev %s{private}", + LOGD("[SyncStateMachine][SwitchState] tableVer:%d, from state %u move to state %u with event %u dev %s{private}", (*tableIter).version, eventToStateIter->first, currentState_, event, syncContext_->GetDeviceId().c_str()); return E_OK; } @@ -177,19 +174,25 @@ void SyncStateMachine::SwitchStateAndStep(uint8_t event) int SyncStateMachine::ExecNextTask() { - if (syncContext_->IsTargetQueueEmpty()) { - syncContext_->SetTaskExecStatus(ISyncTaskContext::FINISHED); - syncContext_->Clear(); - LOGD("[SyncStateMachine] All sync task finished!"); - return -E_NO_SYNC_TASK; - } - syncContext_->MoveToNextTarget(); - int errCode = PrepareNextSyncTask(); - if (errCode != E_OK) { - LOGE("[SyncStateMachine] PrepareSync failed"); - syncContext_->SetOperationStatus(SyncOperation::FAILED); + while (!syncContext_->IsTargetQueueEmpty()) { + syncContext_->MoveToNextTarget(); + if (syncContext_->IsCurrentSyncTaskCanBeSkipped()) { + syncContext_->SetOperationStatus(SyncOperation::OP_FINISHED_ALL); + syncContext_->SetTaskExecStatus(ISyncTaskContext::FINISHED); + continue; + } + int errCode = PrepareNextSyncTask(); + if (errCode != E_OK) { + LOGE("[SyncStateMachine] PrepareSync failed"); + syncContext_->SetOperationStatus(SyncOperation::OP_FAILED); + } + return errCode; } - return errCode; + // no task left + syncContext_->SetTaskExecStatus(ISyncTaskContext::FINISHED); + syncContext_->Clear(); + LOGD("[SyncStateMachine] All sync task finished!"); + return -E_NO_SYNC_TASK; } int SyncStateMachine::StartWatchDog() @@ -219,7 +222,7 @@ void SyncStateMachine::StopWatchDog() syncContext_->StopTimer(); } -bool SyncStateMachine::StartSaveDataNotify(uint32_t sessionId, uint32_t sequenceId) +bool SyncStateMachine::StartSaveDataNotify(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) { std::lock_guard lockGuard(saveDataNotifyLock_); if (saveDataNotifyTimerId_ > 0) { @@ -232,7 +235,7 @@ bool SyncStateMachine::StartSaveDataNotify(uint32_t sessionId, uint32_t sequence RefObject::IncObjRef(syncContext_); int errCode = RuntimeContext::GetInstance()->SetTimer( SAVE_DATA_NOTIFY_INTERVAL, - [this, sessionId, sequenceId](TimerId timerId) { + [this, sessionId, sequenceId, inMsgId](TimerId timerId) { { std::lock_guard lock(stateMachineLock_); (void)ResetWatchDog(); @@ -242,14 +245,14 @@ bool SyncStateMachine::StartSaveDataNotify(uint32_t sessionId, uint32_t sequence StopSaveDataNotifyNoLock(); return E_OK; } - SendSaveDataNotifyPacket(sessionId, sequenceId); + SendSaveDataNotifyPacket(sessionId, sequenceId, inMsgId); saveDataNotifyCount_++; return E_OK; }, [this]() { - int errCode = RuntimeContext::GetInstance()->ScheduleTask([this](){ RefObject::DecObjRef(syncContext_); }); - if (errCode != E_OK) { - LOGE("[SyncStateMachine] [SaveDataNotify] timer finalizer ScheduleTask, errCode %d", errCode); + int ret = RuntimeContext::GetInstance()->ScheduleTask([this](){ RefObject::DecObjRef(syncContext_); }); + if (ret != E_OK) { + LOGE("[SyncStateMachine] [SaveDataNotify] timer finalizer ScheduleTask, errCode %d", ret); } }, saveDataNotifyTimerId_); @@ -283,39 +286,50 @@ bool SyncStateMachine::StartFeedDogForSync(uint32_t time, SyncDirectionFlag flag LOGE("[SyncStateMachine][feedDog] start wrong flag:%d", flag); return false; } + + int cnt = GetFeedDogTimeout(time / SAVE_DATA_NOTIFY_INTERVAL); + LOGI("[SyncStateMachine][feedDog] start cnt:%d, flag:%d", cnt, flag); + std::lock_guard lockGuard(feedDogLock_[flag]); - if (feedDogTimerId_[flag] > 0) { - feedDogCount_[flag] = 0; + watchDogController_[flag].refCount++; + LOGD("af incr refCount = %d", watchDogController_[flag].refCount); + + if (watchDogController_[flag].feedDogTimerId > 0) { + // update the upperLimit, if the new cnt is bigger then last upperLimit + if (cnt > watchDogController_[flag].feedDogUpperLimit) { + LOGD("update feedDogUpperLimit = %d", cnt); + watchDogController_[flag].feedDogUpperLimit = cnt; + } + watchDogController_[flag].feedDogCnt = 0; LOGW("[SyncStateMachine][feedDog] timer has been started!, flag:%d", flag); return false; } - int cnt = time / SAVE_DATA_NOTIFY_INTERVAL; - LOGI("[SyncStateMachine][feedDog] start cnt:%d, flag:%d", cnt, flag); // Incref to make sure context still alive before timer stopped. RefObject::IncObjRef(syncContext_); + watchDogController_[flag].feedDogUpperLimit = cnt; int errCode = RuntimeContext::GetInstance()->SetTimer( SAVE_DATA_NOTIFY_INTERVAL, - [this, flag, cnt](TimerId timerId) { + [this, flag](TimerId timerId) { { std::lock_guard lock(stateMachineLock_); (void)ResetWatchDog(); } std::lock_guard innerLock(feedDogLock_[flag]); - if (feedDogCount_[flag] >= cnt) { + if (watchDogController_[flag].feedDogCnt >= watchDogController_[flag].feedDogUpperLimit) { StopFeedDogForSyncNoLock(flag); return E_OK; } - feedDogCount_[flag]++; + watchDogController_[flag].feedDogCnt++; return E_OK; }, [this]() { - int errCode = RuntimeContext::GetInstance()->ScheduleTask([this](){ RefObject::DecObjRef(syncContext_); }); - if (errCode != E_OK) { - LOGE("[SyncStateMachine] [feedDog] timer finalizer ScheduleTask, errCode %d", errCode); + int ret = RuntimeContext::GetInstance()->ScheduleTask([this](){ RefObject::DecObjRef(syncContext_); }); + if (ret != E_OK) { + LOGE("[SyncStateMachine] [feedDog] timer finalizer ScheduleTask, errCode %d", ret); } }, - feedDogTimerId_[flag]); + watchDogController_[flag].feedDogTimerId); if (errCode != E_OK) { LOGW("[SyncStateMachine][feedDog] start timer failed err %d !", errCode); return false; @@ -323,6 +337,14 @@ bool SyncStateMachine::StartFeedDogForSync(uint32_t time, SyncDirectionFlag flag return true; } +uint8_t SyncStateMachine::GetFeedDogTimeout(int timeoutCount) const +{ + if (timeoutCount > UINT8_MAX) { + return UINT8_MAX; + } + return timeoutCount; +} + void SyncStateMachine::StopFeedDogForSync(SyncDirectionFlag flag) { if (flag != SyncDirectionFlag::SEND && flag != SyncDirectionFlag::RECEIVE) { @@ -339,16 +361,30 @@ void SyncStateMachine::StopFeedDogForSyncNoLock(SyncDirectionFlag flag) LOGE("[SyncStateMachine][feedDog] stop wrong flag:%d", flag); return; } - if (feedDogTimerId_[flag] == 0) { + if (watchDogController_[flag].feedDogTimerId == 0) { return; } LOGI("[SyncStateMachine][feedDog] stop flag:%d", flag); - RuntimeContext::GetInstance()->RemoveTimer(feedDogTimerId_[flag]); - feedDogTimerId_[flag] = 0; - feedDogCount_[flag] = 0; + RuntimeContext::GetInstance()->RemoveTimer(watchDogController_[flag].feedDogTimerId); + watchDogController_[flag].feedDogTimerId = 0; + watchDogController_[flag].feedDogCnt = 0; + watchDogController_[flag].refCount = 0; } void SyncStateMachine::SetCurStateErrStatus() { } + +void SyncStateMachine::DecRefCountOfFeedDogTimer(SyncDirectionFlag flag) +{ + std::lock_guard lockGuard(feedDogLock_[flag]); + if (watchDogController_[flag].feedDogTimerId == 0) { + return; + } + if (--watchDogController_[flag].refCount <= 0) { + LOGD("stop feed dog timer, refcount = %d", watchDogController_[flag].refCount); + StopFeedDogForSyncNoLock(flag); + } + LOGD("af dec refcount = %d", watchDogController_[flag].refCount); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.h index 8d6c2066a6d34485351ef85bd3c15428c42c6425..da86fceb89ffe46b6f204190d22fb38c0802f5c7 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_state_machine.h @@ -18,6 +18,7 @@ #include +#include "isync_interface.h" #include "isync_state_machine.h" namespace DistributedDB { @@ -30,13 +31,22 @@ struct StateSwitchTable { std::map switchTable; // the 1st uint8_t is current state }; +struct WatchDogController { + TimerId feedDogTimerId = 0; + uint8_t feedDogCnt = 0; + uint8_t feedDogUpperLimit = 0; + /* this variable will +1 when call StartFeedDogForSync, -1 when recv one ack, + when it become <= 0, we stop the watch dog. */ + int refCount = 0; +}; + class SyncStateMachine : public ISyncStateMachine { public: SyncStateMachine(); - virtual ~SyncStateMachine(); + ~SyncStateMachine() override; // Init the SingleVerSyncStateMachine - int Initialize(ISyncTaskContext *context, IKvDBSyncInterface *syncInterface, std::shared_ptr &metadata, + int Initialize(ISyncTaskContext *context, ISyncInterface *syncInterface, std::shared_ptr &metadata, ICommunicator *communicator) override; // start a sync step @@ -51,6 +61,8 @@ public: // start a timer to ResetWatchDog when sync data one (key,value) size bigger than mtu bool StartFeedDogForSync(uint32_t time, SyncDirectionFlag flag) override; + uint8_t GetFeedDogTimeout(int timeoutCount) const; + // stop timer to ResetWatchDog when sync data one (key,value) size bigger than mtu void StopFeedDogForSync(SyncDirectionFlag flag) override; protected: @@ -83,7 +95,7 @@ protected: virtual int PrepareNextSyncTask() = 0; // Called by StartSaveDataNotifyTimer, Sub class should realize this function to send a heartbeet packet - virtual void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId) = 0; + virtual void SendSaveDataNotifyPacket(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId) = 0; // Used to parse state table to switch machine state, this function must be called in stateMachineLock int SwitchMachineState(uint8_t event); @@ -104,7 +116,7 @@ protected: void StopWatchDog(); // Start a timer to send data notify packet to keep remote device not timeout - bool StartSaveDataNotify(uint32_t sessionId, uint32_t sequenceId); + bool StartSaveDataNotify(uint32_t sessionId, uint32_t sequenceId, uint32_t inMsgId); // Stop send save data notify void StopSaveDataNotify(); @@ -115,10 +127,12 @@ protected: // stop a timer to ResetWatchDog when sync data bigger than mtu without lock void StopFeedDogForSyncNoLock(SyncDirectionFlag flag); + void DecRefCountOfFeedDogTimer(SyncDirectionFlag flag); + DISABLE_COPY_ASSIGN_MOVE(SyncStateMachine); ISyncTaskContext *syncContext_; - IKvDBSyncInterface *storageInterface_; + ISyncInterface *storageInterface_; ICommunicator *communicator_; std::shared_ptr metadata_; std::mutex stateMachineLock_; @@ -136,8 +150,7 @@ protected: // used for one (key,value) bigger than mtu size, in this case, send packet need more longger time std::mutex feedDogLock_[SYNC_DIRECTION_NUM]; - TimerId feedDogTimerId_[SYNC_DIRECTION_NUM] = {0, 0}; - uint8_t feedDogCount_[SYNC_DIRECTION_NUM] = {0, 0}; + WatchDogController watchDogController_[SYNC_DIRECTION_NUM] = {{0}, {0}}; }; } // namespace DistributedDB #endif // SYNC_STATE_MACHINE_H diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.cpp index d6d350addfa0aab3a45943932b61224a15e8bc1c..68343ba16ecbc5975548b59224b67e92dfb7e7c4 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.cpp @@ -78,5 +78,10 @@ bool SyncTarget::IsAutoSync() const } return operation_->IsAutoSync(); } + +uint32_t SyncTarget::GetResponseSessionId() const +{ + return 0; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.h index 93c01f1173aa08d98ac566f19ad9205048c2e7f5..23efcd763dd91db4bb526265764721b24db9e3fd 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_target.h @@ -22,7 +22,7 @@ namespace DistributedDB { class SyncTarget : public ISyncTarget { public: SyncTarget() : operation_(nullptr), taskType_(0), mode_(0) {}; - virtual ~SyncTarget(); + ~SyncTarget() override; // Get the Sync Id of this task int GetSyncId() const override; @@ -39,7 +39,7 @@ public: // Get the mode of this task request or response int GetMode() const override; - // Set a Sync Status, it will increase the ref of operation + // Set a SyncOperation void SetSyncOperation(SyncOperation *operation) override; // Get a SyncOperation @@ -48,6 +48,8 @@ public: // Is this target is a auto sync bool IsAutoSync() const override; + uint32_t GetResponseSessionId() const override; + protected: SyncOperation *operation_; int taskType_; // sync task or response task; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp index e2a7b940df32997b813e26c0b7d84c7a62ec35ea..099a76277aa74bbe22a2f4ccc010534f62931592 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.cpp @@ -16,18 +16,23 @@ #include "sync_task_context.h" #include +#include +#include "db_constant.h" #include "db_errno.h" -#include "log_print.h" -#include "isync_state_machine.h" #include "hash.h" +#include "isync_state_machine.h" +#include "log_print.h" #include "time_helper.h" -#include "db_constant.h" namespace DistributedDB { std::mutex SyncTaskContext::synTaskContextSetLock_; std::set SyncTaskContext::synTaskContextSet_; +namespace { + const int NEGOTIATION_LIMIT = 2; +} + SyncTaskContext::SyncTaskContext() : syncOperation_(nullptr), syncId_(0), @@ -43,7 +48,12 @@ SyncTaskContext::SyncTaskContext() remoteSoftwareVersion_(0), remoteSoftwareVersionId_(0), isCommNormal_(true), - taskErrCode_(E_OK) + taskErrCode_(E_OK), + syncTaskRetryStatus_(false), + isSyncRetry_(false), + negotiationCount_(0), + isAutoSubscribe_(false), + isNeedResetAbilitySync_(false) { } @@ -102,25 +112,32 @@ void SyncTaskContext::SetOperationStatus(int status) int finalStatus = status; int operationStatus = syncOperation_->GetStatus(deviceId_); - if (status == SyncOperation::SEND_FINISHED && operationStatus == SyncOperation::RECV_FINISHED) { + if (status == SyncOperation::OP_SEND_FINISHED && operationStatus == SyncOperation::OP_RECV_FINISHED) { if (GetTaskErrCode() == -E_EKEYREVOKED) { - finalStatus = SyncOperation::EKEYREVOKED_FAILURE; + finalStatus = SyncOperation::OP_EKEYREVOKED_FAILURE; } else { - finalStatus = SyncOperation::FINISHED_ALL; + finalStatus = SyncOperation::OP_FINISHED_ALL; } - } else if (status == SyncOperation::RECV_FINISHED && operationStatus == SyncOperation::SEND_FINISHED) { + } else if (status == SyncOperation::OP_RECV_FINISHED && operationStatus == SyncOperation::OP_SEND_FINISHED) { if (GetTaskErrCode() == -E_EKEYREVOKED) { - finalStatus = SyncOperation::EKEYREVOKED_FAILURE; + finalStatus = SyncOperation::OP_EKEYREVOKED_FAILURE; } else { - finalStatus = SyncOperation::FINISHED_ALL; + finalStatus = SyncOperation::OP_FINISHED_ALL; } } syncOperation_->SetStatus(deviceId_, finalStatus); + if (finalStatus >= SyncOperation::OP_FINISHED_ALL) { + SaveLastPushTaskExecStatus(finalStatus); + } if (syncOperation_->CheckIsAllFinished()) { syncOperation_->Finished(); } } +void SyncTaskContext::SaveLastPushTaskExecStatus(int finalStatus) +{ +} + void SyncTaskContext::Clear() { StopTimer(); @@ -130,10 +147,11 @@ void SyncTaskContext::Clear() isAutoSync_ = false; requestSessionId_ = 0; isNeedRetry_ = NO_NEED_RETRY; - mode_ = SyncOperation::INVALID; - status_ = SyncOperation::WAITING; + mode_ = SyncModeType::INVALID_MODE; + status_ = SyncOperation::OP_WAITING; taskErrCode_ = E_OK; packetId_ = 0; + isAutoSubscribe_ = false; } int SyncTaskContext::RemoveSyncOperation(int syncId) @@ -187,10 +205,11 @@ int SyncTaskContext::GetOperationStatus() const { std::lock_guard lock(operationLock_); if (syncOperation_ == nullptr) { - return SyncOperation::FINISHED_ALL; + return SyncOperation::OP_FINISHED_ALL; } return syncOperation_->GetStatus(deviceId_); } + void SyncTaskContext::SetMode(int mode) { mode_ = mode; @@ -200,9 +219,13 @@ int SyncTaskContext::GetMode() const { return mode_; } + void SyncTaskContext::MoveToNextTarget() { ClearSyncOperation(); + TaskParam param; + // call other system api without lock + param.timeout = communicator_->GetTimeout(deviceId_); std::lock_guard lock(targetQueueLock_); while (!requestTargetQueue_.empty() || !responseTargetQueue_.empty()) { ISyncTarget *tmpTarget = nullptr; @@ -225,7 +248,7 @@ void SyncTaskContext::MoveToNextTarget() tmpTarget = nullptr; continue; } - CopyTargetData(tmpTarget); + CopyTargetData(tmpTarget, param); delete tmpTarget; tmpTarget = nullptr; break; @@ -260,11 +283,6 @@ bool SyncTaskContext::IsAutoSync() const int SyncTaskContext::StartTimer() { - if (!isAutoSync_) { - timeout_ = DBConstant::MANUAL_SYNC_TIMEOUT; - } else { - timeout_ = DBConstant::AUTO_SYNC_TIMEOUT; - } std::lock_guard lockGuard(timerLock_); if (timerId_ > 0) { return -E_UNEXPECTED_DATA; @@ -274,9 +292,9 @@ int SyncTaskContext::StartTimer() TimerAction timeOutCallback = std::bind(&SyncTaskContext::TimeOut, this, std::placeholders::_1); int errCode = RuntimeContext::GetInstance()->SetTimer(timeout_, timeOutCallback, [this]() { - int errCode = RuntimeContext::GetInstance()->ScheduleTask([this](){ RefObject::DecObjRef(this); }); - if (errCode != E_OK) { - LOGE("[SyncTaskContext] timer finalizer ScheduleTask, errCode %d", errCode); + int ret = RuntimeContext::GetInstance()->ScheduleTask([this](){ RefObject::DecObjRef(this); }); + if (ret != E_OK) { + LOGE("[SyncTaskContext] timer finalizer ScheduleTask, errCode %d", ret); } }, timerId); if (errCode != E_OK) { @@ -488,7 +506,8 @@ void SyncTaskContext::CommErrHandlerFuncInner(int errCode, uint32_t sessionId) } if (errCode == E_OK) { - isCommNormal_ = true; + // when communicator sent message failed, the state machine will get the error and exit this sync task + // it seems unnecessary to change isCommNormal_ value, so just return here return; } @@ -511,13 +530,17 @@ int SyncTaskContext::TimeOut(TimerId id) return errCode; } -void SyncTaskContext::CopyTargetData(const ISyncTarget *target) +void SyncTaskContext::CopyTargetData(const ISyncTarget *target, const TaskParam &taskParam) { mode_ = target->GetMode(); - status_ = SyncOperation::SYNCING; + status_ = SyncOperation::OP_SYNCING; isNeedRetry_ = SyncTaskContext::NO_NEED_RETRY; taskErrCode_ = E_OK; packetId_ = 0; + isCommNormal_ = true; // reset comm status here + syncTaskRetryStatus_ = isSyncRetry_; + timeout_ = static_cast(taskParam.timeout); + negotiationCount_ = 0; target->GetSyncOperation(syncOperation_); ReSetSequenceId(); @@ -526,13 +549,18 @@ void SyncTaskContext::CopyTargetData(const ISyncTarget *target) RefObject::IncObjRef(syncOperation_); syncId_ = syncOperation_->GetSyncId(); isAutoSync_ = syncOperation_->IsAutoSync(); + isAutoSubscribe_ = syncOperation_->IsAutoControlCmd(); + if (isAutoSync_ || mode_ == SUBSCRIBE_QUERY || mode_ == UNSUBSCRIBE_QUERY) { + syncTaskRetryStatus_ = true; + } requestSessionId_ = Hash::Hash32Func(deviceId_ + std::to_string(syncId_) + std::to_string(TimeHelper::GetSysCurrentTime())); - LOGI("[SyncTaskContext][copyTarget] mode_ = %d, syncId_ = %d, isAutoSync_ = %d, dev = %s{private}", - mode_, syncId_, isAutoSync_, deviceId_.c_str()); + LOGI("[SyncTaskContext][copyTarget] mode=%d,syncId=%d,isAutoSync=%d,isRetry=%d,dev=%s{private}", + mode_, syncId_, isAutoSync_, syncTaskRetryStatus_, deviceId_.c_str()); } else { isAutoSync_ = false; - LOGI("[SyncTaskContext][copyTarget] for response data dev %s{private} ", deviceId_.c_str()); + LOGI("[SyncTaskContext][copyTarget] for response data dev %s{private},isRetry=%d", deviceId_.c_str(), + syncTaskRetryStatus_); } } @@ -574,9 +602,10 @@ void SyncTaskContext::CancelCurrentSyncRetryIfNeed(int newTargetMode) if (!isAutoSync_) { return; } - if (newTargetMode == mode_ || newTargetMode == SyncOperation::PUSH_AND_PULL) { - SetRetryTime(ISyncStateMachine::RETRY_TIME); - ModifyTimer(DBConstant::AUTO_SYNC_TIMEOUT); + int mode = SyncOperation::TransferSyncMode(newTargetMode); + if (newTargetMode == mode_ || mode == SyncModeType::PUSH_AND_PULL) { + SetRetryTime(AUTO_RETRY_TIMES); + ModifyTimer(timeout_); } } @@ -589,4 +618,75 @@ void SyncTaskContext::SetTaskErrCode(int errCode) { taskErrCode_ = errCode; } + +bool SyncTaskContext::IsSyncTaskNeedRetry() const +{ + return syncTaskRetryStatus_; +} + +void SyncTaskContext::SetSyncRetry(bool isRetry) +{ + isSyncRetry_ = isRetry; +} + +int SyncTaskContext::GetSyncRetryTimes() const +{ + if (IsAutoSync() || mode_ == SUBSCRIBE_QUERY || mode_ == UNSUBSCRIBE_QUERY) { + return AUTO_RETRY_TIMES; + } + return MANUAL_RETRY_TIMES; +} + +int SyncTaskContext::GetSyncRetryTimeout(int retryTime) const +{ + int timeoutTime = GetTimeoutTime(); + if (IsAutoSync()) { + // set the new timeout value with 2 raised to the power of retryTime. + return timeoutTime * static_cast(pow(2, retryTime)); + } + return timeoutTime; +} + +void SyncTaskContext::ClearAllSyncTask() +{ +} + +bool SyncTaskContext::IsAutoLiftWaterMark() const +{ + return negotiationCount_ < NEGOTIATION_LIMIT; +} + +void SyncTaskContext::IncNegotiationCount() +{ + negotiationCount_++; +} + +bool SyncTaskContext::IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query) +{ + return stateMachine_->IsNeedTriggerQueryAutoSync(inMsg, query); +} + +bool SyncTaskContext::IsAutoSubscribe() const +{ + return isAutoSubscribe_; +} + +bool SyncTaskContext::GetIsNeedResetAbilitySync() const +{ + return isNeedResetAbilitySync_; +} + +void SyncTaskContext::SetIsNeedResetAbilitySync(bool isNeedReset) +{ + isNeedResetAbilitySync_ = isNeedReset; +} + +bool SyncTaskContext::IsCurrentSyncTaskCanBeSkipped() const +{ + return false; +} + +void SyncTaskContext::ResetLastPushTaskStatus() +{ +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.h index 3a542cafe94970b405dd3c3c97396c094994ef8c..6efd23ad652a88691c7d0c6c841fdf236f77dd57 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_task_context.h @@ -19,14 +19,14 @@ #include #include -#include "isync_task_context.h" -#include "sync_target.h" -#include "semaphore_utils.h" -#include "sync_operation.h" #include "icommunicator.h" #include "ikvdb_sync_interface.h" +#include "isync_task_context.h" #include "meta_data.h" #include "runtime_context.h" +#include "semaphore_utils.h" +#include "sync_operation.h" +#include "sync_target.h" #include "time_helper.h" namespace DistributedDB { @@ -34,7 +34,9 @@ enum SyncDirectionFlag { SEND = 0, RECEIVE = 1, }; - +struct TaskParam { + uint32_t timeout = 0; +}; class ISyncStateMachine; class SyncTaskContext : public ISyncTaskContext { @@ -174,6 +176,32 @@ public: void SetTaskErrCode(int errCode) override; + bool IsSyncTaskNeedRetry() const override; + + void SetSyncRetry(bool isRetry) override; + + int GetSyncRetryTimes() const override; + + int GetSyncRetryTimeout(int retryTime) const override; + + void ClearAllSyncTask() override; + + bool IsAutoLiftWaterMark() const override; + + void IncNegotiationCount() override; + + // check if need trigger query auto sync and get query from inMsg + bool IsNeedTriggerQueryAutoSync(Message *inMsg, QuerySyncObject &query) override; + + bool IsAutoSubscribe() const override; + + bool IsCurrentSyncTaskCanBeSkipped() const override; + + virtual void ResetLastPushTaskStatus(); + + bool GetIsNeedResetAbilitySync() const; + void SetIsNeedResetAbilitySync(bool isNeedReset) override; + protected: const static int KILL_WAIT_SECONDS = INT32_MAX; @@ -181,7 +209,7 @@ protected: virtual int TimeOut(TimerId id); - virtual void CopyTargetData(const ISyncTarget *target); + virtual void CopyTargetData(const ISyncTarget *target, const TaskParam &taskParam); void CommErrHandlerFuncInner(int errCode, uint32_t sessionId); @@ -193,6 +221,8 @@ protected: void CancelCurrentSyncRetryIfNeed(int newTargetMode); + virtual void SaveLastPushTaskExecStatus(int finalStatus); + mutable std::mutex targetQueueLock_; std::list requestTargetQueue_; std::list responseTargetQueue_; @@ -204,7 +234,7 @@ protected: int status_; int taskExecStatus_; std::string deviceId_; - IKvDBSyncInterface *syncInterface_; + ISyncInterface *syncInterface_; ICommunicator *communicator_; ISyncStateMachine *stateMachine_; TimeOffset timeOffset_ = 0; @@ -233,8 +263,14 @@ protected: bool isCommNormal_; int taskErrCode_; uint64_t packetId_ = 0; // used for assignment to reSendMap_.ReSendInfo.packetId in 103 version or above - - // For gloable ISyncTaskContext Set, used by CommErrCallback. + bool syncTaskRetryStatus_; + bool isSyncRetry_; + uint32_t negotiationCount_; + bool isAutoSubscribe_; + // syncFinished_ need to set false if isNeedResetSyncFinished_ is true when start do abilitySync interface + std::atomic isNeedResetAbilitySync_; + + // For global ISyncTaskContext Set, used by CommErrCallback. static std::mutex synTaskContextSetLock_; static std::set synTaskContextSet_; }; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/sync_types.h b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_types.h new file mode 100644 index 0000000000000000000000000000000000000000..9dddff22c1d19304cb8d9e03983ccb1b1e333861 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/sync_types.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SYNC_TYPES_H +#define SYNC_TYPES_H + +#include +#include "query_sync_object.h" +#include "sync_config.h" + +namespace DistributedDB { +enum MessageId { + TIME_SYNC_MESSAGE = 1, + DATA_SYNC_MESSAGE, + COMMIT_HISTORY_SYNC_MESSAGE, + MULTI_VER_DATA_SYNC_MESSAGE, + VALUE_SLICE_SYNC_MESSAGE, + LOCAL_DATA_CHANGED, + ABILITY_SYNC_MESSAGE, + QUERY_SYNC_MESSAGE, + CONTROL_SYNC_MESSAGE, + UNKNOW_MESSAGE, +}; + +enum SyncModeType { + PUSH, + PULL, + PUSH_AND_PULL, + AUTO_PUSH, + AUTO_PULL, + RESPONSE_PULL, + QUERY_PUSH, + QUERY_PULL, + QUERY_PUSH_PULL, + SUBSCRIBE_QUERY, + UNSUBSCRIBE_QUERY, + AUTO_SUBSCRIBE_QUERY, + INVALID_MODE +}; + +enum class SyncType { + MANUAL_FULL_SYNC_TYPE = 1, + AUTO_SYNC_TYPE, + QUERY_SYNC_TYPE, + INVALID_SYNC_TYPE, +}; + +enum ControlCmdType { + SUBSCRIBE_QUERY_CMD, + UNSUBSCRIBE_QUERY_CMD, + INVALID_CONTROL_CMD, +}; + +struct UpdateWaterMark { + bool normalUpdateMark = false; + bool deleteUpdateMark = false; +}; + +struct InternalSyncParma { + std::vector devices; + int mode = 0; + bool isQuerySync = false; + QuerySyncObject syncQuery; +}; + +constexpr uint32_t SEND_TIME_OUT = 3000; // 3s +constexpr int NOT_SURPPORT_SEC_CLASSIFICATION = 0xff; +constexpr uint8_t QUERY_SYNC_MODE_BASE = SyncModeType::QUERY_PUSH; +constexpr int AUTO_RETRY_TIMES = 3; +constexpr int MANUAL_RETRY_TIMES = 1; +constexpr int TIME_SYNC_WAIT_TIME = 5000; // 5s +constexpr uint64_t MAX_PACKETID = 10000000000; // max packetId +constexpr int NOTIFY_MIN_MTU_SIZE = 30 * 1024; // 30k + +constexpr int MAX_SUBSCRIBE_NUM_PER_DEV = 4; +constexpr int MAX_SUBSCRIBE_NUM_PER_DB = 8; +constexpr int MAX_DEVICES_NUM = 32; + +// index 0 for packetId in data request +// if ack reserve size is 1, reserve is {localWaterMark} +// if ack reserve size is above 2, reserve is {localWaterMark, packetId, deletedWaterMark...} +constexpr uint32_t REQUEST_PACKET_RESERVED_INDEX_PACKETID = 0; +constexpr uint32_t ACK_PACKET_RESERVED_INDEX_PACKETID = 1; // index 1 for packetId +constexpr uint32_t ACK_PACKET_RESERVED_INDEX_LOCAL_WATER_MARK = 0; // index 0 for localWaterMark +constexpr uint32_t ACK_PACKET_RESERVED_INDEX_DELETE_WATER_MARK = 2; // index 2 for deleteDataWaterMark +constexpr uint64_t MAX_TIMESTAMP = INT64_MAX; +constexpr uint8_t REMOVE_DEVICE_DATA_MARK = 1; +constexpr uint8_t SUPPORT_MARK = 1; // used for set is support one ability +} + +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_factory.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_factory.cpp index bfc3b2af29a3c0466303b5710d57a343f597ce11..b0c8d9d4989db497217b63ce012e8a942a721202 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_factory.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_factory.cpp @@ -15,20 +15,28 @@ #include "syncer_factory.h" -#include "single_ver_syncer.h" -#include "multi_ver_syncer.h" #include "ikvdb_sync_interface.h" +#include "multi_ver_syncer.h" +#include "single_ver_kv_syncer.h" +#ifdef RELATIONAL_STORE +#include "single_ver_relational_syncer.h" +#endif namespace DistributedDB { std::shared_ptr SyncerFactory::GetSyncer(int type) { - if (type == IKvDBSyncInterface::SYNC_SVD) { - std::shared_ptr singleVerSyncer(std::make_shared()); + if (type == ISyncInterface::SYNC_SVD) { + std::shared_ptr singleVerSyncer(std::make_shared()); return singleVerSyncer; #ifndef OMIT_MULTI_VER - } else if (type == IKvDBSyncInterface::SYNC_MVD) { + } else if (type == ISyncInterface::SYNC_MVD) { std::shared_ptr multiVerSyncer(std::make_shared()); return multiVerSyncer; +#endif +#ifdef RELATIONAL_STORE + } else if (type == ISyncInterface::SYNC_RELATION) { + std::shared_ptr relationalSyncer(std::make_shared()); + return relationalSyncer; #endif } else { return nullptr; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/syncer_factory.h b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_factory.h similarity index 100% rename from services/distributeddataservice/libs/distributeddb/syncer/include/syncer_factory.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/syncer_factory.h diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp index 365ae3a8f1682935ea83e60e4df786f032b4eb88..3679cd30cec55ca561005be810949cafa1542b9c 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/syncer_proxy.cpp @@ -25,7 +25,7 @@ SyncerProxy::SyncerProxy() { } -int SyncerProxy::Initialize(IKvDBSyncInterface *syncInterface) +int SyncerProxy::Initialize(ISyncInterface *syncInterface) { if (syncInterface == nullptr) { return -E_INVALID_ARGS; @@ -56,7 +56,7 @@ int SyncerProxy::Close() int SyncerProxy::Sync(const std::vector &devices, int mode, const std::function &)> &onComplete, - const std::function &onFinalize, bool wait = false) + const std::function &onFinalize, bool wait) { if (syncer_ == nullptr) { return -E_NOT_INIT; @@ -64,6 +64,14 @@ int SyncerProxy::Sync(const std::vector &devices, int mode, return syncer_->Sync(devices, mode, onComplete, onFinalize, wait); } +int SyncerProxy::Sync(const SyncParma &parma) +{ + if (syncer_ == nullptr) { + return -E_NOT_INIT; + } + return syncer_->Sync(parma); +} + int SyncerProxy::RemoveSyncOperation(int syncId) { if (syncer_ == nullptr) { @@ -72,10 +80,18 @@ int SyncerProxy::RemoveSyncOperation(int syncId) return syncer_->RemoveSyncOperation(syncId); } +int SyncerProxy::StopSync() +{ + if (syncer_ == nullptr) { + return -E_NOT_INIT; + } + return syncer_->StopSync(); +} + uint64_t SyncerProxy::GetTimeStamp() { if (syncer_ == nullptr) { - return SyncerFactory::GetSyncer(IKvDBSyncInterface::SYNC_SVD)->GetTimeStamp(); + return SyncerFactory::GetSyncer(ISyncInterface::SYNC_SVD)->GetTimeStamp(); } return syncer_->GetTimeStamp(); } @@ -91,7 +107,7 @@ void SyncerProxy::EnableAutoSync(bool enable) int SyncerProxy::EraseDeviceWaterMark(const std::string &deviceId, bool isNeedHash) { if (syncer_ == nullptr) { - return SyncerFactory::GetSyncer(IKvDBSyncInterface::SYNC_SVD)->EraseDeviceWaterMark(deviceId, isNeedHash); + return SyncerFactory::GetSyncer(ISyncInterface::SYNC_SVD)->EraseDeviceWaterMark(deviceId, isNeedHash); } return syncer_->EraseDeviceWaterMark(deviceId, isNeedHash); } @@ -159,4 +175,20 @@ int SyncerProxy::SetStaleDataWipePolicy(WipePolicy policy) } return syncer_->SetStaleDataWipePolicy(policy); } + +int SyncerProxy::SetSyncRetry(bool isRetry) +{ + if (syncer_ == nullptr) { + return -E_NOT_INIT; + } + return syncer_->SetSyncRetry(isRetry); +} + +int SyncerProxy::SetEqualIdentifier(const std::string &identifier, const std::vector &targets) +{ + if (syncer_ == nullptr) { + return -E_NOT_INIT; + } + return syncer_->SetEqualIdentifier(identifier, targets); +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.cpp index fc97aea3db16153ec67e3914b552f2b6dab81569..9841961b19f94cc3ef6261d7aeb5ac20d0bb79f4 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.cpp @@ -58,7 +58,7 @@ TimeHelper::~TimeHelper() storage_ = nullptr; } -int TimeHelper::Initialize(const IKvDBSyncInterface *inStorage, std::shared_ptr &inMetadata) +int TimeHelper::Initialize(const ISyncInterface *inStorage, std::shared_ptr &inMetadata) { if ((inStorage == nullptr) || (inMetadata == nullptr)) { return -E_INVALID_ARGS; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/time_helper.h b/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.h old mode 100755 new mode 100644 similarity index 92% rename from services/distributeddataservice/libs/distributeddb/syncer/include/time_helper.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.h index 357f7d73ef14334fd62bd24bb1427384e700ffad..8baa05019e4047fcf070274f2bb12c2afa79a251 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/time_helper.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/time_helper.h @@ -20,7 +20,6 @@ #include "meta_data.h" #include "runtime_context.h" -#include "ref_object.h" namespace DistributedDB { class TimeHelper { @@ -42,7 +41,7 @@ public: ~TimeHelper(); // Init the TimeHelper - int Initialize(const IKvDBSyncInterface *inStorage, std::shared_ptr &inMetadata); + int Initialize(const ISyncInterface *inStorage, std::shared_ptr &inMetadata); // Get TimeStamp when write data into db, export interface; TimeStamp GetTime(); @@ -61,7 +60,7 @@ private: static TimeStamp lastSystemTimeUs_; static TimeStamp currentIncCount_; static const uint64_t MAX_INC_COUNT = 9; // last bit from 0-9 - const IKvDBSyncInterface *storage_; + const ISyncInterface *storage_; std::shared_ptr metadata_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.cpp b/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.cpp index adf1348e849eba4f795fda9f45255383185eba55..2a4e778bb5dcce0ba42b144369107d1376683d7c 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.cpp +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.cpp @@ -29,7 +29,6 @@ namespace { constexpr uint64_t TIME_SYNC_INTERVAL = 24 * 60 * 60 * 1000; // 24h constexpr int TRIP_DIV_HALF = 2; constexpr int64_t MAX_TIME_OFFSET_NOISE = 1 * 1000 * 10000; // 1s for 100ns - constexpr int TIME_SYNC_WAIT_TIME = 5; // 5s } // Class TimeSyncPacket @@ -151,7 +150,7 @@ int TimeSync::RegisterTransformFunc() } int TimeSync::Initialize(ICommunicator *communicator, std::shared_ptr &metadata, - const IKvDBSyncInterface *storage, const DeviceID &deviceId) + const ISyncInterface *storage, const DeviceID &deviceId) { if ((communicator == nullptr) || (storage == nullptr) || (metadata == nullptr)) { return -E_INVALID_ARGS; @@ -191,7 +190,7 @@ void TimeSync::Finalize() LOGD("[TimeSync] Finalized!"); } -int TimeSync::SyncStart(const CommErrHandler &handler) +int TimeSync::SyncStart(const CommErrHandler &handler, uint32_t sessionId) { isOnline_ = true; TimeSyncPacket packet; @@ -204,7 +203,7 @@ int TimeSync::SyncStart(const CommErrHandler &handler) if (message == nullptr) { return -E_OUT_OF_MEMORY; } - + message->SetSessionId(sessionId); message->SetMessageType(TYPE_REQUEST); message->SetPriority(Priority::NORMAL); int errCode = message->SetCopiedObject<>(packet); @@ -313,12 +312,17 @@ int TimeSync::DeSerialization(const uint8_t *buffer, uint32_t length, Message *i return inMsg->SetCopiedObject<>(packet); } -int TimeSync::AckRecv(const Message *message) +int TimeSync::AckRecv(const Message *message, uint32_t targetSessionId) { + // only check when sessionId is not 0, because old version timesync sessionId is 0. + if (message != nullptr && message->GetSessionId() != 0 && + message->GetErrorNo() == E_FEEDBACK_COMMUNICATOR_NOT_FOUND && message->GetSessionId() == targetSessionId) { + LOGE("[AbilitySync][AckMsgCheck] Remote db is closed"); + return -E_FEEDBACK_COMMUNICATOR_NOT_FOUND; + } if (!IsPacketValid(message, TYPE_RESPONSE)) { return -E_INVALID_ARGS; } - const TimeSyncPacket *packet = message->GetObject(); if (packet == nullptr) { LOGE("[TimeSync] AckRecv packet is null"); @@ -397,6 +401,7 @@ int TimeSync::RequestRecv(const Message *message) if (ackMessage == nullptr) { return -E_OUT_OF_MEMORY; } + ackMessage->SetSessionId(message->GetSessionId()); ackMessage->SetPriority(Priority::NORMAL); ackMessage->SetMessageType(TYPE_RESPONSE); ackMessage->SetTarget(deviceId_); @@ -483,7 +488,7 @@ int TimeSync::TimeSyncDriver(TimerId timerId) return E_OK; } -int TimeSync::GetTimeOffset(TimeOffset &outOffset) +int TimeSync::GetTimeOffset(TimeOffset &outOffset, uint32_t timeout, uint32_t sessionId) { if (!isSynced_) { { @@ -491,16 +496,18 @@ int TimeSync::GetTimeOffset(TimeOffset &outOffset) isAckReceived_ = false; } CommErrHandler handler = std::bind(&TimeSync::CommErrHandlerFunc, std::placeholders::_1, this); - int errCode = SyncStart(handler); - LOGD("TimeSync::GetTimeOffset start, current = %llu, errCode:%d", TimeHelper::GetSysCurrentTime(), errCode); + int errCode = SyncStart(handler, sessionId); + LOGD("TimeSync::GetTimeOffset start, current time = %llu, errCode = %d，timeout = %u ms", + TimeHelper::GetSysCurrentTime(), errCode, timeout); std::unique_lock lock(cvLock_); - if (errCode != E_OK || !conditionVar_.wait_for(lock, std::chrono::seconds(TIME_SYNC_WAIT_TIME), + if (errCode != E_OK || !conditionVar_.wait_for(lock, std::chrono::milliseconds(timeout), [this](){ return this->isAckReceived_ == true; })) { LOGD("TimeSync::GetTimeOffset, retryTime_ = %d", retryTime_); retryTime_++; if (retryTime_ < MAX_RETRY_TIME) { lock.unlock(); - return GetTimeOffset(outOffset); + LOGI("TimeSync::GetTimeOffset timeout, try again"); + return GetTimeOffset(outOffset, timeout); } retryTime_ = 0; return -E_TIMEOUT; @@ -542,7 +549,7 @@ void TimeSync::CommErrHandlerFunc(int errCode, TimeSync *timeSync) void TimeSync::ResetTimer() { - std::unique_lock lock(timeDriverLock_); + std::lock_guard lock(timeDriverLock_); RuntimeContext::GetInstance()->RemoveTimer(driverTimerId_, true); int errCode = RuntimeContext::GetInstance()->SetTimer( TIME_SYNC_INTERVAL, driverCallback_, nullptr, driverTimerId_); diff --git a/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.h index 7730e5c34b6201fa02eb54da56213c8b6301a82c..df84e6fb15649c7929b97f4e665b975170e4a5f4 100755 --- a/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/time_sync.h @@ -20,7 +20,6 @@ #include "meta_data.h" #include "sync_task_context.h" #include "time_helper.h" -#include "ref_object.h" namespace DistributedDB { class TimeSyncPacket { @@ -73,16 +72,16 @@ public: static int DeSerialization(const uint8_t *buffer, uint32_t length, Message *inMsg); // register to communicator int Initialize(ICommunicator *communicator, std::shared_ptr &metadata, - const IKvDBSyncInterface *storage, const DeviceID &deviceId); + const ISyncInterface *storage, const DeviceID &deviceId); - int SyncStart(const CommErrHandler &handler = nullptr); // send timesync request + int SyncStart(const CommErrHandler &handler = nullptr, uint32_t sessionId = 0); // send timesync request - int AckRecv(const Message *message); + int AckRecv(const Message *message, uint32_t targetSessionId = 0); int RequestRecv(const Message *message); // Get timeoffset from metadata - int GetTimeOffset(TimeOffset &outOffset); + int GetTimeOffset(TimeOffset &outOffset, uint32_t timeout, uint32_t sessionId = 0); bool IsNeedSync() const; diff --git a/services/distributeddataservice/libs/distributeddb/syncer/include/value_slice_sync.h b/services/distributeddataservice/libs/distributeddb/syncer/src/value_slice_sync.h old mode 100755 new mode 100644 similarity index 100% rename from services/distributeddataservice/libs/distributeddb/syncer/include/value_slice_sync.h rename to services/distributeddataservice/libs/distributeddb/syncer/src/value_slice_sync.h index 6348c13e11b3c899d555eabdeccf410fadc35051..e8ff764f1d5455aadfb0a12baeb4a5c05a0fedc5 --- a/services/distributeddataservice/libs/distributeddb/syncer/include/value_slice_sync.h +++ b/services/distributeddataservice/libs/distributeddb/syncer/src/value_slice_sync.h @@ -19,8 +19,8 @@ #ifndef OMIT_MULTI_VER #include -#include "multi_ver_kvdb_sync_interface.h" #include "icommunicator.h" +#include "multi_ver_kvdb_sync_interface.h" #include "multi_ver_sync_task_context.h" namespace DistributedDB { diff --git a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn index c73f3c60c0a13faae6375f445af807bb06ab3081..7b98fee3f510bb675998ff3cafe223706c0199cf 100755 --- a/services/distributeddataservice/libs/distributeddb/test/BUILD.gn +++ b/services/distributeddataservice/libs/distributeddb/test/BUILD.gn @@ -51,6 +51,7 @@ config("module_private_config") { "USE_SQLITE_SYMBOLS", "USING_DB_JSON_EXTRACT_AUTOMATICALLY", "LOW_LEVEL_MEM_DEV", + "JSONCPP_USE_BUILDER", "OMIT_FLATBUFFER", ] } @@ -61,6 +62,8 @@ ohos_source_set("src_file") { sources = [ "../common/src/auto_launch.cpp", + "../common/src/data_compression.cpp", + "../common/src/db_ability.cpp", "../common/src/db_common.cpp", "../common/src/db_constant.cpp", "../common/src/evloop/src/event_impl.cpp", @@ -93,6 +96,7 @@ ohos_source_set("src_file") { "../common/src/time_tick_monitor.cpp", "../common/src/types_export.cpp", "../common/src/value_object.cpp", + "../common/src/zlib_compression.cpp", "../communicator/src/combine_status.cpp", "../communicator/src/communicator.cpp", "../communicator/src/communicator_aggregator.cpp", @@ -105,6 +109,7 @@ ohos_source_set("src_file") { "../communicator/src/protocol_proto.cpp", "../communicator/src/send_task_scheduler.cpp", "../communicator/src/serial_buffer.cpp", + "../interfaces/src/intercepted_data_impl.cpp", "../interfaces/src/kv_store_changed_data_impl.cpp", "../interfaces/src/kv_store_delegate_impl.cpp", "../interfaces/src/kv_store_delegate_manager.cpp", @@ -146,6 +151,7 @@ ohos_source_set("src_file") { "../storage/src/result_entries_window.cpp", "../storage/src/single_ver_natural_store_commit_notify_data.cpp", "../storage/src/sqlite/query_object.cpp", + "../storage/src/sqlite/query_sync_object.cpp", "../storage/src/sqlite/sqlite_local_kvdb.cpp", "../storage/src/sqlite/sqlite_local_kvdb_connection.cpp", "../storage/src/sqlite/sqlite_local_kvdb_snapshot.cpp", @@ -153,6 +159,8 @@ ohos_source_set("src_file") { "../storage/src/sqlite/sqlite_local_storage_executor.cpp", "../storage/src/sqlite/sqlite_multi_ver_data_storage.cpp", "../storage/src/sqlite/sqlite_multi_ver_transaction.cpp", + "../storage/src/sqlite/sqlite_query_helper.cpp", + "../storage/src/sqlite/sqlite_single_ver_continue_token.cpp", "../storage/src/sqlite/sqlite_single_ver_database_upgrader.cpp", "../storage/src/sqlite/sqlite_single_ver_forward_cursor.cpp", "../storage/src/sqlite/sqlite_single_ver_natural_store.cpp", @@ -162,6 +170,7 @@ ohos_source_set("src_file") { "../storage/src/sqlite/sqlite_single_ver_storage_engine.cpp", "../storage/src/sqlite/sqlite_single_ver_storage_executor.cpp", "../storage/src/sqlite/sqlite_single_ver_storage_executor_cache.cpp", + "../storage/src/sqlite/sqlite_single_ver_storage_executor_subscribe.cpp", "../storage/src/sqlite/sqlite_storage_engine.cpp", "../storage/src/sqlite/sqlite_storage_executor.cpp", "../storage/src/sqlite/sqlite_utils.cpp", @@ -174,6 +183,7 @@ ohos_source_set("src_file") { "../storage/src/upgrader/single_ver_schema_database_upgrader.cpp", "../syncer/src/ability_sync.cpp", "../syncer/src/commit_history_sync.cpp", + "../syncer/src/communicator_proxy.cpp", "../syncer/src/device_manager.cpp", "../syncer/src/generic_syncer.cpp", "../syncer/src/meta_data.cpp", @@ -182,8 +192,12 @@ ohos_source_set("src_file") { "../syncer/src/multi_ver_sync_state_machine.cpp", "../syncer/src/multi_ver_sync_task_context.cpp", "../syncer/src/multi_ver_syncer.cpp", + "../syncer/src/query_sync_water_mark_helper.cpp", + "../syncer/src/single_ver_data_packet.cpp", "../syncer/src/single_ver_data_sync.cpp", "../syncer/src/single_ver_data_sync_with_sliding_window.cpp", + "../syncer/src/single_ver_kv_syncer.cpp", + "../syncer/src/single_ver_serialize_manager.cpp", "../syncer/src/single_ver_sync_engine.cpp", "../syncer/src/single_ver_sync_state_machine.cpp", "../syncer/src/single_ver_sync_target.cpp", @@ -191,6 +205,7 @@ ohos_source_set("src_file") { "../syncer/src/single_ver_syncer.cpp", "../syncer/src/sliding_window_receiver.cpp", "../syncer/src/sliding_window_sender.cpp", + "../syncer/src/subscribe_manager.cpp", "../syncer/src/sync_engine.cpp", "../syncer/src/sync_operation.cpp", "../syncer/src/sync_state_machine.cpp", @@ -204,11 +219,12 @@ ohos_source_set("src_file") { "unittest/common/common/distributeddb_data_generate_unit_test.cpp", "unittest/common/common/distributeddb_tools_unit_test.cpp", "unittest/common/interfaces/process_system_api_adapter_impl.cpp", + "unittest/common/syncer/generic_virtual_device.cpp", + "unittest/common/syncer/kv_virtual_device.cpp", + "unittest/common/syncer/virtual_communicator.cpp", + "unittest/common/syncer/virtual_communicator_aggregator.cpp", "unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp", "unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp", - "unittest/common/syncer/vitural_communicator.cpp", - "unittest/common/syncer/vitural_communicator_aggregator.cpp", - "unittest/common/syncer/vitural_device.cpp", ] configs = [ ":module_private_config" ] @@ -216,6 +232,7 @@ ohos_source_set("src_file") { deps = [ "//third_party/googletest:gtest_main", "//third_party/sqlite:sqlite", + "//third_party/zlib:libz", "//utils/native/base:utils", ] @@ -241,8 +258,10 @@ template("distributeddb_unittest") { configs = [ ":module_private_config" ] deps += [ ":src_file", + "//third_party/googletest:gmock_main", "//third_party/googletest:gtest_main", "//third_party/sqlite:sqlite", + "//third_party/zlib:libz", "//utils/native/base:utils", ] configs += [ "//third_party/jsoncpp:jsoncpp_config" ] @@ -343,12 +362,20 @@ distributeddb_unittest("DistributedDBSingleVerP2PSyncTest") { [ "unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp" ] } +distributeddb_unittest("DistributedDBSyncModuleTest") { + sources = [ "unittest/common/syncer/distributeddb_sync_module_test.cpp" ] +} + distributeddb_unittest("DistributedDBInterfacesNBDelegateTest") { sources = [ "unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp", ] } +distributeddb_unittest("DistributedDBCommonTest") { + sources = [ "unittest/common/common/distributeddb_common_test.cpp" ] +} + distributeddb_unittest("DistributedDBInterfacesNBDelegateLocalBatchTest") { sources = [ "unittest/common/interfaces/distributeddb_interfaces_nb_delegate_local_batch_test.cpp" ] } @@ -392,58 +419,6 @@ distributeddb_unittest("DistributedDBStorageRegisterObserverTest") { ] } -ohos_unittest("DistributedDBStorageEncryptTest") { - module_out_path = module_output_path - include_dirs = [ - "../include", - "../common/include", - "//third_party/sqlite/include", - "//utils/native/base/include", - ] - - sources = [ - "../common/src/log_print.cpp", - "//third_party/sqlite/src/sqlite3.c", - "unittest/common/storage/distributeddb_storage_encrypt_test.cpp", - ] - - configs = [ ":module_private_config" ] - external_deps = [ "hiviewdfx_hilog_native:libhilog" ] - - defines = [ - "NDEBUG=1", - "HAVE_USLEEP=1", - "SQLITE_HAVE_ISNAN", - "SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576", - "SQLITE_THREADSAFE=2", - "SQLITE_TEMP_STORE=3", - "SQLITE_POWERSAFE_OVERWRITE=1", - "SQLITE_DEFAULT_FILE_FORMAT=4", - "SQLITE_DEFAULT_AUTOVACUUM=1", - "SQLITE_ENABLE_MEMORY_MANAGEMENT=1", - "SQLITE_ENABLE_FTS3", - "SQLITE_ENABLE_FTS4", - "SQLITE_OMIT_COMPILEOPTION_DIAGS", - "SQLITE_OMIT_LOAD_EXTENSION", - "SQLITE_DEFAULT_FILE_PERMISSIONS=0600", - "SQLITE_SECURE_DELETE", - "SQLITE_ENABLE_BATCH_ATOMIC_WRITE", - "USE_PREAD64", - "fdatasync=fdatasync", - "HAVE_MALLOC_H=1", - "HAVE_MALLOC_USABLE_SIZE", - "SQLITE_DIRECT_OVERFLOW_READ", - "SQLITE_HAS_CODEC", - "SQLITE_CODEC_ATTACH_CHANGED", - ] - ldflags = [ "-Wl,--exclude-libs,ALL" ] - deps = [ - "//third_party/googletest:gtest_main", - "//third_party/openssl:libcrypto_static", - "//utils/native/base:utils", - ] -} - distributeddb_unittest("DistributedDBCommunicatorTest") { sources = [ "unittest/common/communicator/adapter_stub.cpp", @@ -564,6 +539,10 @@ distributeddb_unittest("DistributedDBAutoLaunchUnitTest") { sources = [ "unittest/common/common/distributeddb_auto_launch_test.cpp" ] } +distributeddb_unittest("DistributedDBDataCompressionTest") { + sources = [ "unittest/common/common/distributeddb_data_compression_test.cpp" ] +} + ############################################################################### distributeddb_unittest("DistributedDBJsonPrecheckUnitTest") { sources = @@ -598,6 +577,26 @@ distributeddb_unittest("DistributedDBInterfacesSchemaDatabaseUpgradeTest") { sources = [ "unittest/common/interfaces/distributeddb_interfaces_schema_database_upgrade_test.cpp" ] } +distributeddb_unittest("DistributedDBStorageQuerySyncTest") { + sources = + [ "unittest/common/storage/distributeddb_storage_query_sync_test.cpp" ] +} + +distributeddb_unittest("DistributedDBSingleVerP2PQuerySyncTest") { + sources = [ + "unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp", + ] +} + +distributeddb_unittest("DistributedDBCommunicatorProxyTest") { + sources = + [ "unittest/common/syncer/distributeddb_communicator_proxy_test.cpp" ] +} + +distributeddb_unittest("DistributedDBSingleVerP2PSubscribeSyncTest") { + sources = [ "unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp" ] +} + ############################################################################### group("unittest") { testonly = true @@ -606,7 +605,9 @@ group("unittest") { deps += [ ":DistributedDBAbilitySyncTest", ":DistributedDBAutoLaunchUnitTest", + ":DistributedDBCommonTest", ":DistributedDBCommunicatorDeepTest", + ":DistributedDBCommunicatorProxyTest", ":DistributedDBCommunicatorSendReceiveTest", ":DistributedDBCommunicatorTest", ":DistributedDBDeviceIdentifierTest", @@ -642,15 +643,17 @@ group("unittest") { ":DistributedDBParcelTest", ":DistributedDBSchemaObjectTest", ":DistributedDBSchemalTest", + ":DistributedDBSingleVerP2PQuerySyncTest", + ":DistributedDBSingleVerP2PSubscribeSyncTest", ":DistributedDBSingleVerP2PSyncCheckTest", ":DistributedDBSingleVerP2PSyncTest", ":DistributedDBSingleVersionResultSetTest", ":DistributedDBSqliteRegisterTest", ":DistributedDBStorageCommitStorageTest", ":DistributedDBStorageDataOperationTest", - ":DistributedDBStorageEncryptTest", ":DistributedDBStorageIndexOptimizeTest", ":DistributedDBStorageMemorySingleVerNaturalStoreTest", + ":DistributedDBStorageQuerySyncTest", ":DistributedDBStorageRegisterConflictTest", ":DistributedDBStorageRegisterObserverTest", ":DistributedDBStorageResultAndJsonOptimizeTest", @@ -658,6 +661,7 @@ group("unittest") { ":DistributedDBStorageSingleVerUpgradeTest", ":DistributedDBStorageTransactionDataTest", ":DistributedDBStorageTransactionRecordTest", + ":DistributedDBSyncModuleTest", ":DistributedDBSyncerDeviceManagerTest", ":DistributedDBTimeSyncTest", ":RuntimeContextProcessSystemApiAdapterImplTest", diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp index 989c2a3e54ca202e7e72c0457ad107a09aab890b..7ad339b78cf5392e689f73296576f6603172c4b6 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_auto_launch_test.cpp @@ -15,15 +15,15 @@ #include #include "auto_launch.h" -#include "db_errno.h" -#include "log_print.h" #include "db_common.h" -#include "kvdb_manager.h" +#include "db_errno.h" #include "distributeddb_tools_unit_test.h" -#include "vitural_communicator_aggregator.h" -#include "platform_specific.h" #include "kv_store_nb_conflict_data.h" +#include "kvdb_manager.h" #include "kvdb_pragma.h" +#include "log_print.h" +#include "platform_specific.h" +#include "virtual_communicator_aggregator.h" using namespace std; using namespace testing::ext; @@ -135,6 +135,7 @@ static void GetProperty(KvDBProperties &prop, std::string &identifier, std::stri void DistributedDBAutoLaunchUnitTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); if (DistributedDBToolsUnitTest::RemoveTestDbFiles( g_testDir + "/" + DBCommon::TransferStringToHex(g_identifierA) + "/single_ver") != 0) { LOGE("rm test db files error!"); @@ -198,7 +199,9 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch001, TestSize.Level3) * @tc.steps: step1. right param A enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, nullptr, nullptr, 0, nullptr); + AutoLaunchOption option; + option.notifier = nullptr; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, nullptr, option); EXPECT_TRUE(errCode == E_OK); /** @@ -206,14 +209,14 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch001, TestSize.Level3) * @tc.expected: step2. failed. */ g_propB.SetStringProp(KvDBProperties::IDENTIFIER_DATA, ""); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, option); EXPECT_TRUE(errCode != E_OK); /** * @tc.steps: step3. right param C enable * @tc.expected: step3. success. */ - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, nullptr, option); EXPECT_TRUE(errCode == E_OK); /** @@ -270,10 +273,11 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch002, TestSize.Level3) * @tc.steps: step1. right param A B enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, observer, 0, nullptr); - EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, observer, 0, nullptr); - EXPECT_TRUE(errCode == E_OK); + AutoLaunchOption option; + option.notifier = nullptr; + option.observer = observer; + EXPECT_TRUE(RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, option) == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, option) == E_OK); /** * @tc.steps: step2. RunOnConnectCallback @@ -302,10 +306,8 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch002, TestSize.Level3) * @tc.steps: step4. param A B disable * @tc.expected: step4. notifier WRITE_CLOSED */ - errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierA); - EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierB); - EXPECT_TRUE(errCode == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierA) == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierB) == E_OK); std::unique_lock lock(cvMutex); cv.wait(lock, [&finished] {return finished;}); @@ -345,9 +347,12 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch003, TestSize.Level3) * @tc.steps: step1. right param A B enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, observer, 0, nullptr); + AutoLaunchOption option; + option.notifier = nullptr; + option.observer = observer; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, observer, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, option); EXPECT_TRUE(errCode == E_OK); /** @@ -401,28 +406,30 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch004, TestSize.Level3) * @tc.steps: step1. right param A~H enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, nullptr, nullptr, 0, nullptr); + AutoLaunchOption option; + option.notifier = nullptr; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propD, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propD, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propE, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propE, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propF, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propF, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propG, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propG, nullptr, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propH, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propH, nullptr, option); EXPECT_TRUE(errCode == E_OK); /** * @tc.steps: step2. right param I enable * @tc.expected: step2. -E_MAX_LIMITS. */ - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propI, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propI, nullptr, option); EXPECT_TRUE(errCode == -E_MAX_LIMITS); /** @@ -436,7 +443,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch004, TestSize.Level3) * @tc.steps: step4. right param I enable * @tc.expected: step4. E_OK. */ - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propI, nullptr, nullptr, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propI, nullptr, option); EXPECT_TRUE(errCode == E_OK); /** @@ -498,7 +505,10 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch005, TestSize.Level3) * @tc.steps: step2. right param A enable * @tc.expected: step2. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, observer, 0, nullptr); + AutoLaunchOption option; + option.notifier = nullptr; + option.observer = observer; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, option); EXPECT_TRUE(errCode == E_OK); /** @@ -562,11 +572,13 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch006, TestSize.Level3) cv.notify_one(); }); thread.detach(); - + AutoLaunchOption option; + option.notifier = nullptr; + option.observer = observer; for (int i = 0; i < TEST_ENABLE_CNT; i++) { - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, observer, 0, nullptr); + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, notifier, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, observer, 0, nullptr); + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, option); EXPECT_TRUE(errCode == E_OK); errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierA); @@ -847,10 +859,14 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch012, TestSize.Level3) * @tc.steps: step1. right param A B enable * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, TestAutoLaunchNotifier, nullptr, - CONFLICT_FOREIGN_KEY_ONLY, ConflictNotifierCallback); + AutoLaunchOption option; + option.notifier = ConflictNotifierCallback; + option.conflictType = CONFLICT_FOREIGN_KEY_ONLY; + int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propA, TestAutoLaunchNotifier, option); EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, nullptr, 0, nullptr); + AutoLaunchOption option1; + option1.notifier = nullptr; + errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, nullptr, option1); EXPECT_TRUE(errCode == E_OK); /** @@ -917,10 +933,10 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch013, TestSize.Level3) * @tc.steps: step1. right param b c enable, a SetAutoLaunchRequestCallback * @tc.expected: step1. success. */ - int errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, nullptr, 0, nullptr); - EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, notifier, nullptr, 0, nullptr); - EXPECT_TRUE(errCode == E_OK); + AutoLaunchOption option; + option.notifier = nullptr; + EXPECT_TRUE(RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propB, notifier, option) == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->EnableKvStoreAutoLaunch(g_propC, notifier, option) == E_OK); KvStoreObserverUnitTest *observer = new (std::nothrow) KvStoreObserverUnitTest; ASSERT_TRUE(observer != nullptr); @@ -978,9 +994,7 @@ HWTEST_F(DistributedDBAutoLaunchUnitTest, AutoLaunch013, TestSize.Level3) * @tc.steps: step4. param A B disable * @tc.expected: step4. notifier WRITE_CLOSED */ - errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierB); - EXPECT_TRUE(errCode == E_OK); - errCode = RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierC); - EXPECT_TRUE(errCode == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierB) == E_OK); + EXPECT_TRUE(RuntimeContext::GetInstance()->DisableKvStoreAutoLaunch(g_identifierC) == E_OK); g_communicatorAggregator->RunOnConnectCallback(REMOTE_DEVICE_ID, false); } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c667a4e22e93c914bc1490541704017780cf41ee --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_common_test.cpp @@ -0,0 +1,506 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "db_errno.h" +#include "db_common.h" +#include "distributeddb_data_generate_unit_test.h" +#include "log_print.h" +#include "platform_specific.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; + +namespace { + std::string g_testDir; + + // define some variables to init a KvStoreDelegateManager object. + KvStoreDelegateManager g_mgr(APP_ID, USER_ID); + KvStoreConfig g_config; + + // define the g_kvDelegateCallback, used to get some information when open a kv store. + DBStatus g_kvDelegateStatus = INVALID_ARGS; + + KvStoreNbDelegate *g_kvNbDelegatePtr = nullptr; + auto g_kvNbDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, + std::placeholders::_1, std::placeholders::_2, std::ref(g_kvDelegateStatus), std::ref(g_kvNbDelegatePtr)); +} + +class DistributedDBCommonTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBCommonTest::SetUpTestCase(void) +{ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + g_config.dataDir = g_testDir; + g_mgr.SetKvStoreConfig(g_config); +} + +void DistributedDBCommonTest::TearDownTestCase(void) {} + +void DistributedDBCommonTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + DistributedDBToolsUnitTest::TestDirInit(g_testDir); +} + +void DistributedDBCommonTest::TearDown(void) +{ + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGI("rm test db files error!"); + } +} + +/** + * @tc.name: RemoveAllFilesOfDirectory + * @tc.desc: Test delete all file and dir. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, RemoveAllFilesOfDirectory, TestSize.Level1) +{ + EXPECT_EQ(DBCommon::CreateDirectory(g_testDir + "/dirLevel1_1/"), E_OK); + EXPECT_EQ(DBCommon::CreateDirectory(g_testDir + "/dirLevel1_1/" + "/dirLevel2_1/"), E_OK); + EXPECT_EQ(DBCommon::CreateDirectory(g_testDir + "/dirLevel1_1/" + "/dirLevel2_2/"), E_OK); + EXPECT_EQ(DBCommon::CreateDirectory(g_testDir + "/dirLevel1_1/" + "/dirLevel2_2/" + "/dirLevel3_1/"), E_OK); + + EXPECT_EQ(OS::CreateFileByFileName(g_testDir + "/fileLevel1_1"), E_OK); + EXPECT_EQ(OS::CreateFileByFileName(g_testDir + "/dirLevel1_1/" + "/fileLevel2_1"), E_OK); + EXPECT_EQ(DBCommon::CreateDirectory(g_testDir + "/dirLevel1_1/" + "/dirLevel2_2/" + + "/dirLevel3_1/"+ "/fileLevel4_1/"), E_OK); + + EXPECT_EQ(DBCommon::RemoveAllFilesOfDirectory(g_testDir), E_OK); + + EXPECT_EQ(OS::CheckPathExistence(g_testDir), false); +} + +#ifdef RUNNING_ON_LINUX +/** + * @tc.name: SameProcessReLockFile + * @tc.desc: Test same process repeat lock same file. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, SameProcessReLockFile, TestSize.Level1) +{ + // block mode + EXPECT_EQ(OS::CreateFileByFileName(g_testDir + "/blockmode"), E_OK); + OS::FileHandle fd; + EXPECT_EQ(OS::OpenFile(g_testDir + "/blockmode", fd), E_OK); + + EXPECT_EQ(OS::FileLock(fd, true), E_OK); + EXPECT_EQ(OS::FileLock(fd, true), E_OK); + + // normal mode + OS::FileHandle fd2; + EXPECT_EQ(OS::CreateFileByFileName(g_testDir + "/normalmode"), E_OK); + EXPECT_EQ(OS::OpenFile(g_testDir + "/normalmode", fd2), E_OK); + EXPECT_EQ(OS::FileLock(fd2, true), E_OK); + EXPECT_EQ(OS::FileLock(fd2, true), E_OK); + + // unlock + EXPECT_EQ(OS::FileUnlock(fd), E_OK); + EXPECT_EQ(OS::FileUnlock(fd2), E_OK); // unlock success will close fd +} + +/** + * @tc.name: SameProcessReUnLockFile + * @tc.desc: Test same process repeat lock same file. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, SameProcessReUnLockFile, TestSize.Level1) +{ + // unlock normal file twice + EXPECT_EQ(OS::CreateFileByFileName(g_testDir + "/normalmode"), E_OK); + OS::FileHandle fd; + EXPECT_EQ(OS::OpenFile(g_testDir + "/normalmode", fd), E_OK); + EXPECT_EQ(OS::FileUnlock(fd), E_OK); + EXPECT_EQ(OS::FileUnlock(fd), E_OK); // unlock success will close fd + + EXPECT_EQ(OS::FileLock(fd, true), -E_SYSTEM_API_FAIL); + EXPECT_EQ(OS::FileLock(fd, true), -E_SYSTEM_API_FAIL); + ASSERT_EQ(fd.handle, -1); + + // block mode + EXPECT_EQ(OS::CreateFileByFileName(g_testDir + "/blockmode"), E_OK); + EXPECT_EQ(OS::OpenFile(g_testDir + "/blockmode", fd), E_OK); + + EXPECT_EQ(OS::FileLock(fd, false), E_OK); + EXPECT_EQ(OS::FileLock(fd, false), E_OK); + + EXPECT_EQ(OS::FileUnlock(fd), E_OK); + EXPECT_EQ(OS::FileUnlock(fd), E_OK); +} + +// Distributed db is not recommended to use multiple processes to access +// This testcase only guard for some wrong use on current product +#if defined(RUN_MULTI_PROCESS_TEST) +namespace { +// use file sync diff process information +bool waitForStep(int step, int retryTimes) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + while (retryTimes >= 0 && !OS::CheckPathExistence(g_testDir + "/LOCK_step_" + std::to_string(step))) { + retryTimes = retryTimes - 1; // wait 10ms one times + std::this_thread::sleep_for(std::chrono::milliseconds(10)); // once 10 ms + } + return (retryTimes > 0); +} + +void createStepFlag(int step) +{ + EXPECT_EQ(OS::CreateFileByFileName(g_testDir + "/LOCK_step_" + std::to_string(step)), E_OK); +} +} + +/** + * @tc.name: DiffProcessLockFile + * @tc.desc: Test different process repeat lock same file. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, DiffProcessLockFile, TestSize.Level1) +{ + OS::FileHandle fd; + EXPECT_EQ(OS::OpenFile(g_testDir + DBConstant::DB_LOCK_POSTFIX, fd), E_OK); + EXPECT_EQ(OS::FileLock(fd, false), E_OK); + sleep(1); + LOGI("begin fork new process!!"); + pid_t pid = fork(); + ASSERT_TRUE(pid >= 0); + if (pid < 0) { + return; + } + else if (pid == 0) { + LOGI("child process begin!"); + OS::FileHandle ChildFd; + EXPECT_EQ(OS::OpenFile(g_testDir + DBConstant::DB_LOCK_POSTFIX, ChildFd), E_OK); + ASSERT_TRUE(waitForStep(1, 10)); + EXPECT_EQ(OS::FileLock(ChildFd, false), -E_BUSY); + createStepFlag(2); + EXPECT_EQ(OS::CloseFile(ChildFd), E_OK); + exit(0); + } else { + LOGI("main process begin!"); + EXPECT_EQ(OS::FileLock(fd, false), E_OK); + createStepFlag(1); + + ASSERT_TRUE(waitForStep(2, 100)); + EXPECT_EQ(OS::CloseFile(fd), E_OK); // fd close, lock invalid + } +} + +/** + * @tc.name: DiffProcessLockFileBlocked + * @tc.desc: Test different process repeat lock same file. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, DiffProcessLockFileBlocked, TestSize.Level1) +{ + EXPECT_EQ(OS::CreateFileByFileName(g_testDir + DBConstant::DB_LOCK_POSTFIX), E_OK); + OS::FileHandle fd; + EXPECT_EQ(OS::OpenFile(g_testDir + DBConstant::DB_LOCK_POSTFIX, fd), E_OK); + EXPECT_EQ(OS::FileLock(fd, true), E_OK); + sleep(1); + LOGI("begin fork new process!!"); + int count = 10; // wait 10 times 10 ms for block wait + pid_t pid = fork(); + ASSERT_TRUE(pid >= 0); + if (pid < 0) { + return; + } + else if (pid == 0) { + LOGI("child process begin!"); + EXPECT_FALSE(OS::CheckPathExistence(g_testDir + "/LOCK_step_1")); + OS::FileHandle ChildFd; + EXPECT_EQ(OS::OpenFile(g_testDir + DBConstant::DB_LOCK_POSTFIX, ChildFd), E_OK); + EXPECT_EQ(OS::FileLock(ChildFd, true), E_OK); + createStepFlag(1); + EXPECT_EQ(OS::FileUnlock(ChildFd), E_OK); + LOGI("child process finish!"); + exit(0); + } else { + LOGI("main process begin!"); + while (count--) { + LOGI("main process waiting!"); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); // once 10 ms + } + ASSERT_FALSE(waitForStep(1, 10)); + EXPECT_EQ(OS::FileUnlock(fd), E_OK); + ASSERT_TRUE(waitForStep(1, 10)); + } +} + +/** + * @tc.name: DiffProcessGetDBBlocked + * @tc.desc: Test block other process get kvstore when db locked. + * @tc.type: FUNC + * @tc.require: AR000CQDV7 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, DiffProcessGetDBBlocked, TestSize.Level1) +{ + std::string storeId = "DiffProcessGetDBBlocked"; + std::string origId = USER_ID + "-" + APP_ID + "-" + storeId; + std::string identifier = DBCommon::TransferHashString(origId); + std::string hexDir = DBCommon::TransferStringToHex(identifier); + std::string lockFile = g_testDir + "/" + hexDir + DBConstant::DB_LOCK_POSTFIX; + EXPECT_EQ(DBCommon::CreateDirectory(g_testDir + "/" + hexDir), E_OK); + EXPECT_EQ(OS::CreateFileByFileName(lockFile), E_OK); + LOGI("Create lock file[%s]", lockFile.c_str()); + + LOGI("begin fork new process!!"); + pid_t pid = fork(); + OS::FileHandle fd; + ASSERT_TRUE(pid >= 0); + if (pid == 0) { + LOGI("child process begin!"); + ASSERT_TRUE(waitForStep(1, 10)); + KvStoreNbDelegate::Option option = {true, false, false}; + option.isNeedIntegrityCheck = true; + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + EXPECT_TRUE(g_kvDelegateStatus == BUSY); + ASSERT_TRUE(g_kvNbDelegatePtr == nullptr); + createStepFlag(2); + exit(0); + } else { + LOGI("main process begin!"); + EXPECT_EQ(OS::OpenFile(lockFile, fd), E_OK); + EXPECT_EQ(OS::FileLock(fd, false), E_OK); + createStepFlag(1); + } + + // Prevent the child process from not being completed, the main process ends to clean up resources + EXPECT_TRUE(waitForStep(2, 1000)); + EXPECT_EQ(OS::FileUnlock(fd), E_OK); +} + +/** + * @tc.name: DiffProcessDeleteDBBlocked + * @tc.desc: Test block other process delete kvstore when db locked. + * @tc.type: FUNC + * @tc.require: AR000CQDV7 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, DiffProcessDeleteDBBlocked, TestSize.Level1) +{ + std::string storeId = "DiffProcessDeleteDBBlocked"; + std::string origId = USER_ID + "-" + APP_ID + "-" + storeId; + std::string identifier = DBCommon::TransferHashString(origId); + std::string hexDir = DBCommon::TransferStringToHex(identifier); + std::string lockFile = g_testDir + "/" + hexDir + DBConstant::DB_LOCK_POSTFIX; + EXPECT_EQ(DBCommon::CreateDirectory(g_testDir + "/" + hexDir), E_OK); + EXPECT_EQ(OS::CreateFileByFileName(lockFile), E_OK); + LOGI("Create lock file[%s]", lockFile.c_str()); + + KvStoreNbDelegate::Option option = {true, false, false}; + option.isNeedIntegrityCheck = true; + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + + LOGI("begin fork new process!!"); + pid_t pid = fork(); + OS::FileHandle fd; + ASSERT_TRUE(pid >= 0); + if (pid == 0) { + LOGI("child process begin!"); + ASSERT_TRUE(waitForStep(1, 10)); + EXPECT_EQ(g_mgr.DeleteKvStore(storeId), BUSY); + createStepFlag(2); + exit(0); + } else { + LOGI("main process begin!"); + EXPECT_EQ(OS::OpenFile(lockFile, fd), E_OK); + EXPECT_EQ(OS::FileLock(fd, false), E_OK); + createStepFlag(1); + } + + // Prevent the child process from not being completed, the main process ends to clean up resources + EXPECT_TRUE(waitForStep(2, 1000)); + EXPECT_EQ(OS::FileUnlock(fd), E_OK); + g_mgr.CloseKvStore(g_kvNbDelegatePtr); +} + +/** + * @tc.name: DiffProcessGetDBBlocked001 + * @tc.desc: Test block other process get kvstore when db locked. + * @tc.type: FUNC + * @tc.require: AR000CQDV7 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, DiffProcessGetDBBlocked001, TestSize.Level1) +{ + std::string storeId = "DiffProcessGetDBBlocked001"; + std::string origId = USER_ID + "-" + APP_ID + "-" + storeId; + std::string identifier = DBCommon::TransferHashString(origId); + std::string hexDir = DBCommon::TransferStringToHex(identifier); + std::string lockFile = g_testDir + "/" + hexDir + DBConstant::DB_LOCK_POSTFIX; + EXPECT_EQ(DBCommon::CreateDirectory(g_testDir + "/" + hexDir), E_OK); + EXPECT_EQ(OS::CreateFileByFileName(lockFile), E_OK); + LOGI("Create lock file[%s]", lockFile.c_str()); + + LOGI("begin fork new process!!"); + pid_t pid = fork(); + OS::FileHandle fd; + ASSERT_TRUE(pid >= 0); + if (pid == 0) { + LOGI("child process begin!"); + ASSERT_TRUE(waitForStep(1, 10)); + KvStoreNbDelegate::Option option = {true, false, false}; + option.isNeedIntegrityCheck = true; + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + createStepFlag(2); + exit(0); + } else { + LOGI("main process begin!"); + EXPECT_EQ(OS::OpenFile(lockFile, fd), E_OK); + EXPECT_EQ(OS::FileLock(fd, false), E_OK); + createStepFlag(1); + } + ASSERT_TRUE(waitForStep(1, 100)); + + EXPECT_EQ(OS::FileUnlock(fd), E_OK); + + ASSERT_TRUE(waitForStep(2, 100)); +} + +/** + * @tc.name: DiffProcessGetDB + * @tc.desc: Test block other process get kvstore. + * @tc.type: FUNC + * @tc.require: AR000CQDV7 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, DiffProcessGetDB, TestSize.Level1) +{ + std::string storeId = "DiffProcessGetDB"; + KvStoreNbDelegate::Option option = {true, false, false}; + option.isNeedIntegrityCheck = true; + LOGI("begin fork new process!!"); + pid_t pid = fork(); + ASSERT_TRUE(pid >= 0); + if (pid == 0) { + LOGI("child process begin!"); + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + createStepFlag(2); + EXPECT_TRUE(waitForStep(1, 1000)); + exit(0); + } else { + LOGI("main process begin!"); + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + createStepFlag(1); + } + EXPECT_TRUE(waitForStep(2, 100)); + // Prevent the child process from not being completed, the main process ends to clean up resources + g_mgr.CloseKvStore(g_kvNbDelegatePtr); +} + +/** + * @tc.name: DiffProcessDeleteDB + * @tc.desc: Test block other process delete kvstore. + * @tc.type: FUNC + * @tc.require: AR000CQDV7 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, DiffProcessDeleteDB, TestSize.Level1) +{ + std::string storeId = "DiffProcessGetDB"; + KvStoreNbDelegate::Option option = {true, false, false}; + option.isNeedIntegrityCheck = true; + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + g_mgr.CloseKvStore(g_kvNbDelegatePtr); + LOGI("begin fork new process!!"); + pid_t pid = fork(); + ASSERT_TRUE(pid >= 0); + if (pid == 0) { + LOGI("child process begin!"); + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + createStepFlag(2); + EXPECT_TRUE(waitForStep(1, 1000)); + exit(0); + } else { + LOGI("main process begin!"); + g_mgr.DeleteKvStore(storeId); + createStepFlag(1); + } + EXPECT_TRUE(waitForStep(2, 100)); + + // Prevent the child process from not being completed, the main process ends to clean up resources + EXPECT_TRUE(waitForStep(1, 100)); +} + +/** + * @tc.name: DiffProcessGetAndDeleteDB + * @tc.desc: Test block other process delete kvstore. + * @tc.type: FUNC + * @tc.require: AR000CQDV7 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBCommonTest, DiffProcessGetAndDeleteDB, TestSize.Level1) +{ + std::string storeId = "DiffProcessGetAndDeleteDB"; + KvStoreNbDelegate::Option option = {true, false, false}; + option.isNeedIntegrityCheck = true; + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + g_mgr.CloseKvStore(g_kvNbDelegatePtr); + LOGI("begin fork new process!!"); + pid_t pid = fork(); + ASSERT_TRUE(pid >= 0); + if (pid == 0) { + LOGI("child process begin!"); + g_mgr.DeleteKvStore(storeId); // one process OK, one process NOT_FOUND + createStepFlag(2); + EXPECT_TRUE(waitForStep(1, 1000)); + exit(0); + } else { + LOGI("main process begin!"); + g_mgr.DeleteKvStore(storeId); + createStepFlag(1); + } + EXPECT_TRUE(waitForStep(2, 100)); + + // Prevent the child process from not being completed, the main process ends to clean up resources + EXPECT_TRUE(waitForStep(1, 1000)); +} +#endif +#endif + diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_data_compression_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_data_compression_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e3754c6530831edfbf73133870182a910d2d1052 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_data_compression_test.cpp @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "distributeddb_tools_unit_test.h" +#include "data_compression.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { +#ifndef OMIT_ZLIB +// LENGTH IS 680. +unsigned char g_srcStr[] = + "I come from Alabama with my banjo on my knee," + "I'm going to Louisiana, my true love for to see," + "It rained all night the day I left," + "The weather it was dry," + "The sun so hot, I froze to death," + "Susanna don't you cry," + "Oh, Susanna," + "Oh don't you cry for me," + "For I come from Alabama," + "With my banjo on my knee," + "I had a dream the other night when everything was still," + "I thought I saw Susanna a-coming down the hill," + "The buckwheat cake was in her mouth," + "The tear was in her eye," + "Says I, I'm coming from the south," + "Susanna, don't you cry," + "Oh, Susanna," + "Oh don't you cry for me," + "I'm going to Louisiana," + "With my banjo on my knee," + "Oh, Susanna," + "Oh don't you cry for me," + "I'm going to Louisiana," + "With my banjo on my knee."; +} +#endif +class DistributedDBDataCompressionTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBDataCompressionTest::SetUpTestCase(void) +{} + +void DistributedDBDataCompressionTest::TearDownTestCase(void) +{} + +void DistributedDBDataCompressionTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); +} + +void DistributedDBDataCompressionTest::TearDown(void) +{} + +/** + * @tc.name: DataCompression1 + * @tc.desc: To test the function compress and uncompress works well in normal situation. + * @tc.type: FUNC + * @tc.require: AR000G3QTT + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBDataCompressionTest, DataCompression1, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare a source data. And compress it. + * @tc.expected: step1. Compress successfully. Compressed data length is less than srcLen. + */ +#ifndef OMIT_ZLIB + const int origLen = sizeof(g_srcStr); + vector srcData(g_srcStr, g_srcStr + sizeof(g_srcStr)); + + vector compressedData; + EXPECT_EQ(DataCompression::GetInstance(CompressAlgorithm::ZLIB)->Compress(srcData, compressedData), E_OK); + EXPECT_LT(compressedData.size(), srcData.size()); + + /** + * @tc.steps:step2. Uncompress the compressed data. + * @tc.expected: step2. Uncompress successfully. Uncompressed data equals to source data. + */ + vector uncompressedData; + EXPECT_EQ(DataCompression::GetInstance(CompressAlgorithm::ZLIB)->Uncompress( + compressedData, uncompressedData, origLen), E_OK); + EXPECT_EQ(srcData, uncompressedData); +#endif // OMIT_ZLIB +} + +/** + * @tc.name: DataCompression2 + * @tc.desc: To test uncompress failed when compressed is destroyed. + * @tc.type: FUNC + * @tc.require: AR000G3QTT + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBDataCompressionTest, DataCompression2, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare a source data. And compress it. + * @tc.expected: step1. Compress successfully. Compressed data length is less than srcLen. + */ +#ifndef OMIT_ZLIB + const int origLen = sizeof(g_srcStr); + vector srcData(g_srcStr, g_srcStr + sizeof(g_srcStr)); + + vector compressedData; + EXPECT_EQ(DataCompression::GetInstance(CompressAlgorithm::ZLIB)->Compress(srcData, compressedData), E_OK); + EXPECT_LT(compressedData.size(), srcData.size()); + + /** + * @tc.steps:step2. Destroy the compressed data. + */ + *(compressedData.begin()) = ~*(compressedData.begin()); + + /** + * @tc.steps:step3. Uncompress the compressed data. + * @tc.expected: step3. Uncompressed failed and return -E_SYSTEM_API_FAIL. + */ + vector uncompressedData; + EXPECT_EQ(DataCompression::GetInstance(CompressAlgorithm::ZLIB)->Uncompress( + compressedData, uncompressedData, origLen), -E_SYSTEM_API_FAIL); +#endif // OMIT_ZLIB +} + +/** + * @tc.name: DataCompression3 + * @tc.desc: To test uncompress works when bufferLen is larger but under 30M limit, + and uncompress failed when bufferLen is beyond the 30M limit. + * @tc.type: FUNC + * @tc.require: AR000G3QTT + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBDataCompressionTest, DataCompression3, TestSize.Level1) +{ + /** + * @tc.steps:step1. Prepare a source data. And compress it. + * @tc.expected: step1. Compress successfully. Compressed data length is less than srcLen. + */ +#ifndef OMIT_ZLIB + vector srcData(g_srcStr, g_srcStr + sizeof(g_srcStr)); + + vector compressedData; + EXPECT_EQ(DataCompression::GetInstance(CompressAlgorithm::ZLIB)->Compress(srcData, compressedData), E_OK); + EXPECT_LT(compressedData.size(), srcData.size()); + + /** + * @tc.steps:step2. Set origLen a larger num under 30M limit. And uncompress. + * @tc.expected: step1. Uncompress successfully. + */ + vector uncompressedData; + int incorrectLen = 10000; + EXPECT_EQ(DataCompression::GetInstance(CompressAlgorithm::ZLIB)->Uncompress( + compressedData, uncompressedData, incorrectLen), E_OK); + EXPECT_EQ(srcData, uncompressedData); + + /** + * @tc.steps:step2. Set origLen a larger num beyond 30M limit. And uncompress. + * @tc.expected: step1. Uncompress failed and return E_INVALID_ARGS. + */ + uncompressedData.clear(); + incorrectLen = 31457281; // 30M + 1 + EXPECT_EQ(DataCompression::GetInstance(CompressAlgorithm::ZLIB)->Uncompress( + compressedData, uncompressedData, incorrectLen), -E_INVALID_ARGS); +#endif // OMIT_ZLIB +} diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_data_generate_unit_test.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_data_generate_unit_test.h index 99d5489bf9e6349c2153a5ebb008b7662ce4083b..e78ce75ec1017508839ebbe76f5440bb3f3e1bac 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_data_generate_unit_test.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_data_generate_unit_test.h @@ -24,6 +24,7 @@ namespace DistributedDBUnitTest { // define some variables to init a KvStoreDelegateManager object. const std::string APP_ID = "app0"; +const std::string SCHEMA_APP_ID = "app1"; const std::string USER_ID = "user0"; const std::string STORE_ID_LOCAL = "distributed_local_db_test"; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_json_precheck_unit_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_json_precheck_unit_test.cpp index 6586ce0e1a526612fdd71080da76a9d758d9a8ef..835a27d50e702acc254e5496d31caad032420718 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_json_precheck_unit_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_json_precheck_unit_test.cpp @@ -14,9 +14,10 @@ */ #ifndef OMIT_JSON -#include #include +#include #include "db_errno.h" +#include "distributeddb_tools_unit_test.h" #include "json_object.h" using namespace std; @@ -62,7 +63,7 @@ class DistributedDBJsonPrecheckUnitTest : public testing::Test { public: static void SetUpTestCase(void); static void TearDownTestCase(void); - void SetUp() {}; + void SetUp(); void TearDown() {}; }; @@ -82,6 +83,11 @@ void DistributedDBJsonPrecheckUnitTest::TearDownTestCase(void) JsonObject::SetMaxNestDepth(g_oriMaxNestDepth); } +void DistributedDBJsonPrecheckUnitTest::SetUp() +{ + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); +} + /** * @tc.name: Precheck Valid String 001 * @tc.desc: json string is legal @@ -89,7 +95,7 @@ void DistributedDBJsonPrecheckUnitTest::TearDownTestCase(void) * @tc.require: AR000DR9K3 * @tc.author: yiguang */ -HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseValidString001, TestSize.Level0) +HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseValidString001, TestSize.Level1) { /** * @tc.steps: step1. Check legal json string with nesting depth of 12. @@ -114,7 +120,7 @@ HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseValidString001, TestSize.Level0 * @tc.require: AR000DR9K3 * @tc.author: yiguang */ -HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseValidString002, TestSize.Level0) +HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseValidString002, TestSize.Level1) { /** * @tc.steps: step1. Check legal json string with nesting depth of 6. @@ -139,7 +145,7 @@ HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseValidString002, TestSize.Level0 * @tc.require: AR000DR9K3 * @tc.author: yiguang */ -HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseInvalidString001, TestSize.Level0) +HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseInvalidString001, TestSize.Level1) { /** * @tc.steps: step1. Parsing of illegal json string with nesting depth greater than 10 success. @@ -157,7 +163,7 @@ HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseInvalidString001, TestSize.Leve * @tc.require: AR000DR9K3 * @tc.author: yiguang */ -HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseInvalidString002, TestSize.Level0) +HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseInvalidString002, TestSize.Level1) { /** * @tc.steps: step1. Parsing of illegal json string with nesting depth less than 10 success. @@ -175,7 +181,7 @@ HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseInvalidString002, TestSize.Leve * @tc.require: AR000DR9K3 * @tc.author: yiguang */ -HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseInvalidString003, TestSize.Level0) +HWTEST_F(DistributedDBJsonPrecheckUnitTest, ParseInvalidString003, TestSize.Level1) { /** * @tc.steps: step1. Detect illegal json string with nesting depth greater than 10. diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_notification_chain_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_notification_chain_test.cpp index 27061306f29fa627bb419f2ef6d8703bd65b4312..7208d44505ee235e9503d3f0668cd56d118c0fbf 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_notification_chain_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_notification_chain_test.cpp @@ -16,12 +16,14 @@ #include #include "db_errno.h" +#include "distributeddb_tools_unit_test.h" #include "log_print.h" #include "notification_chain.h" using namespace testing::ext; using namespace DistributedDB; using namespace std; +using namespace DistributedDBUnitTest; namespace { const EventType COMMIT_EVENT = 1; @@ -31,8 +33,8 @@ namespace { int g_onEventTestNum = 0; bool g_onFinalizeCalled = false; - auto g_onEventFunction = [](const void *arg) { - g_onEventTestNum = *(reinterpret_cast(arg)); + auto g_onEventFunction = [](void *arg) { + g_onEventTestNum = *(reinterpret_cast(arg)); LOGI("g_onEventFunction called."); }; @@ -71,6 +73,7 @@ void DistributedDBNotificationChainTest::TearDownTestCase(void) void DistributedDBNotificationChainTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: Register a listener to the NotificationChain */ diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_parcel_unit_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_parcel_unit_test.cpp index b064bb604af931059380d19da2574b96afef91ca..ff527723342fca9efcc0819106acf8a95d710689 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_parcel_unit_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_parcel_unit_test.cpp @@ -42,6 +42,7 @@ void DistributedDBParcelTest::TearDownTestCase(void) void DistributedDBParcelTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); } void DistributedDBParcelTest::TearDown(void) @@ -55,7 +56,7 @@ void DistributedDBParcelTest::TearDown(void) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteInt001, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteInt001, TestSize.Level1) { /** * @tc.steps: step1. create a vector, and write it into a buffer; @@ -112,7 +113,7 @@ HWTEST_F(DistributedDBParcelTest, WriteInt001, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteVector001, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteVector001, TestSize.Level1) { /** * @tc.steps: step1. create a vector, and write it into a buffer; @@ -147,7 +148,7 @@ HWTEST_F(DistributedDBParcelTest, WriteVector001, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteVector002, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteVector002, TestSize.Level1) { /** * @tc.steps: step1. create an empty vector, and write it into a buffer; @@ -182,7 +183,7 @@ HWTEST_F(DistributedDBParcelTest, WriteVector002, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteVector003, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteVector003, TestSize.Level1) { /** * @tc.steps: step1. create a vector, and write it into a buffer; @@ -217,7 +218,7 @@ HWTEST_F(DistributedDBParcelTest, WriteVector003, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteVector004, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteVector004, TestSize.Level1) { /** * @tc.steps: step1. create an empty vector, and write it into a buffer; @@ -252,7 +253,7 @@ HWTEST_F(DistributedDBParcelTest, WriteVector004, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteVector005, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteVector005, TestSize.Level1) { /** * @tc.steps: step1. create a vector, and write it into a buffer; @@ -288,7 +289,7 @@ HWTEST_F(DistributedDBParcelTest, WriteVector005, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteVector006, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteVector006, TestSize.Level1) { /** * @tc.steps: step1. create an empty vector, and write it into a buffer; @@ -323,7 +324,7 @@ HWTEST_F(DistributedDBParcelTest, WriteVector006, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteVector007, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteVector007, TestSize.Level1) { /** * @tc.steps: step1. create a vector, and write it into a buffer; @@ -363,7 +364,7 @@ HWTEST_F(DistributedDBParcelTest, WriteVector007, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteVector008, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteVector008, TestSize.Level1) { /** * @tc.steps: step1. create a vector, and write it into a buffer; @@ -403,7 +404,7 @@ HWTEST_F(DistributedDBParcelTest, WriteVector008, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteVector009, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteVector009, TestSize.Level1) { /** * @tc.steps: step1. create a vector, and write it into a buffer; @@ -471,7 +472,7 @@ HWTEST_F(DistributedDBParcelTest, WriteVector010, TestSize.Level2) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteString001, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteString001, TestSize.Level1) { /** * @tc.steps: step1. create a string, and write it into a buffer; @@ -505,7 +506,7 @@ HWTEST_F(DistributedDBParcelTest, WriteString001, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteString002, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteString002, TestSize.Level1) { /** * @tc.steps: step1. create a string, and write it into a buffer; @@ -545,7 +546,7 @@ HWTEST_F(DistributedDBParcelTest, WriteString002, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteString003, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteString003, TestSize.Level1) { /** * @tc.steps: step1. create a string, and write it into a buffer; @@ -585,7 +586,7 @@ HWTEST_F(DistributedDBParcelTest, WriteString003, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteString004, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteString004, TestSize.Level1) { /** * @tc.steps: step1. create a string, and write it into a buffer; @@ -619,7 +620,7 @@ HWTEST_F(DistributedDBParcelTest, WriteString004, TestSize.Level0) * @tc.require: AR000CQE0U * @tc.author: weifeng */ -HWTEST_F(DistributedDBParcelTest, WriteString005, TestSize.Level0) +HWTEST_F(DistributedDBParcelTest, WriteString005, TestSize.Level1) { /** * @tc.steps: step1. create a string, and write it into a buffer; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_schema_object_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_schema_object_test.cpp index 77286bd96da6d2a0461551dd9563feb0a054d965..5f53b5eeb4aec50f96ba6c2aa14f40634b205c3a 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_schema_object_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_schema_object_test.cpp @@ -18,6 +18,7 @@ #include #include "db_errno.h" +#include "distributeddb_tools_unit_test.h" #include "log_print.h" #include "schema_object.h" #include "schema_utils.h" @@ -402,10 +403,15 @@ class DistributedDBSchemaObjectTest : public testing::Test { public: static void SetUpTestCase(void) {}; static void TearDownTestCase(void) {}; - void SetUp() {}; - void TearDown() {}; + void SetUp() override; + void TearDown() override {}; }; +void DistributedDBSchemaObjectTest::SetUp() +{ + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); +} + /** * @tc.name: Parse Valid Schema 001 * @tc.desc: Parse Valid Schema @@ -413,7 +419,7 @@ public: * @tc.require: AR000DR9K3 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, ParseValidSchema001, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, ParseValidSchema001, TestSize.Level1) { /** * @tc.steps: step1. Parse valid schema with full define @@ -455,7 +461,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, ParseValidSchema001, TestSize.Level0) * @tc.require: AR000DR9K3 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, ParseInvalidSchema001, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, ParseInvalidSchema001, TestSize.Level1) { /** * @tc.steps: step1. Parse invalid schema which is not valid json @@ -505,7 +511,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, ParseInvalidSchema001, TestSize.Level0) * @tc.require: AR000DR9K3 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, ParseInvalidSchema002, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, ParseInvalidSchema002, TestSize.Level1) { /** * @tc.steps: step1. Parse invalid schema which is empty define @@ -555,7 +561,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, ParseInvalidSchema002, TestSize.Level0) * @tc.require: AR000DR9K3 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, ParseInvalidSchema003, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, ParseInvalidSchema003, TestSize.Level1) { /** * @tc.steps: step1. Parse invalid schema with invalid array content @@ -605,7 +611,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, ParseInvalidSchema003, TestSize.Level0) * @tc.require: AR000DR9K4 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, CompareEqualExactly001, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, CompareEqualExactly001, TestSize.Level1) { SchemaObject schemaOri; int errCode = schemaOri.ParseFromSchemaString(VALID_SCHEMA_FULL_DEFINE); @@ -626,7 +632,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, CompareEqualExactly001, TestSize.Level0) * @tc.require: AR000DR9K4 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, CompareUnequalCompatible001, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, CompareUnequalCompatible001, TestSize.Level1) { SchemaObject compatibleSchema; int errCode = compatibleSchema.ParseFromSchemaString(SCHEMA_COMPARE_BASELINE); @@ -661,7 +667,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, CompareUnequalCompatible001, TestSize.Le * @tc.require: AR000DR9K4 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, CompareUnequalCompatibleUpgrade001, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, CompareUnequalCompatibleUpgrade001, TestSize.Level1) { SchemaObject compatibleSchema; int errCode = compatibleSchema.ParseFromSchemaString(SCHEMA_COMPARE_BASELINE); @@ -696,7 +702,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, CompareUnequalCompatibleUpgrade001, Test * @tc.require: AR000DR9K4 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, CompareUnequalIncompatible001, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, CompareUnequalIncompatible001, TestSize.Level1) { SchemaObject strictSchema; int errCode = strictSchema.ParseFromSchemaString(SchemaSwitchMode(SCHEMA_COMPARE_BASELINE)); @@ -771,7 +777,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, CompareUnequalIncompatible001, TestSize. * @tc.require: AR000DR9K5 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, CheckValue001, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, CheckValue001, TestSize.Level1) { SchemaObject schemaStrict; int errCode = schemaStrict.ParseFromSchemaString(VALID_SCHEMA_FULL_DEFINE); @@ -851,7 +857,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, CheckValue001, TestSize.Level0) * @tc.require: AR000DR9K5 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, CheckValue002, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, CheckValue002, TestSize.Level1) { SchemaObject schemaStrict; int errCode = schemaStrict.ParseFromSchemaString(VALID_SCHEMA_FULL_DEFINE); @@ -873,8 +879,9 @@ HWTEST_F(DistributedDBSchemaObjectTest, CheckValue002, TestSize.Level0) EXPECT_TRUE(stepOne == -E_VALUE_MATCH); std::string valueToString = value1.ToString(); + EXPECT_EQ(strValue.size(), valueToString.size()); std::string valueBeforeOffset = valueToString.substr(0, beforeOffset.size()); - EXPECT_TRUE(valueBeforeOffset == beforeOffset); + EXPECT_EQ(valueBeforeOffset, beforeOffset); } /** @@ -884,7 +891,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, CheckValue002, TestSize.Level0) * @tc.require: AR000DR9K5 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, ValueEdit001, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, ValueEdit001, TestSize.Level1) { /** * @tc.steps: step1. Insert value to ValueObject in different depth @@ -932,7 +939,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, ValueEdit001, TestSize.Level0) * @tc.require: AR000DR9K5 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, ValueEdit002, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, ValueEdit002, TestSize.Level1) { /** * @tc.steps: step1. Insert value to ValueObject in different depth @@ -987,7 +994,7 @@ void CheckValueLackField(const SchemaObject &schema, const std::string &oriValue * @tc.require: AR000DR9K5 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, ValueLackField001, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, ValueLackField001, TestSize.Level1) { SchemaObject schema; int errCode = schema.ParseFromSchemaString(SCHEMA_FOR_TEST_NOTNULL_AND_DEFAULT); @@ -1046,7 +1053,7 @@ HWTEST_F(DistributedDBSchemaObjectTest, ValueLackField001, TestSize.Level0) * @tc.require: AR000DR9K5 * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBSchemaObjectTest, ValueLackField002, TestSize.Level0) +HWTEST_F(DistributedDBSchemaObjectTest, ValueLackField002, TestSize.Level1) { SchemaObject schema; int errCode = schema.ParseFromSchemaString(SCHEMA_FOR_TEST_NOTNULL_AND_DEFAULT); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_schema_unit_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_schema_unit_test.cpp index 748ebe65898fb11b8a721a9002e43528a7254d90..3284cb84392a6f178ac9e1988cd17e0a13f335cf 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_schema_unit_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_schema_unit_test.cpp @@ -17,9 +17,9 @@ #include -#include "schema_utils.h" #include "db_errno.h" #include "log_print.h" +#include "schema_utils.h" using namespace testing::ext; using namespace DistributedDB; @@ -44,6 +44,7 @@ void DistributedDBSchemalTest::TearDownTestCase(void) void DistributedDBSchemalTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); } void DistributedDBSchemalTest::TearDown(void) @@ -182,7 +183,7 @@ void PreBoolDataForParseAndCheckSchemaAttribute003() * @tc.require: AR000DR9K3 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute001, TestSize.Level0) +HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute001, TestSize.Level1) { /** * @tc.steps: step1. Preset shcema attribute strings that are correctly written according to the definition. @@ -234,7 +235,7 @@ HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute001, TestSize.Lev * @tc.require: AR000DR9K3 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute002, TestSize.Level0) +HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute002, TestSize.Level1) { /** * @tc.steps: step1. Preset shcema attributes based on definition error. @@ -288,7 +289,7 @@ HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute002, TestSize.Lev * @tc.require: AR000DR9K3 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute003, TestSize.Level0) +HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute003, TestSize.Level1) { /** * @tc.steps: step1. Preset shcema attributes based on defining correct format and content. @@ -317,7 +318,7 @@ HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute003, TestSize.Lev * @tc.require: AR000DR9K3 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute004, TestSize.Level0) +HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute004, TestSize.Level1) { /** * @tc.steps: step1. Preset shcema attributes based on defining incorrect format and content. @@ -366,7 +367,7 @@ HWTEST_F(DistributedDBSchemalTest, ParseAndCheckSchemaAttribute004, TestSize.Lev * @tc.require: AR000DR9K3 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBSchemalTest, CheckFieldName001, TestSize.Level0) +HWTEST_F(DistributedDBSchemalTest, CheckFieldName001, TestSize.Level1) { /** * @tc.steps: step1. Enter the preset correct string array into CheckFieldName and check. @@ -391,7 +392,7 @@ HWTEST_F(DistributedDBSchemalTest, CheckFieldName001, TestSize.Level0) * @tc.require: AR000DR9K3 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBSchemalTest, CheckFieldName002, TestSize.Level0) +HWTEST_F(DistributedDBSchemalTest, CheckFieldName002, TestSize.Level1) { /** * @tc.steps: step1. Enter the preset incorrect string array into CheckFieldName and check. @@ -419,7 +420,7 @@ HWTEST_F(DistributedDBSchemalTest, CheckFieldName002, TestSize.Level0) * @tc.require: AR000DR9K3 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBSchemalTest, ParseAndCheckFieldPath001, TestSize.Level0) +HWTEST_F(DistributedDBSchemalTest, ParseAndCheckFieldPath001, TestSize.Level1) { /** * @tc.steps: step1. Enter the array of preset correct strings into ParseAndCheckFieldPath and check result. @@ -476,7 +477,7 @@ HWTEST_F(DistributedDBSchemalTest, ParseAndCheckFieldPath001, TestSize.Level0) * @tc.require: AR000DR9K3 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBSchemalTest, ParseAndCheckFieldPath002, TestSize.Level0) +HWTEST_F(DistributedDBSchemalTest, ParseAndCheckFieldPath002, TestSize.Level1) { /** * @tc.steps: step1. Enter the array of preset illegal strings into ParseAndCheckFieldPath and check result. diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp index 308d2964f92fc2d0ec7de7e1abe85303fd9a1c71..e15f218f8af58bbc950e5e726a48781f7f68e0b3 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.cpp @@ -14,20 +14,23 @@ */ #include "distributeddb_tools_unit_test.h" -#include "platform_specific.h" -#include -#include #include - +#include +#include +#include #include +#include +#include #include -#include + #include "db_common.h" -#include "value_hash_calc.h" #include "db_constant.h" #include "generic_single_ver_kv_entry.h" +#include "platform_specific.h" +#include "single_ver_data_packet.h" +#include "value_hash_calc.h" using namespace DistributedDB; @@ -513,7 +516,8 @@ bool DistributedDBToolsUnitTest::IsKvEntryExist(const DistributedDB::Entry &entr return isFound; } -int DistributedDBToolsUnitTest::ModifyDatabaseFile(const std::string &fileDir) +int DistributedDBToolsUnitTest::ModifyDatabaseFile(const std::string &fileDir, uint64_t modifyPos, + uint32_t modifyCnt, uint32_t value) { LOGI("Modify database file:%s", fileDir.c_str()); std::fstream dataFile(fileDir, std::fstream::binary | std::fstream::out | std::fstream::in); @@ -538,13 +542,15 @@ int DistributedDBToolsUnitTest::ModifyDatabaseFile(const std::string &fileDir) } } - const int sqliteCountPos = 0; - if (!dataFile.seekp(sqliteCountPos)) { + if (fileSize <= modifyPos) { + return E_OK; + } + + if (!dataFile.seekp(modifyPos)) { return -E_UNEXPECTED_DATA; } - uint32_t currentCount = 0x1F1F1F1F; // add the random value to corrupt the head. - for (int i = 0; i < 256; i++) { // 256 is 1024 / 4 times. - if (!dataFile.write(reinterpret_cast(¤tCount), sizeof(uint32_t))) { + for (uint32_t i = 0; i < modifyCnt; i++) { + if (!dataFile.write(reinterpret_cast(&value), sizeof(uint32_t))) { return -E_UNEXPECTED_DATA; } } @@ -578,11 +584,18 @@ int DistributedDBToolsUnitTest::GetSyncDataNextTest(SQLiteSingleVerNaturalStore int DistributedDBToolsUnitTest::PutSyncDataTest(SQLiteSingleVerNaturalStore *store, const std::vector &dataItems, const std::string &deviceName) +{ + QueryObject query(Query::Select()); + return PutSyncDataTest(store, dataItems, deviceName, query); +} + +int DistributedDBToolsUnitTest::PutSyncDataTest(SQLiteSingleVerNaturalStore *store, + const std::vector &dataItems, const std::string &deviceName, const QueryObject &query) { std::vector entries; std::vector items = dataItems; for (auto &item : items) { - GenericSingleVerKvEntry *entry = new (std::nothrow) GenericSingleVerKvEntry(); + auto *entry = new (std::nothrow) GenericSingleVerKvEntry(); if (entry == nullptr) { ReleaseSingleVerEntry(entries); return -E_OUT_OF_MEMORY; @@ -592,7 +605,7 @@ int DistributedDBToolsUnitTest::PutSyncDataTest(SQLiteSingleVerNaturalStore *sto entries.push_back(entry); } - int errCode = store->PutSyncData(entries, deviceName); + int errCode = store->PutSyncDataWithQuery(query, entries, deviceName); ReleaseSingleVerEntry(entries); return errCode; } @@ -699,6 +712,28 @@ bool KvStoreObserverUnitTest::IsCleared() const return isCleared_; } +DBStatus DistributedDBToolsUnitTest::SyncTest(KvStoreNbDelegate* delegate, + const std::vector& devices, SyncMode mode, + std::map& statuses, const Query &query) +{ + statuses.clear(); + DBStatus callStatus = delegate->Sync(devices, mode, + [&statuses, this](const std::map& statusMap) { + statuses = statusMap; + std::unique_lock innerlock(this->syncLock_); + this->syncCondVar_.notify_one(); + }, query, false); + + std::unique_lock lock(syncLock_); + syncCondVar_.wait(lock, [callStatus, &statuses](){ + if (callStatus != OK) { + return true; + } + return !statuses.empty(); + }); + return callStatus; +} + DBStatus DistributedDBToolsUnitTest::SyncTest(KvStoreNbDelegate* delegate, const std::vector& devices, SyncMode mode, std::map& statuses, bool wait) @@ -758,4 +793,53 @@ void KvStoreCorruptInfo::Reset() { databaseInfoVect_.clear(); } + +int DistributedDBToolsUnitTest::GetRandInt(const int randMin, const int randMax) +{ + std::random_device randDev; + std::mt19937 genRand(randDev()); + std::uniform_int_distribution disRand(randMin, randMax); + return disRand(genRand); +} + +int64_t DistributedDBToolsUnitTest::GetRandInt64(const int64_t randMin, const int64_t randMax) +{ + std::random_device randDev; + std::mt19937_64 genRand(randDev()); + std::uniform_int_distribution disRand(randMin, randMax); + return disRand(genRand); +} + +void DistributedDBToolsUnitTest::PrintTestCaseInfo() +{ + testing::UnitTest *test = testing::UnitTest::GetInstance(); + ASSERT_NE(test, nullptr); + const testing::TestInfo *testInfo = test->current_test_info(); + ASSERT_NE(testInfo, nullptr); + LOGI("Start unit test: %s.%s", testInfo->test_case_name(), testInfo->name()); +} + +int DistributedDBToolsUnitTest::BuildMessage(const DataSyncMessageInfo &messageInfo, + DistributedDB::Message *&message) +{ + auto packet = new (std::nothrow) DataRequestPacket; + if (packet == nullptr) { + return -E_OUT_OF_MEMORY; + } + message = new (std::nothrow) Message(messageInfo.messageId_); + if (message == nullptr) { + delete packet; + packet = nullptr; + return -E_OUT_OF_MEMORY; + } + packet->SetBasicInfo(messageInfo.sendCode_, messageInfo.version_, messageInfo.mode_); + packet->SetWaterMark(messageInfo.localMark_, messageInfo.peerMark_, messageInfo.deleteMark_); + std::vector reserved {messageInfo.packetId_}; + packet->SetReserved(reserved); + message->SetMessageType(messageInfo.messageType_); + message->SetSessionId(messageInfo.sessionId_); + message->SetSequenceId(messageInfo.sequenceId_); + message->SetExternalObject(packet); + return E_OK; +} } // namespace DistributedDBUnitTest diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h index 0b01f496ca94482dae6857ac0b934a6f72e4e84f..d6a8e2903252a2705449d10ddbdb9c4f957f2aaf 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/distributeddb_tools_unit_test.h @@ -16,49 +16,66 @@ #ifndef DISTRIBUTEDDB_TOOLS_UNIT_TEST_H #define DISTRIBUTEDDB_TOOLS_UNIT_TEST_H +#include +#include #include +#include +#include #include #include #include -#include -#include #include -#include -#include +#include "db_types.h" #include "kv_store_changed_data.h" #include "kv_store_delegate_impl.h" #include "kv_store_delegate_manager.h" -#include "kv_store_observer.h" #include "kv_store_nb_delegate.h" +#include "kv_store_observer.h" #include "kv_store_snapshot_delegate_impl.h" #include "log_print.h" -#include "types.h" -#include "db_types.h" -#include "sqlite_utils.h" +#include "message.h" +#include "query.h" #include "single_ver_kv_entry.h" #include "sqlite_single_ver_natural_store.h" +#include "sqlite_utils.h" +#include "sync_types.h" +#include "types.h" namespace DistributedDBUnitTest { static const int DIR_PERMISSION = 0750; struct DatabaseInfo { - std::string appId; - std::string userId; - std::string storeId; - std::string dir; + std::string appId{}; + std::string userId{}; + std::string storeId{}; + std::string dir{}; int dbUserVersion = 0; }; struct SyncInputArg { - uint64_t begin_; - uint64_t end_; - uint32_t blockSize_; + uint64_t begin_{}; + uint64_t end_{}; + uint32_t blockSize_{}; SyncInputArg(uint64_t begin, uint64_t end, uint32_t blockSize) : begin_(begin), end_(end), blockSize_(blockSize) {} }; +struct DataSyncMessageInfo { + int messageId_ = DistributedDB::INVALID_MESSAGE_ID; + uint16_t messageType_ = DistributedDB::TYPE_INVALID; + uint32_t sequenceId_ = 0; + uint32_t sessionId_ = 0; + int sendCode_ = DistributedDB::E_OK; + uint32_t version_ = 0; + int32_t mode_ = DistributedDB::PUSH; + DistributedDB::WaterMark localMark_ = 0; + DistributedDB::WaterMark peerMark_ = 0; + DistributedDB::WaterMark deleteMark_ = 0; + uint64_t packetId_ = 0; +}; + class DistributedDBToolsUnitTest final { public: DistributedDBToolsUnitTest() {} @@ -129,6 +146,11 @@ public: const std::vector& devices, DistributedDB::SyncMode mode, std::map& statuses, bool wait = false); + // sync test helper + DistributedDB::DBStatus SyncTest(DistributedDB::KvStoreNbDelegate* delegate, + const std::vector& devices, DistributedDB::SyncMode mode, + std::map& statuses, const DistributedDB::Query &query); + static void GetRandomKeyValue(std::vector &value, uint32_t defaultSize = 0); static bool IsValueEqual(const DistributedDB::Value &read, const DistributedDB::Value &origin); @@ -156,7 +178,8 @@ public: static int CreateMockMultiDb(DatabaseInfo &dbInfo, DistributedDB::OpenDbProperties &properties); - static int ModifyDatabaseFile(const std::string &fileDir); + static int ModifyDatabaseFile(const std::string &fileDir, uint64_t modifyPos = 0, + uint32_t modifyCnt = 256, uint32_t value = 0x1F1F1F1F); static int GetSyncDataTest(const SyncInputArg &syncInputArg, DistributedDB::SQLiteSingleVerNaturalStore *store, std::vector &dataItems, DistributedDB::ContinueToken &continueStmtToken); @@ -167,6 +190,10 @@ public: static int PutSyncDataTest(DistributedDB::SQLiteSingleVerNaturalStore *store, const std::vector &dataItems, const std::string &deviceName); + static int PutSyncDataTest(DistributedDB::SQLiteSingleVerNaturalStore *store, + const std::vector &dataItems, const std::string &deviceName, + const DistributedDB::QueryObject &query); + static int ConvertItemsToSingleVerEntry(const std::vector &dataItems, std::vector &entries); @@ -181,11 +208,18 @@ public: static int GetResourceDir(std::string& dir); + static int GetRandInt(const int randMin, const int randMax); + static int64_t GetRandInt64(const int64_t randMin, const int64_t randMax); + + static void PrintTestCaseInfo(); + + static int BuildMessage(const DataSyncMessageInfo &messageInfo, DistributedDB::Message *&message); + private: static int OpenMockMultiDb(DatabaseInfo &dbInfo, DistributedDB::OpenDbProperties &properties); - std::mutex syncLock_; - std::condition_variable syncCondVar_; + std::mutex syncLock_{}; + std::condition_variable syncCondVar_{}; }; class KvStoreObserverUnitTest : public DistributedDB::KvStoreObserver { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/evloop_timer_unit_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/evloop_timer_unit_test.cpp index 2eaf3881bf115e4e8f499bde36c51248ac7c820f..038ddbb9564b757d6dd2b96fdcf95a576ca4dbbf 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/evloop_timer_unit_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/common/evloop_timer_unit_test.cpp @@ -13,15 +13,16 @@ * limitations under the License. */ +#include #include #include -#include #include "db_errno.h" -#include "log_print.h" -#include "platform_specific.h" +#include "distributeddb_tools_unit_test.h" #include "evloop/include/ievent.h" #include "evloop/include/ievent_loop.h" +#include "log_print.h" +#include "platform_specific.h" using namespace testing::ext; using namespace DistributedDB; @@ -69,6 +70,7 @@ void DistributedDBEventLoopTimerTest::TearDownTestCase(void) {} void DistributedDBEventLoopTimerTest::SetUp(void) { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: Create a loop object. */ @@ -401,8 +403,8 @@ HWTEST_F(DistributedDBEventLoopTimerTest, EventLoopTimerTest007, TestSize.Level2 return -E_STALE; } lastTime = TimerTester::GetCurrentTime(); - int errCode = timer->SetTimeout(counter * TIME_PIECE_1000); - EXPECT_EQ(errCode, E_OK); + int ret = timer->SetTimeout(counter * TIME_PIECE_1000); + EXPECT_EQ(ret, E_OK); return E_OK; }, nullptr); EXPECT_EQ(errCode, E_OK); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp index 491612491d9fa509cc000f078f9dfdf081d265fd..3dac9b661ec5e06e301f0d4e69279063dd00e1ba 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.cpp @@ -14,15 +14,16 @@ */ #include "adapter_stub.h" -#include "log_print.h" #include "db_errno.h" #include "endian_convert.h" #include "frame_header.h" +#include "log_print.h" using namespace DistributedDB; namespace { const uint32_t MTU_SIZE = 5 * 1024 * 1024; // 5 M, 1024 is scale + const uint32_t TIME_OUT = 5 * 1000; // 5 S, 1000 is scale } /* @@ -53,6 +54,16 @@ uint32_t AdapterStub::GetMtuSize(const std::string &target) return GetMtuSize(); } +uint32_t AdapterStub::GetTimeout() +{ + return TIME_OUT; +} + +uint32_t AdapterStub::GetTimeout(const std::string &target) +{ + return GetTimeout(); +} + int AdapterStub::GetLocalIdentity(std::string &outTarget) { outTarget = localTarget_; @@ -111,6 +122,11 @@ int AdapterStub::RegSendableCallback(const SendableCallback &onSendable, const F return RegCallBack(onSendable, onSendableHandle_, inOper, onSendableFinalizer_); } + +bool AdapterStub::IsDeviceOnline(const std::string &device) +{ + return true; +} /* * Extended Part */ diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h index 5b39c4048c1d454dd27f644bdc75f15a88aaee53..e1e754474462bb18e8afcc0528927d65f2cbc9fe 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/adapter_stub.h @@ -36,6 +36,8 @@ public: void StopAdapter() override; uint32_t GetMtuSize() override; uint32_t GetMtuSize(const std::string &target) override; + uint32_t GetTimeout() override; + uint32_t GetTimeout(const std::string &target) override; int GetLocalIdentity(std::string &outTarget) override; int SendBytes(const std::string &dstTarget, const uint8_t *bytes, uint32_t length) override; @@ -44,6 +46,8 @@ public: int RegTargetChangeCallback(const TargetChangeCallback &onChange, const Finalizer &inOper) override; int RegSendableCallback(const SendableCallback &onSendable, const Finalizer &inOper) override; + bool IsDeviceOnline(const std::string &device) override; + /* * Extended Part */ diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_common.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_common.cpp index 16f4a564e6e80a04f326f6299f26d7eced6f2c0e..2fcd18b37fa700adc1054e8286629ca437fa6ef5 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_common.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_common.cpp @@ -15,10 +15,10 @@ #include "distributeddb_communicator_common.h" #include -#include "securec.h" #include "db_errno.h" #include "log_print.h" #include "message_transform.h" +#include "securec.h" using namespace std; using namespace DistributedDB; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_deep_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_deep_test.cpp index 5e67864bb1a50a33769611348bc10befc8d9cdaf..d6bda1267a7719563f4a2c31fa9ebbfa12f1bc1d 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_deep_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_deep_test.cpp @@ -15,13 +15,13 @@ #include #include -#include #include -#include "message.h" #include "db_errno.h" +#include "distributeddb_communicator_common.h" +#include "distributeddb_tools_unit_test.h" #include "log_print.h" +#include "message.h" #include "serial_buffer.h" -#include "distributeddb_communicator_common.h" using namespace std; using namespace testing::ext; @@ -113,6 +113,7 @@ void ReleaseAllCommunicator() void DistributedDBCommunicatorDeepTest::SetUp() { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: Alloc communicator AA, AB, BB, BC, CC, CA */ diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_send_receive_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_send_receive_test.cpp index 8b781d55a65b98366fbf1ab665cfd59988828d71..9fe08435dca6b1948b290db8c3d7fe93800bac18 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_send_receive_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_send_receive_test.cpp @@ -14,12 +14,12 @@ */ #include -#include #include -#include "message.h" -#include "log_print.h" #include "db_errno.h" #include "distributeddb_communicator_common.h" +#include "distributeddb_tools_unit_test.h" +#include "log_print.h" +#include "message.h" using namespace std; using namespace testing::ext; @@ -71,6 +71,7 @@ void DistributedDBCommunicatorSendReceiveTest::TearDownTestCase(void) void DistributedDBCommunicatorSendReceiveTest::SetUp() { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: Alloc communicator AA, BA, BB */ diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_test.cpp index 6afabe1589c31d5a818c4508ff4b8290dff78936..57bac997efc0167af773e3b6376e577ffa3d24cd 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/communicator/distributeddb_communicator_test.cpp @@ -14,12 +14,12 @@ */ #include -#include #include -#include "log_print.h" #include "db_errno.h" -#include "endian_convert.h" #include "distributeddb_communicator_common.h" +#include "distributeddb_tools_unit_test.h" +#include "endian_convert.h" +#include "log_print.h" using namespace std; using namespace testing::ext; @@ -80,9 +80,7 @@ void DistributedDBCommunicatorTest::TearDownTestCase(void) void DistributedDBCommunicatorTest::SetUp() { - /** - * @tc.setup: Do nothing - */ + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); } void DistributedDBCommunicatorTest::TearDown() diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_auto_launch_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_auto_launch_test.cpp index cbc1d03172ac683f01e182c8a9dce3f99dbe3dce..abad6458f90f68cd73686568c9ce2b6712cce393 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_auto_launch_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_auto_launch_test.cpp @@ -13,19 +13,18 @@ * limitations under the License. */ +#include #include - #include -#include -#include "db_errno.h" -#include "runtime_context.h" #include "db_common.h" #include "db_constant.h" +#include "db_errno.h" +#include "distributeddb_tools_unit_test.h" +#include "kv_store_delegate_manager.h" #include "kvdb_manager.h" #include "kvdb_pragma.h" -#include "kv_store_delegate_manager.h" -#include "distributeddb_tools_unit_test.h" +#include "runtime_context.h" using namespace testing::ext; using namespace DistributedDB; @@ -169,6 +168,7 @@ void DistributedDBInterfacesAutoLaunchTest::TearDownTestCase(void) void DistributedDBInterfacesAutoLaunchTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvStoreStatus = INVALID_ARGS; g_kvStore = nullptr; } @@ -324,10 +324,7 @@ void GetSyncData(const std::string &storeId) ContinueToken token = nullptr; DataSizeSpecInfo syncDataSizeInfo = {DBConstant::MAX_VALUE_SIZE, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; kvStore->GetSyncData(0, UINT64_MAX / 2, entries, token, syncDataSizeInfo); // half of the max timestamp. - for (auto &item : entries) { - kvStore->ReleaseKvEntry(item); - item = nullptr; - } + SingleVerKvEntry::Release(entries); if (token != nullptr) { kvStore->ReleaseContinueToken(token); } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_syncdb_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_syncdb_test.cpp index 57277217a89a037481556da627c44c9646db58ce..4f44a1c9848aa613f6401cc17fd1e21b44f3f187 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_syncdb_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_syncdb_test.cpp @@ -13,10 +13,9 @@ * limitations under the License. */ -#include - #include #include +#include #include #include "db_common.h" @@ -109,6 +108,7 @@ void DistributedDBInterfacesDataOperationSyncDBTest::TearDownTestCase(void) void DistributedDBInterfacesDataOperationSyncDBTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); // init values. g_valueStatus = INVALID_ARGS; g_value.clear(); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp index c7d6036e2ac276cf52a371ebd359a45fa42ce454..5956d322b36cafc512a8819391a022e5036a8ba8 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_data_operation_test.cpp @@ -362,6 +362,7 @@ void DistributedDBInterfacesDataOperationTest::TearDownTestCase(void) void DistributedDBInterfacesDataOperationTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); // init values. g_valueStatus = INVALID_ARGS; g_value.clear(); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_corrupt_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_corrupt_test.cpp index eadd167b50283b047080555aee802c05346a340d..795d7e661c037377ca03ce2804d2c0ebffe29baf 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_corrupt_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_corrupt_test.cpp @@ -15,9 +15,8 @@ #include #include -#include - #include +#include #include "db_common.h" #include "db_constant.h" @@ -78,13 +77,13 @@ namespace { DistributedDBToolsUnitTest::GetRandomKeyValue(value); DBStatus status = OK; if (kvDelegate != nullptr) { - DBStatus status = kvDelegate->Put(key, value); + status = kvDelegate->Put(key, value); if (status != OK) { return status; } } if (kvNbDelegate != nullptr) { - DBStatus status = kvNbDelegate->Put(key, value); + status = kvNbDelegate->Put(key, value); if (status != OK) { return status; } @@ -117,6 +116,7 @@ void DistributedDBInterfacesDatabaseCorruptTest::TearDownTestCase(void) void DistributedDBInterfacesDatabaseCorruptTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvDelegatePtr = nullptr; } @@ -352,3 +352,154 @@ HWTEST_F(DistributedDBInterfacesDatabaseCorruptTest, DatabaseCorruptionHandleTes EXPECT_EQ(g_mgr.DeleteKvStore("corrupt6"), OK); EXPECT_EQ(g_mgr.DeleteKvStore("corrupt7"), OK); } + +namespace { +const uint32_t MODIFY_SIZE = 12; // Modify size is 12 * sizeof(uint32_t); +const uint32_t MODIFY_VALUE = 0xF3F3F3F3; // random value, make sure to destroy the page header. +void TestDatabaseIntegrityCheckOption(const std::string &storeId, bool isEncrypted) +{ + KvStoreNbDelegate::Option nbOption = {true, false, isEncrypted}; + nbOption.isNeedIntegrityCheck = false; + nbOption.isNeedRmCorruptedDb = false; + if (isEncrypted) { + Key randPassword; + DistributedDBToolsUnitTest::GetRandomKeyValue(randPassword, PASSWD_SIZE); + ASSERT_EQ(nbOption.passwd.SetValue(randPassword.data(), randPassword.size()), CipherPassword::ErrorCode::OK); + } + auto filePath = GetKvStoreDirectory(storeId, DBConstant::DB_TYPE_SINGLE_VER); + g_mgr.GetKvStore(storeId, nbOption, g_kvNbDelegateCallback); + ASSERT_EQ(g_kvNbDelegateStatus, OK); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + ASSERT_EQ(PutDataIntoDatabase(nullptr, g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + + /** + * @tc.steps: step1. Modify the database file header to destroy the header and call the GetKvStore. + * @tc.expected: step1. Returns null kv store and the errCode is INVALID_PASSWD_OR_CORRUPTED_DB. + */ + DistributedDBToolsUnitTest::ModifyDatabaseFile(filePath, 0, MODIFY_SIZE, MODIFY_VALUE); + g_mgr.GetKvStore(storeId, nbOption, g_kvNbDelegateCallback); + ASSERT_EQ(g_kvNbDelegateStatus, INVALID_PASSWD_OR_CORRUPTED_DB); + ASSERT_TRUE(g_kvNbDelegatePtr == nullptr); + + /** + * @tc.steps: step2. call the GetKvStore with integrity check option is true. + * @tc.expected: step2. Returns null kv store and the errCode is INVALID_PASSWD_OR_CORRUPTED_DB. + */ + nbOption.isNeedIntegrityCheck = true; + g_mgr.GetKvStore(storeId, nbOption, g_kvNbDelegateCallback); + ASSERT_EQ(g_kvNbDelegateStatus, INVALID_PASSWD_OR_CORRUPTED_DB); + ASSERT_TRUE(g_kvNbDelegatePtr == nullptr); + + /** + * @tc.steps: step3. call the GetKvStore with remove corrupted database option is true. + * @tc.expected: step3. Returns non-null kv store and the errCode is OK. + */ + nbOption.isNeedIntegrityCheck = false; + nbOption.isNeedRmCorruptedDb = true; + g_mgr.GetKvStore(storeId, nbOption, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + + ASSERT_EQ(PutDataIntoDatabase(nullptr, g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + + /** + * @tc.steps: step4. Modify the second page of the database file and Get the kv store. + * @tc.expected: step4. Returns non-null kv store and the errCode is OK(GetKvStore skip the check of the page 2). + */ + size_t filePos = isEncrypted ? 1024 : 4096; // 1024 and 4096 is the page size. + DistributedDBToolsUnitTest::ModifyDatabaseFile(filePath, filePos, MODIFY_SIZE, MODIFY_VALUE); + nbOption.isNeedRmCorruptedDb = false; + g_mgr.GetKvStore(storeId, nbOption, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + + /** + * @tc.steps: step5. Get the kv store with check the integrity. + * @tc.expected: step5. Returns null kv store and the errCode is INVALID_PASSWD_OR_CORRUPTED_DB. + */ + nbOption.isNeedIntegrityCheck = true; + g_mgr.GetKvStore(storeId, nbOption, g_kvNbDelegateCallback); + ASSERT_EQ(g_kvNbDelegateStatus, INVALID_PASSWD_OR_CORRUPTED_DB); + ASSERT_TRUE(g_kvNbDelegatePtr == nullptr); + + /** + * @tc.steps: step5. Get the kv store with check the integrity and the rm corrupted database option. + * @tc.expected: step5. Returns non-null kv store and the errCode is OK. + */ + nbOption.isNeedRmCorruptedDb = true; + g_mgr.GetKvStore(storeId, nbOption, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore(storeId), OK); +} +} + +/** + * @tc.name: DatabaseIntegrityCheck001 + * @tc.desc: Test the integrity check option. + * @tc.type: FUNC + * @tc.require: AR000D487C SR000D4878 + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBInterfacesDatabaseCorruptTest, DatabaseIntegrityCheck001, TestSize.Level2) +{ + LOGI("Begin to check the unencrypted database"); + TestDatabaseIntegrityCheckOption("integrity_check001", false); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + LOGI("Begin to check the encrypted database"); + TestDatabaseIntegrityCheckOption("integrity_check002", true); +} + +/** + * @tc.name: DatabaseIntegrityCheck002 + * @tc.desc: Test the integrity check interface. + * @tc.type: FUNC + * @tc.require: AR000D487C SR000D4878 + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBInterfacesDatabaseCorruptTest, DatabaseIntegrityCheck002, TestSize.Level1) +{ + CipherPassword passwd; + KvStoreNbDelegate::Option nbOption = {true, false, false, CipherType::DEFAULT, passwd}; + nbOption.isNeedIntegrityCheck = true; + nbOption.isNeedRmCorruptedDb = true; + auto filePath = GetKvStoreDirectory("integrity021", DBConstant::DB_TYPE_SINGLE_VER); + for (uint32_t i = 1; i < 4; i++) { + LOGI("%u th test!", i); + g_mgr.GetKvStore("integrity021", nbOption, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + ASSERT_EQ(g_kvNbDelegatePtr->CheckIntegrity(), OK); + ASSERT_EQ(PutDataIntoDatabase(nullptr, g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + DistributedDBToolsUnitTest::ModifyDatabaseFile(filePath, i * 4096, MODIFY_SIZE, MODIFY_VALUE); // page size 4096 + g_mgr.GetKvStore("integrity021", nbOption, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_EQ(g_kvNbDelegatePtr->CheckIntegrity(), OK); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("integrity021"), OK); + } + LOGI("Begin the encrypted check"); + Key randomPassword; + DistributedDBToolsUnitTest::GetRandomKeyValue(randomPassword, PASSWD_SIZE); + int errCode = passwd.SetValue(randomPassword.data(), randomPassword.size()); + ASSERT_EQ(errCode, CipherPassword::ErrorCode::OK); + nbOption = {true, false, true, CipherType::DEFAULT, passwd}; + nbOption.isNeedIntegrityCheck = true; + nbOption.isNeedRmCorruptedDb = true; + filePath = GetKvStoreDirectory("integrity022", DBConstant::DB_TYPE_SINGLE_VER); + for (uint32_t i = 1; i < 4; i++) { + LOGI("%u th test the encrypted database!", i); + g_mgr.GetKvStore("integrity022", nbOption, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + ASSERT_EQ(g_kvNbDelegatePtr->CheckIntegrity(), OK); + ASSERT_EQ(PutDataIntoDatabase(nullptr, g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + DistributedDBToolsUnitTest::ModifyDatabaseFile(filePath, i * 1024, MODIFY_SIZE, MODIFY_VALUE); // page size 1024 + g_mgr.GetKvStore("integrity022", nbOption, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_EQ(g_kvNbDelegatePtr->CheckIntegrity(), OK); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + EXPECT_EQ(g_mgr.DeleteKvStore("integrity022"), OK); + } +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_test.cpp index 6057dd90d47b8da3ec9d808392c80d7ac5a23bbf..bc2d8abbf29549d6dfa78225f6b85a382dfbddb0 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_database_test.cpp @@ -13,17 +13,17 @@ * limitations under the License. */ -#include #include - +#include #include + #include "db_common.h" -#include "platform_specific.h" -#include "kvdb_manager.h" #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" -#include "runtime_context.h" +#include "kvdb_manager.h" +#include "platform_specific.h" #include "process_system_api_adapter_impl.h" +#include "runtime_context.h" using namespace testing::ext; using namespace DistributedDB; @@ -328,19 +328,22 @@ void DistributedDBInterfacesDatabaseTest::SetUpTestCase(void) void DistributedDBInterfacesDatabaseTest::TearDownTestCase(void) { - if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { - LOGE("rm test db files error!"); - } RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(nullptr); } void DistributedDBInterfacesDatabaseTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvDelegatePtr = nullptr; } -void DistributedDBInterfacesDatabaseTest::TearDown(void) {} +void DistributedDBInterfacesDatabaseTest::TearDown(void) +{ + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error!"); + } +} /** * @tc.name: GetKvStore001 @@ -385,7 +388,7 @@ HWTEST_F(DistributedDBInterfacesDatabaseTest, GetKvStore001, TestSize.Level1) option = {false, true, false}; g_mgr.GetKvStore("distributed_db_test3", option, g_kvDelegateCallback); ASSERT_TRUE(g_kvDelegatePtr == nullptr); - EXPECT_TRUE(g_kvDelegateStatus == DB_ERROR); + EXPECT_NE(g_kvDelegateStatus, OK); /** * @tc.steps: step6. Obtain the kvStore through the GetKvStore interface of the delegate manager @@ -395,7 +398,7 @@ HWTEST_F(DistributedDBInterfacesDatabaseTest, GetKvStore001, TestSize.Level1) option = {false, false, false}; g_mgr.GetKvStore("distributed_db_test4", option, g_kvDelegateCallback); ASSERT_TRUE(g_kvDelegatePtr == nullptr); - EXPECT_TRUE(g_kvDelegateStatus == DB_ERROR); + EXPECT_NE(g_kvDelegateStatus, OK); /** * @tc.steps: step7. Obtain the kvStore through the GetKvStore interface of the delegate manager @@ -1294,6 +1297,26 @@ namespace { } } +/** + * @tc.name: FreqOpenCloseDel001 + * @tc.desc: Open/close/delete the kv store concurrently. + * @tc.type: FUNC + * @tc.require: AR000DR9K2 + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBInterfacesDatabaseTest, FreqOpenCloseDel001, TestSize.Level2) +{ + std::string storeId = "FrqOpenCloseDelete001"; + std::thread t1(OpenCloseDatabase, storeId); + std::thread t2([&]() { + for (int i = 0; i < 10000; i++) { + g_mgr.DeleteKvStore(storeId); + } + }); + t1.join(); + t2.join(); +} + /** * @tc.name: FreqOpenClose001 * @tc.desc: Open and close the kv store concurrently. @@ -1346,6 +1369,68 @@ HWTEST_F(DistributedDBInterfacesDatabaseTest, CheckKvStoreDir001, TestSize.Level g_mgr.CloseKvStore(g_kvNbDelegatePtr); g_kvNbDelegatePtr = nullptr; EXPECT_EQ(g_mgr.DeleteKvStore(storeId), OK); - LOGE("[%s]", dataBaseDir.c_str()); + LOGI("[%s]", dataBaseDir.c_str()); ASSERT_EQ(OS::CheckPathExistence(dataBaseDir), false); } + +/** + * @tc.name: CompressionRate1 + * @tc.desc: Open the kv store with invalid compressionRate and open successfully. + * @tc.type: FUNC + * @tc.require: AR000G3QTT + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBInterfacesDatabaseTest, CompressionRate1, TestSize.Level1) +{ + /** + * @tc.steps: step1. Open the kv store with the option that comressionRate is invalid. + * @tc.expected: step1. Open kv store successfully. Returns OK. + */ + KvStoreNbDelegate::Option option; + option.isNeedCompressOnSync = true; + option.compressionRate = 0; // 0 is invalid. + const std::string storeId("CompressionRate1"); + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + + g_mgr.CloseKvStore(g_kvNbDelegatePtr); + g_kvNbDelegatePtr = nullptr; + EXPECT_EQ(g_mgr.DeleteKvStore(storeId), OK); +} + +HWTEST_F(DistributedDBInterfacesDatabaseTest, DataInterceptor1, TestSize.Level1) +{ + /** + * @tc.steps: step1. Open the kv store with the option that comressionRate is invalid. + * @tc.expected: step1. Open kv store successfully. Returns OK. + */ + KvStoreNbDelegate::Option option; + const std::string storeId("DataInterceptor1"); + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + + g_kvNbDelegatePtr->SetPushDataInterceptor( + [](InterceptedData &data, const std::string &sourceID, const std::string &targetID) { + int errCode = OK; + auto entries = data.GetEntries(); + for (size_t i = 0; i < entries.size(); i++) { + if (entries[i].key.empty() || entries[i].key.at(0) != 'A') { + continue; + } + auto newKey = entries[i].key; + newKey[0] = 'B'; + errCode = data.ModifyKey(i, newKey); + if (errCode != OK) { + break; + } + } + return errCode; + } + ); + + g_mgr.CloseKvStore(g_kvNbDelegatePtr); + g_kvNbDelegatePtr = nullptr; + EXPECT_EQ(g_mgr.DeleteKvStore(storeId), OK); +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_device_identifier_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_device_identifier_test.cpp index 77526cd26d06e8f5cf45dc2c8122bd033a13fa7c..ea23f80efbbe9812594e29f59f340f1a1898a9bf 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_device_identifier_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_device_identifier_test.cpp @@ -21,9 +21,9 @@ #include "db_errno.h" #include "distributeddb_data_generate_unit_test.h" #include "kv_store_nb_delegate_impl.h" +#include "platform_specific.h" #include "sqlite_single_ver_natural_store.h" #include "sqlite_single_ver_natural_store_connection.h" -#include "platform_specific.h" using namespace testing::ext; using namespace DistributedDB; @@ -78,6 +78,7 @@ void DistributedDBDeviceIdentifierTest::TearDownTestCase(void) void DistributedDBDeviceIdentifierTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); KvStoreNbDelegate::Option option = {true, false, false}; g_mgr.GetKvStore(STORE_ID, option, g_kvNbDelegateCallback); EXPECT_TRUE(g_kvDelegateStatus == OK); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_encrypt_database_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_encrypt_database_test.cpp index 6f954e0ea1a6b4770cb64bfa3c8605e9ed238e0e..31c815784ffd8399c0073e7621901a5edf32a94b 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_encrypt_database_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_encrypt_database_test.cpp @@ -60,6 +60,7 @@ void DistributedDBInterfacesEncryptDatabaseTest::TearDownTestCase(void) void DistributedDBInterfacesEncryptDatabaseTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); } void DistributedDBInterfacesEncryptDatabaseTest::TearDown(void) diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_encrypt_delegate_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_encrypt_delegate_test.cpp index 52f6b0373d7c48338456fdce659c446c4b80e7d6..75d4f009017d19bac70e2d272eb91eaebce38f50 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_encrypt_delegate_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_encrypt_delegate_test.cpp @@ -17,11 +17,10 @@ #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" -#include "sqlite_utils.h" -#include "sqlite_single_ver_natural_store.h" -#include "db_errno.h" -#include "log_print.h" #include "kv_store_nb_conflict_data.h" +#include "log_print.h" +#include "sqlite_single_ver_natural_store.h" +#include "sqlite_utils.h" using namespace testing::ext; using namespace DistributedDB; @@ -131,6 +130,7 @@ void DistributedDBInterfacesEncryptDelegateTest::TearDownTestCase(void) void DistributedDBInterfacesEncryptDelegateTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_errCode = INVALID_ARGS; g_kvNbDelegatePtr = nullptr; g_kvDelegatePtr = nullptr; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp index ed731588cfdb9c13f0a2f39dd980b0a7be045524..f0e38770f37755d184b3cc7c580f9a97ca563db3 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_import_and_export_test.cpp @@ -14,7 +14,6 @@ */ #ifndef OMIT_ENCRYPT #include -#include #include "distributeddb_data_generate_unit_test.h" #include "platform_specific.h" @@ -121,6 +120,7 @@ void DistributedDBInterfacesImportAndExportTest::TearDownTestCase(void) void DistributedDBInterfacesImportAndExportTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_junkFilesList.clear(); g_kvDelegateStatus = INVALID_ARGS; g_kvNbDelegatePtr = nullptr; @@ -344,7 +344,7 @@ HWTEST_F(DistributedDBInterfacesImportAndExportTest, ExportParameterCheck001, Te * @tc.steps: step1. The filePath path does not exist. * @tc.expected: step1. Return INVALID_ARGS. */ - std::string invalidFileName = g_exportFileDir + "/jadaksdjadkjsa/" + "/ExportParameterCheck001.$$"; + std::string invalidFileName = g_exportFileDir + "/tempNotCreated/" + "/ExportParameterCheck001.$$"; CipherPassword passwd; EXPECT_EQ(g_kvNbDelegatePtr->Export(invalidFileName, passwd), INVALID_ARGS); @@ -411,7 +411,7 @@ HWTEST_F(DistributedDBInterfacesImportAndExportTest, ExportParameterCheck002, Te * @tc.steps: step1. The filePath path does not exist. * @tc.expected: step1. Return INVALID_ARGS. */ - std::string invalidExportFileName = g_exportFileDir + "/jadaksdjadkjsa/" + "/ExportParameterCheck002.$$"; + std::string invalidExportFileName = g_exportFileDir + "/tempNotCreated/" + "/ExportParameterCheck002.$$"; CipherPassword passwd; EXPECT_EQ(g_kvDelegatePtr->Export(invalidExportFileName, passwd), INVALID_ARGS); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_index_unit_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_index_unit_test.cpp index 561d04573e6f8e9951c846086cb5ce2eee4d88e0..dd7e39e7cd2c031ca2756e9b94374bc3ae8ffd9d 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_index_unit_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_index_unit_test.cpp @@ -16,12 +16,12 @@ #ifndef OMIT_JSON #include #include -#include "sqlite_import.h" -#include "distributeddb_tools_unit_test.h" -#include "query.h" #include "db_common.h" #include "db_constant.h" +#include "distributeddb_tools_unit_test.h" +#include "query.h" #include "schema_utils.h" +#include "sqlite_import.h" #include "sqlite_local_kvdb_connection.h" using namespace testing::ext; @@ -67,7 +67,7 @@ namespace { return filePath; } - // Query database schema related info + // Query sqlite_master related const string SQL_QUERY_INDEX = "SELECT COUNT(*) FROM sqlite_master where type = 'index' and name = "; int CallbackReturnCount(void *data, int argc, char **argv, char **azColName) { @@ -245,7 +245,7 @@ class DistributedDBInterfacesIndexUnitTest : public testing::Test { public: static void SetUpTestCase(void); static void TearDownTestCase(void); - void SetUp() {}; + void SetUp(); void TearDown() {}; }; @@ -263,6 +263,11 @@ void DistributedDBInterfacesIndexUnitTest::TearDownTestCase(void) } } +void DistributedDBInterfacesIndexUnitTest::SetUp() +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); +} + namespace { void PrepareInfoForCrudIndex001() { @@ -513,7 +518,7 @@ HWTEST_F(DistributedDBInterfacesIndexUnitTest, CreateIndex001, TestSize.Level1) ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); EXPECT_EQ(g_kvDelegateStatus, OK); /** - * @tc.steps:step2. Use the sql statement to get each index count count from the schema table; + * @tc.steps:step2. Use the sql statement to get each index count count from the sqlite_master table; * @tc.expected: step2. count == 1. */ EXPECT_EQ(sqlite3_open_v2(filePath.c_str(), &db, SQLITE_OPEN_READWRITE, nullptr), SQLITE_OK); @@ -583,7 +588,7 @@ HWTEST_F(DistributedDBInterfacesIndexUnitTest, CreateIndex002, TestSize.Level1) ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); EXPECT_EQ(g_kvDelegateStatus, OK); /** - * @tc.steps:step2. Use the sql statement to get each index count count from the schema table; + * @tc.steps:step2. Use the sql statement to get each index count count from the sqlite_master table; * @tc.expected: step2. count == 1. */ EXPECT_EQ(sqlite3_open_v2(filePath.c_str(), &db, SQLITE_OPEN_READWRITE, nullptr), SQLITE_OK); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_local_batch_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_local_batch_test.cpp index df424a19fac5a94f0392385de2d5a925a6cab45c..50a53dbbe4ead2514a4444e93c523744047969e1 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_local_batch_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_local_batch_test.cpp @@ -16,11 +16,11 @@ #include #include -#include "db_errno.h" #include "db_constant.h" -#include "log_print.h" +#include "db_errno.h" #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" +#include "log_print.h" #include "sqlite_single_ver_natural_store.h" using namespace testing::ext; @@ -111,6 +111,7 @@ void DistributedDBInterfacesNBDelegateLocalBatchTest::TearDownTestCase(void) void DistributedDBInterfacesNBDelegateLocalBatchTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvNbDelegatePtr = nullptr; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_schema_put_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_schema_put_test.cpp index 844f2d6098495e8216d49d73b28d92d1f99d2d5d..7cebfcaeddcdeb413ed1a6ce865f5aab335fde2f 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_schema_put_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_schema_put_test.cpp @@ -14,10 +14,9 @@ */ #ifndef OMIT_JSON +#include #include - #include -#include #include "distributeddb_tools_unit_test.h" #include "kv_store_delegate_manager.h" @@ -190,7 +189,9 @@ void DistributedDBInterfacesNBDelegateSchemaPutTest::TearDownTestCase(void) } void DistributedDBInterfacesNBDelegateSchemaPutTest::SetUp(void) -{} +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); +} void DistributedDBInterfacesNBDelegateSchemaPutTest::TearDown(void) { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp index 6bf5c1c632670f9e081329b2d229732af00aa232..65de8fe63dcbaa8a9096d999bf6162efaa92ec4d 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_delegate_test.cpp @@ -16,18 +16,16 @@ #include #include +#include "db_common.h" +#include "db_constant.h" +#include "db_errno.h" #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" -#include "sqlite_utils.h" -#include "sqlite_single_ver_natural_store.h" -#include "db_errno.h" #include "log_print.h" -#include "db_common.h" -#include "db_constant.h" -#include "kv_store_nb_conflict_data.h" -#include "runtime_context.h" -#include "process_system_api_adapter_impl.h" #include "platform_specific.h" +#include "process_system_api_adapter_impl.h" +#include "runtime_context.h" +#include "sqlite_single_ver_natural_store.h" using namespace testing::ext; using namespace DistributedDB; @@ -208,6 +206,7 @@ void DistributedDBInterfacesNBDelegateTest::TearDownTestCase(void) void DistributedDBInterfacesNBDelegateTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvNbDelegatePtr = nullptr; g_kvDelegatePtr = nullptr; @@ -220,10 +219,6 @@ void DistributedDBInterfacesNBDelegateTest::TearDown(void) g_kvNbDelegatePtr = nullptr; } RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(nullptr); -#ifdef HW_USING_LABEL_FUNC_STUB - sqlite3_release_label_info(); - sqlite3_set_lock_status(UNLOCKED); -#endif } /** @@ -1784,162 +1779,4 @@ HWTEST_F(DistributedDBInterfacesNBDelegateTest, SingleVerGetSecurityOption002, T EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); g_kvNbDelegatePtr = nullptr; EXPECT_TRUE(g_mgr.DeleteKvStore("SingleVerGetSecurityOption002") == OK); -} - -#ifdef HW_USING_LABEL_FUNC_STUB -namespace { -void SetSqliteLabel(const std::string &storeId, const SecurityOption &secOption) -{ - std::string identifier = DBCommon::TransferHashString(USER_ID + "-" + APP_ID + "-" + storeId); - std::string identifierDir = DBCommon::TransferStringToHex(identifier); - - std::string s3SeceDir = g_testDir + "/" + identifierDir + "/single_ver/"; - sqlite3_set_label_info((s3SeceDir + "main").c_str(), secOption.securityLabel, secOption.securityFlag); - sqlite3_set_label_info((s3SeceDir + "cache").c_str(), secOption.securityLabel, secOption.securityFlag); - sqlite3_set_label_info((s3SeceDir + "meta").c_str(), SecurityLabel::S2, SecurityFlag::ECE); -} - -void ResetSqliteLabel(std::shared_ptr &adapter) -{ - sqlite3_release_label_info(); - sqlite3_init_label_info(); - if (adapter != nullptr) { - adapter.reset(); - } - - adapter = std::make_shared(); - RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(adapter); -} - -void CreateDiffSecOptDb(const std::string &storeId, LockState lockState = UNLOCKED) -{ - if (lockState == LOCKED) { - LOGD("Is locked state!"); - } - - sqlite3_set_lock_status(lockState); - std::shared_ptr adapter = std::make_shared(); - RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(adapter); - - KvStoreNbDelegate::Option option; - SecurityOption secOption{SecurityLabel::S2, SecurityFlag::ECE}; - SetSqliteLabel(storeId, secOption); - option.secOption = secOption; - g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); - ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); - EXPECT_EQ(g_kvDelegateStatus, OK); - g_mgr.CloseKvStore(g_kvNbDelegatePtr); - g_mgr.DeleteKvStore(storeId); - ResetSqliteLabel(adapter); - - secOption = {SecurityLabel::S3, SecurityFlag::SECE}; - SetSqliteLabel(storeId, secOption); - option.secOption = secOption; - g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); - ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); - EXPECT_EQ(g_kvDelegateStatus, OK); - g_mgr.CloseKvStore(g_kvNbDelegatePtr); - std::string identifier = DBCommon::TransferHashString(USER_ID + "-" + APP_ID + "-" + storeId); - std::string hashIdentifier = DBCommon::TransferStringToHex(identifier); - std::string mainDbPath = g_testDir + "/" + hashIdentifier + "/" + DBConstant::SINGLE_SUB_DIR + "/" + - DBConstant::MAINDB_DIR +"/" + DBConstant::SINGLE_VER_DATA_STORE + DBConstant::SQLITE_DB_EXTENSION; - std::string cacheDbPath = g_testDir + "/" + hashIdentifier + "/" + DBConstant::SINGLE_SUB_DIR + "/" + - DBConstant::CACHEDB_DIR +"/" + DBConstant::SINGLE_VER_CACHE_STORE + DBConstant::SQLITE_DB_EXTENSION; - EXPECT_TRUE(OS::CheckPathExistence(mainDbPath)); - EXPECT_FALSE(OS::CheckPathExistence(cacheDbPath)); - g_mgr.DeleteKvStore(storeId); - ResetSqliteLabel(adapter); - - secOption = {SecurityLabel::S4, SecurityFlag::ECE}; - SetSqliteLabel(storeId, secOption); - option.secOption = secOption; - g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); - if (lockState == LOCKED) { - EXPECT_NE(g_kvDelegateStatus, OK); - return; - } - ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); - EXPECT_EQ(g_kvDelegateStatus, OK); - g_mgr.CloseKvStore(g_kvNbDelegatePtr); - g_mgr.DeleteKvStore(storeId); - ResetSqliteLabel(adapter); -} -} - -/** - * @tc.name: GetKvStoreDbInLockState - * @tc.desc: Test GetKvstore in different security option. - * @tc.type: FUNC - * @tc.require: AR000EV1G2 - * @tc.author: sunpeng - */ -HWTEST_F(DistributedDBInterfacesNBDelegateTest, GetKvStoreInDiffOption, TestSize.Level1) -{ - sqlite3_init_label_info(); - CreateDiffSecOptDb("distributed_nb_Diff_SecOpt_test"); -} - -/** - * @tc.name: GetKvStoreDbInLockState - * @tc.desc: Test GetKvstore in lock state. - * @tc.type: FUNC - * @tc.require: AR000EV1G2 - * @tc.author: sunpeng - */ -HWTEST_F(DistributedDBInterfacesNBDelegateTest, GetKvStoreDbInLockState, TestSize.Level1) -{ - std::shared_ptr adapter = std::make_shared(); - ResetSqliteLabel(adapter); - CreateDiffSecOptDb("GetKvStoreDbInLockState", LOCKED); -} - -/** - * @tc.name: GetKvStoreDbInLockState - * @tc.desc: Test GetKvstore in lock state. - * @tc.type: FUNC - * @tc.require: AR000EV1G2 - * @tc.author: sunpeng - */ -HWTEST_F(DistributedDBInterfacesNBDelegateTest, GetKvStoreCreateCacheDb, TestSize.Level1) -{ - std::shared_ptr adapter = std::make_shared(); - ResetSqliteLabel(adapter); - sqlite3_set_lock_status(UNLOCKED); - - std::string identifier = DBCommon::TransferHashString(USER_ID + "-" + APP_ID + "-" + "storeId"); - std::string hashIdentifier = DBCommon::TransferStringToHex(identifier); - std::string mainDbPath = g_testDir + "/" + hashIdentifier + "/" + DBConstant::SINGLE_SUB_DIR + "/" + - DBConstant::MAINDB_DIR +"/" + DBConstant::SINGLE_VER_DATA_STORE + DBConstant::SQLITE_DB_EXTENSION; - std::string cacheDbPath = g_testDir + "/" + hashIdentifier + "/" + DBConstant::SINGLE_SUB_DIR + "/" + - DBConstant::CACHEDB_DIR +"/" + DBConstant::SINGLE_VER_CACHE_STORE + DBConstant::SQLITE_DB_EXTENSION; - LOGE("Db path is [%s]", mainDbPath.c_str()); - - KvStoreNbDelegate::Option option; - SecurityOption secOption{SecurityLabel::S3, SecurityFlag::SECE}; - SetSqliteLabel("storeId", secOption); - option.secOption = secOption; - g_mgr.GetKvStore("storeId", option, g_kvNbDelegateCallback); - ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); - EXPECT_EQ(g_kvDelegateStatus, OK); - g_mgr.CloseKvStore(g_kvNbDelegatePtr); - - EXPECT_TRUE(OS::CheckPathExistence(mainDbPath)); - EXPECT_FALSE(OS::CheckPathExistence(cacheDbPath)); - - sqlite3_set_lock_status(LOCKED); - - g_mgr.GetKvStore("storeId", option, g_kvNbDelegateCallback); - ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); - EXPECT_TRUE(g_kvDelegateStatus == OK); - - EXPECT_TRUE(OS::CheckPathExistence(mainDbPath)); - EXPECT_TRUE(OS::CheckPathExistence(cacheDbPath)); - - EXPECT_EQ(g_mgr.DeleteKvStore("storeId"), BUSY); - EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); - EXPECT_EQ(g_mgr.DeleteKvStore("storeId"), OK); - - EXPECT_FALSE(OS::CheckPathExistence(mainDbPath)); - EXPECT_FALSE(OS::CheckPathExistence(cacheDbPath)); -} -#endif \ No newline at end of file +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_publish_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_publish_test.cpp index 3ee711a20d9dd57fb9dbbd2ae7eae06fe1442ca5..fb6dd6a3a514af0edce8735f2905f8f83ff4cc5d 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_publish_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_publish_test.cpp @@ -17,7 +17,6 @@ #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" -#include "db_errno.h" #include "log_print.h" using namespace testing::ext; @@ -110,6 +109,7 @@ void DistributedDBInterfacesNBPublishTest::TearDownTestCase(void) void DistributedDBInterfacesNBPublishTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvNbDelegatePtr = nullptr; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_transaction_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_transaction_test.cpp index 7957fbbf7bb39961ccdd38b9664d2606b2759eff..71ae88466c43691ef496618d330ff24ef91794c3 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_transaction_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_transaction_test.cpp @@ -15,11 +15,10 @@ #include +#include "db_common.h" #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" -#include "db_errno.h" #include "log_print.h" -#include "db_common.h" using namespace testing::ext; using namespace DistributedDB; @@ -147,6 +146,7 @@ void DistributedDBInterfacesNBTransactionTest::TearDownTestCase(void) void DistributedDBInterfacesNBTransactionTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvNbDelegatePtr = nullptr; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_unpublish_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_unpublish_test.cpp index 723b9ae154ee33464aabdc8b6dd3906c7e563bde..5958850d9377d7ec816783e8af90876d4888624c 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_unpublish_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_nb_unpublish_test.cpp @@ -98,6 +98,7 @@ void DistributedDBInterfacesNBUnpublishTest::TearDownTestCase(void) void DistributedDBInterfacesNBUnpublishTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); KvStoreNbDelegate::Option option = {true, false, false}; g_mgr.GetKvStore("unpublish_test", option, g_kvNbDelegateCallback); ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_query_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_query_test.cpp index 64f71f62eaa3277d7e2f0b05c65e3955d670115a..e0f097a5832d7a642a0bad7a7658ecb0da65aeef 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_query_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_query_test.cpp @@ -15,6 +15,7 @@ #include +#include "distributeddb_tools_unit_test.h" #include "get_query_info.h" #include "log_print.h" @@ -49,7 +50,7 @@ namespace { } template - std::list CraetCheckList(QueryObjType operFlag, const std::string &fieldName, const T &queryValue) + std::list CreateCheckList(QueryObjType operFlag, const std::string &fieldName, const T &queryValue) { FieldValue fieldValue; QueryValueType type = GetQueryValueType::GetFieldTypeAndValue(queryValue, fieldValue); @@ -78,32 +79,32 @@ namespace { void CheckQueryCompareOper() { Query query1 = Query::Select().NotEqualTo(TEST_FIELD_NAME, 123); // random test data - std::list result = CraetCheckList(QueryObjType::NOT_EQUALTO, TEST_FIELD_NAME, 123); + std::list result = CreateCheckList(QueryObjType::NOT_EQUALTO, TEST_FIELD_NAME, 123); // random num EXPECT_TRUE(CheckQueryContainer(query1, result)); Query query2 = Query::Select().EqualTo(TEST_FIELD_NAME, true); result.clear(); - result = CraetCheckList(QueryObjType::EQUALTO, TEST_FIELD_NAME, true); + result = CreateCheckList(QueryObjType::EQUALTO, TEST_FIELD_NAME, true); EXPECT_TRUE(CheckQueryContainer(query2, result)); Query query3 = Query::Select().GreaterThan(TEST_FIELD_NAME, 0); result.clear(); - result = CraetCheckList(QueryObjType::GREATER_THAN, TEST_FIELD_NAME, 0); + result = CreateCheckList(QueryObjType::GREATER_THAN, TEST_FIELD_NAME, 0); EXPECT_TRUE(CheckQueryContainer(query3, result)); Query query4 = Query::Select().LessThan(TEST_FIELD_NAME, INT_MAX); result.clear(); - result = CraetCheckList(QueryObjType::LESS_THAN, TEST_FIELD_NAME, INT_MAX); + result = CreateCheckList(QueryObjType::LESS_THAN, TEST_FIELD_NAME, INT_MAX); EXPECT_TRUE(CheckQueryContainer(query4, result)); Query query5 = Query::Select().GreaterThanOrEqualTo(TEST_FIELD_NAME, 1.56); // random test data result.clear(); - result = CraetCheckList(QueryObjType::GREATER_THAN_OR_EQUALTO, TEST_FIELD_NAME, 1.56); + result = CreateCheckList(QueryObjType::GREATER_THAN_OR_EQUALTO, TEST_FIELD_NAME, 1.56); // random test data EXPECT_TRUE(CheckQueryContainer(query5, result)); Query query6 = Query::Select().LessThanOrEqualTo(TEST_FIELD_NAME, 100); // random test data result.clear(); - result = CraetCheckList(QueryObjType::LESS_THAN_OR_EQUALTO, TEST_FIELD_NAME, 100); + result = CreateCheckList(QueryObjType::LESS_THAN_OR_EQUALTO, TEST_FIELD_NAME, 100); // random test data EXPECT_TRUE(CheckQueryContainer(query6, result)); } } @@ -126,6 +127,7 @@ void DistributedDBInterfacesQueryDBTest::TearDownTestCase(void) void DistributedDBInterfacesQueryDBTest::SetUp(void) { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); } void DistributedDBInterfacesQueryDBTest::TearDown(void) @@ -139,7 +141,7 @@ void DistributedDBInterfacesQueryDBTest::TearDown(void) * @tc.require: AR000DR9K6 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBInterfacesQueryDBTest, Query001, TestSize.Level0) +HWTEST_F(DistributedDBInterfacesQueryDBTest, Query001, TestSize.Level1) { Query query = Query::Select(); Query queryCopy = query; @@ -149,12 +151,12 @@ HWTEST_F(DistributedDBInterfacesQueryDBTest, Query001, TestSize.Level0) std::string testValue = "testValue"; Query query7 = Query::Select().Like(TEST_FIELD_NAME, testValue); - std::list result = CraetCheckList(QueryObjType::LIKE, TEST_FIELD_NAME, testValue); + std::list result = CreateCheckList(QueryObjType::LIKE, TEST_FIELD_NAME, testValue); EXPECT_TRUE(CheckQueryContainer(query7, result)); Query query8 = Query::Select().NotLike(TEST_FIELD_NAME, "testValue"); result.clear(); - result = CraetCheckList(QueryObjType::NOT_LIKE, TEST_FIELD_NAME, testValue); + result = CreateCheckList(QueryObjType::NOT_LIKE, TEST_FIELD_NAME, testValue); EXPECT_TRUE(CheckQueryContainer(query8, result)); vector fieldValues{1, 1, 1}; @@ -172,7 +174,7 @@ HWTEST_F(DistributedDBInterfacesQueryDBTest, Query001, TestSize.Level0) Query query11 = Query::Select().OrderBy(TEST_FIELD_NAME, false); result.clear(); - result = CraetCheckList(QueryObjType::ORDERBY, TEST_FIELD_NAME, false); + result = CreateCheckList(QueryObjType::ORDERBY, TEST_FIELD_NAME, false); EXPECT_TRUE(CheckQueryContainer(query11, result)); Query query12 = Query::Select().Limit(1, 2); @@ -194,7 +196,7 @@ HWTEST_F(DistributedDBInterfacesQueryDBTest, Query001, TestSize.Level0) * @tc.require: AR000DR9K6 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBInterfacesQueryDBTest, Query002, TestSize.Level0) +HWTEST_F(DistributedDBInterfacesQueryDBTest, Query002, TestSize.Level1) { float testValue = 1.1; Query query = Query::Select().NotEqualTo(".test", testValue); @@ -212,7 +214,7 @@ HWTEST_F(DistributedDBInterfacesQueryDBTest, Query002, TestSize.Level0) * @tc.require: AR000DR9K6 * @tc.author: sunpeng */ -HWTEST_F(DistributedDBInterfacesQueryDBTest, Query003, TestSize.Level0) +HWTEST_F(DistributedDBInterfacesQueryDBTest, Query003, TestSize.Level1) { Query query = Query::Select().EqualTo(TEST_FIELD_NAME, true).And().GreaterThan(TEST_FIELD_NAME, 1); QueryExpression queryExpression = GetQueryInfo::GetQueryExpression(query); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_register_syncdb_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_register_syncdb_test.cpp index aa3af7ec7420c6fc040ba71d5f6244cb91369cc2..cd6e357a6c624f4f0312d3ad1e4635fae1404258 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_register_syncdb_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_register_syncdb_test.cpp @@ -16,12 +16,11 @@ #include #include -#include "distributeddb_tools_unit_test.h" +#include "db_common.h" #include "distributeddb_data_generate_unit_test.h" -#include "kv_store_observer.h" -#include "securec.h" +#include "distributeddb_tools_unit_test.h" #include "platform_specific.h" -#include "db_common.h" +#include "securec.h" using namespace testing::ext; using namespace DistributedDB; @@ -144,6 +143,7 @@ void DistributedDBInterfacesRegisterSyncDBTest::TearDownTestCase(void) void DistributedDBInterfacesRegisterSyncDBTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /* * Here, we create STORE_ID before test, * and it will be closed in TearDown(). diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_resultset_performance.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_resultset_performance.cpp index 4ca43857b08c14963fb0d7ebf4d3f3530b5bb13b..560285b8c4ea4bfedc02e5f5db1d886333a92ec1 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_resultset_performance.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_resultset_performance.cpp @@ -15,11 +15,11 @@ #include #include -#include "types.h" +#include "distributeddb_tools_unit_test.h" #include "kv_store_delegate_manager.h" #include "kv_store_nb_delegate.h" -#include "distributeddb_tools_unit_test.h" #include "log_print.h" +#include "types.h" using namespace testing::ext; using namespace DistributedDB; @@ -34,7 +34,7 @@ namespace { Key g_keyPrefix = {'A', 'B', 'C'}; const int BASE_NUMBER = 100000; - const int INSERT_NUMBER = 10000; + const int INSERT_NUMBER = 100; const int ENTRY_VALUE_SIZE = 3000; const int BATCH_ENTRY_NUMBER = 100; @@ -98,6 +98,7 @@ void DistributedDBInterfacesNBResultsetPerfTest::TearDownTestCase(void) void DistributedDBInterfacesNBResultsetPerfTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvNbDelegatePtr = nullptr; g_kvDelegatePtr = nullptr; @@ -139,14 +140,14 @@ HWTEST_F(DistributedDBInterfacesNBResultsetPerfTest, ResultSetPerfTest001, TestS Key keyGet = g_keyPrefix; keyGet.push_back('1'); - int offset = 5000; // offset 5000 + int offset = 40; // offset 40 LOGI("######## Query resultSet"); - Query query = Query::Select().PrefixKey(keyGet).Limit(128, offset); // limit 128 + Query query = Query::Select().PrefixKey(keyGet).Limit(50, offset); // limit 50 EXPECT_EQ(g_kvNbDelegatePtr->GetEntries(query, readResultSet), OK); ASSERT_TRUE(readResultSet != nullptr); LOGI("######## After get resultset"); int totalCount = readResultSet->GetCount(); - EXPECT_EQ(totalCount, 128); // limit 128 + EXPECT_EQ(totalCount, 50); // limit 50 LOGI("######## After get count:%d", totalCount); readResultSet->MoveToPosition(0); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_schema_database_upgrade_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_schema_database_upgrade_test.cpp index 42da5ec0cefdd2da7bad1e8d48ad9ce4868fe67d..6ca726e0114aaa1f6f04ab1feb8f1aefa554bcb8 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_schema_database_upgrade_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_schema_database_upgrade_test.cpp @@ -15,10 +15,10 @@ #ifndef OMIT_JSON #include -#include "kv_store_delegate_manager.h" #include "distributeddb_tools_unit_test.h" -#include "schema_utils.h" +#include "kv_store_delegate_manager.h" #include "schema_object.h" +#include "schema_utils.h" using namespace testing::ext; using namespace DistributedDB; @@ -164,6 +164,7 @@ void DistributedDBInterfacesSchemaDatabaseUpgradeTest::TearDownTestCase(void) void DistributedDBInterfacesSchemaDatabaseUpgradeTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvDelegatePtr = nullptr; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_single_version_result_set_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_single_version_result_set_test.cpp index eea5641f419f8770733c7866c2e506291b43d15d..87b8e8d31ea858459b69e4a289bae951c1e9f6f6 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_single_version_result_set_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_single_version_result_set_test.cpp @@ -16,19 +16,18 @@ #include #include -#include "distributeddb_data_generate_unit_test.h" -#include "kv_store_nb_delegate_impl.h" -#include "sqlite_single_ver_natural_store.h" -#include "sqlite_single_ver_natural_store_connection.h" #include "db_common.h" #include "db_constant.h" #include "db_types.h" -#include "result_entries_window.h" +#include "distributeddb_data_generate_unit_test.h" #include "ikvdb_raw_cursor.h" +#include "kv_store_nb_delegate_impl.h" #include "kvdb_manager.h" -#include "sqlite_local_kvdb_connection.h" -#include "sqlite_single_ver_forward_cursor.h" #include "platform_specific.h" +#include "result_entries_window.h" +#include "sqlite_single_ver_forward_cursor.h" +#include "sqlite_single_ver_natural_store.h" +#include "sqlite_single_ver_natural_store_connection.h" using namespace testing::ext; using namespace DistributedDB; @@ -95,6 +94,7 @@ void DistributedDBSingleVersionResultSetTest::TearDownTestCase(void) void DistributedDBSingleVersionResultSetTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); KvStoreNbDelegate::Option delegateOption = {true}; g_mgr.GetKvStore(STORE_ID, delegateOption, g_kvNbDelegateCallback); EXPECT_TRUE(g_kvDelegateStatus == OK); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_space_management_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_space_management_test.cpp index 0c5dfc601aebca67b5f2b30b7445d05c59adf973..89b76de93c2d6a8ee539ee09550003beb6a51c6f 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_space_management_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_space_management_test.cpp @@ -13,8 +13,8 @@ * limitations under the License. */ -#include #include +#include #include "db_constant.h" #include "db_common.h" @@ -125,6 +125,7 @@ void DistributedDBInterfacesSpaceManagementTest::TearDownTestCase(void) void DistributedDBInterfacesSpaceManagementTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvNbDelegatePtr = nullptr; g_kvDelegatePtr = nullptr; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_optimization_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_optimization_test.cpp index da78c34500f7269803527bb4139caf7b4469e36c..3f8e6078b7cfa09750fcca3da4580eda28fbd4f4 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_optimization_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_optimization_test.cpp @@ -16,11 +16,10 @@ #include #include -#include "db_errno.h" #include "db_constant.h" -#include "log_print.h" #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" +#include "log_print.h" using namespace testing::ext; using namespace DistributedDB; @@ -72,6 +71,7 @@ void DistributedDBInterfacesTransactionOptimizationTest::TearDownTestCase(void) void DistributedDBInterfacesTransactionOptimizationTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_kvDelegateStatus = INVALID_ARGS; g_kvNbDelegatePtr = nullptr; } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_syncdb_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_syncdb_test.cpp index c78d69dce198272aadf167f74b385881822676f1..765fab83fe861f2532ad5598bca4b752dbdbcc3f 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_syncdb_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_syncdb_test.cpp @@ -16,8 +16,8 @@ #include #include "distributeddb_data_generate_unit_test.h" -#include "distributeddb_tools_unit_test.h" #include "distributeddb_interfaces_transaction_testcase.h" +#include "distributeddb_tools_unit_test.h" using namespace testing::ext; using namespace DistributedDB; @@ -71,6 +71,7 @@ void DistributedDBInterfacesTransactionSyncDBTest::TearDownTestCase(void) void DistributedDBInterfacesTransactionSyncDBTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /* * Here, we create STORE_ID before test, * and it will be closed in TearDown(). diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_test.cpp index f42f0f84010b5419d148de3fefb54e54fe16180b..3dc576de19cf709903f52d5307370f87953906d7 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/distributeddb_interfaces_transaction_test.cpp @@ -15,8 +15,8 @@ #include -#include "distributeddb_interfaces_transaction_testcase.h" #include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_interfaces_transaction_testcase.h" #include "distributeddb_tools_unit_test.h" using namespace testing::ext; @@ -71,6 +71,7 @@ void DistributedDBInterfacesTransactionTest::TearDownTestCase(void) void DistributedDBInterfacesTransactionTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /* * Here, we create STORE_ID before test, * and it will be closed in TearDown(). diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.cpp index 29eb8fb4117587c34cece5dc1274b3682a6a9224..8b451b370eb38da4f1cdc1ae2de49de8d5af829b 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.cpp @@ -15,17 +15,30 @@ #include "process_system_api_adapter_impl.h" -#include #include +#include +#include "distributeddb_tools_unit_test.h" +#include "distributeddb_data_generate_unit_test.h" #include "log_print.h" #include "platform_specific.h" -#include "distributeddb_tools_unit_test.h" + +using namespace DistributedDBUnitTest; namespace DistributedDB { +namespace { + KvStoreDelegateManager g_mgr(APP_ID, USER_ID); + // define the g_kvDelegateCallback, used to get some information when open a kv store. + DBStatus g_kvDelegateStatus = INVALID_ARGS; + KvStoreNbDelegate *g_kvNbDelegatePtr = nullptr; + auto g_kvNbDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, + std::placeholders::_1, std::placeholders::_2, std::ref(g_kvDelegateStatus), std::ref(g_kvNbDelegatePtr)); +} + ProcessSystemApiAdapterImpl::ProcessSystemApiAdapterImpl() : callback_(nullptr), - isLocked_(false) + isLocked_(false), + createDb_(false) { } @@ -59,6 +72,7 @@ DBStatus ProcessSystemApiAdapterImpl::SetSecurityOption(const std::string &fileP if (dirPtr == nullptr) { LOGD("set path secOpt![%s] [%d] [%d]", filePath.c_str(), option.securityFlag, option.securityLabel); pathSecOptDic_[filePath] = option; + closedir(dirPtr); return OK; } @@ -109,6 +123,17 @@ DBStatus ProcessSystemApiAdapterImpl::GetSecurityOption(const std::string &fileP bool ProcessSystemApiAdapterImpl::CheckDeviceSecurityAbility(const std::string &devId, const SecurityOption &option) const { + LOGI("CheckDeviceSecurityAbility!!"); + if (createDb_) { // for close kvstore will close virtual communicator + KvStoreConfig config; + DistributedDBToolsUnitTest::TestDirInit(config.dataDir); + + g_mgr.SetKvStoreConfig(config); + + KvStoreNbDelegate::Option dbOption = {true, false, false}; + g_mgr.GetKvStore("CheckDeviceSecurityAbilityMeta", dbOption, g_kvNbDelegateCallback); + g_mgr.CloseKvStore(g_kvNbDelegatePtr); + } return true; } @@ -121,6 +146,12 @@ void ProcessSystemApiAdapterImpl::SetLockStatus(bool isLock) isLocked_ = isLock; } +void ProcessSystemApiAdapterImpl::SetNeedCreateDb(bool isCreate) +{ + std::lock_guard lock(adapterlock_); + createDb_ = isCreate; +} + void ProcessSystemApiAdapterImpl::ResetSecOptDic() { pathSecOptDic_.clear(); @@ -130,5 +161,6 @@ void ProcessSystemApiAdapterImpl::ResetAdapter() { ResetSecOptDic(); SetLockStatus(false); + g_mgr.DeleteKvStore("CheckDeviceSecurityAbilityMeta"); } }; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.h index 6e82fbbe1ea7d5e33ffa83f908aba8aae6960b46..b3aec5ce646232e7a9839c2a0866d7e70c44c4d7 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/process_system_api_adapter_impl.h @@ -34,6 +34,7 @@ public: DBStatus GetSecurityOption(const std::string &filePath, SecurityOption &option) const override; bool CheckDeviceSecurityAbility(const std::string &devId, const SecurityOption &option) const override; void SetLockStatus(bool isLock); + void SetNeedCreateDb(bool isCreate); void ResetSecOptDic(); void ResetAdapter(); @@ -42,6 +43,7 @@ private: OnAccessControlledEvent callback_; std::map pathSecOptDic_; bool isLocked_; + bool createDb_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/runtime_context_process_system_api_adapter_impl_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/runtime_context_process_system_api_adapter_impl_test.cpp index 74b3360021ad8602acf319ca14c4951846b81946..0ea1055e6d673a85cb7bedbde33a114bbd63ef5e 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/runtime_context_process_system_api_adapter_impl_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/interfaces/runtime_context_process_system_api_adapter_impl_test.cpp @@ -16,14 +16,16 @@ #include #include "db_errno.h" -#include "log_print.h" -#include "runtime_context.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" #include "iprocess_system_api_adapter.h" +#include "log_print.h" #include "process_system_api_adapter_impl.h" -#include "lock_status_observer.h" +#include "runtime_context.h" using namespace testing::ext; using namespace DistributedDB; +using namespace DistributedDBUnitTest; using namespace std; namespace { @@ -31,6 +33,15 @@ namespace { SecurityOption g_option = {0, 0}; const std::string DEV_ID = "devId"; std::shared_ptr g_adapter; + KvStoreDelegateManager g_mgr(APP_ID, USER_ID); + string g_testDir; + KvStoreConfig g_config; + + // define the g_kvDelegateCallback, used to get some information when open a kv store. + DBStatus g_kvDelegateStatus = INVALID_ARGS; + KvStoreNbDelegate *g_kvNbDelegatePtr = nullptr; + auto g_kvNbDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, + placeholders::_1, placeholders::_2, std::ref(g_kvDelegateStatus), std::ref(g_kvNbDelegatePtr)); } class RuntimeContextProcessSystemApiAdapterImplTest : public testing::Test { @@ -47,15 +58,18 @@ void RuntimeContextProcessSystemApiAdapterImplTest::SetUpTestCase(void) */ g_adapter = std::make_shared(); EXPECT_TRUE(g_adapter != nullptr); + DistributedDBToolsUnitTest::TestDirInit(g_testDir); } void RuntimeContextProcessSystemApiAdapterImplTest::TearDownTestCase(void) { RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(nullptr); + DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir); } void RuntimeContextProcessSystemApiAdapterImplTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_adapter->ResetAdapter(); } @@ -65,7 +79,7 @@ void RuntimeContextProcessSystemApiAdapterImplTest::SetUp(void) * @tc.type: FUNC * @tc.require: AR000EV1G2 */ -HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, SetSecurityOption001, TestSize.Level0) +HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, SetSecurityOption001, TestSize.Level1) { /** * @tc.steps: step1. call SetSecurityOption to set SecurityOption before set g_adapter @@ -91,7 +105,7 @@ HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, SetSecurityOption001, Te * @tc.type: FUNC * @tc.require: AR000EV1G2 */ -HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, GetSecurityOption001, TestSize.Level0) +HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, GetSecurityOption001, TestSize.Level1) { /** * @tc.steps: step1. call GetSecurityOption to get SecurityOption before set g_adapter @@ -117,7 +131,7 @@ HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, GetSecurityOption001, Te * @tc.type: FUNC * @tc.require: AR000EV1G2 */ -HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, RegisterLockStatusLister001, TestSize.Level0) +HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, RegisterLockStatusLister001, TestSize.Level1) { int errCode = E_OK; bool lockStatus = false; @@ -175,7 +189,7 @@ HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, RegisterLockStatusLister * @tc.type: FUNC * @tc.require: AR000EV1G2 */ -HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, IsAccessControlled001, TestSize.Level0) +HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, IsAccessControlled001, TestSize.Level1) { /** * @tc.steps: step1. call IsAccessControlled to get Access lock status before set g_adapter @@ -201,7 +215,7 @@ HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, IsAccessControlled001, T * @tc.type: FUNC * @tc.require: AR000EV1G2 */ -HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, CheckDeviceSecurityAbility001, TestSize.Level0) +HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, CheckDeviceSecurityAbility001, TestSize.Level1) { /** * @tc.steps: step1. call CheckDeviceSecurityAbility to check device security ability before set g_adapter @@ -219,4 +233,43 @@ HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, CheckDeviceSecurityAbili RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(g_adapter); isSupported = RuntimeContext::GetInstance()->CheckDeviceSecurityAbility(DEV_ID, g_option); EXPECT_TRUE(isSupported); +} + +namespace { +void FuncCheckDeviceSecurityAbility() +{ + RuntimeContext::GetInstance()->CheckDeviceSecurityAbility("", SecurityOption()); + return; +} +} + +/** + * @tc.name: CheckDeviceSecurityAbility002 + * @tc.desc: Check device security ability with getkvstore frequency. + * @tc.type: FUNC + * @tc.require: AR000EV1G2 + */ +HWTEST_F(RuntimeContextProcessSystemApiAdapterImplTest, CheckDeviceSecurityAbility002, TestSize.Level1) +{ + g_config.dataDir = g_testDir; + g_mgr.SetKvStoreConfig(g_config); + + RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(g_adapter); + g_adapter->SetNeedCreateDb(true); + + const std::string storeId = "CheckDeviceSecurityAbility002"; + std::thread t1(FuncCheckDeviceSecurityAbility); + std::thread t2([&]() { + for (int i = 0; i < 100; i++) { // open close 100 times + LOGI("open store!!"); + KvStoreNbDelegate::Option option1 = {true, false, false}; + g_mgr.GetKvStore(storeId, option1, g_kvNbDelegateCallback); + g_mgr.CloseKvStore(g_kvNbDelegatePtr); + } + }); + std::thread t3(FuncCheckDeviceSecurityAbility); + + t1.join(); + t2.join(); + t3.join(); } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_file_package_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_file_package_test.cpp index cde43b85611950897af66eb3f8b76a28353b97e8..6a4d6cdeb097bc0e844cd6b0b059a37216c47709 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_file_package_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_file_package_test.cpp @@ -13,14 +13,14 @@ * limitations under the License. */ -#include #include +#include +#include "db_errno.h" #include "distributeddb_tools_unit_test.h" -#include "securec.h" #include "package_file.h" -#include "db_errno.h" #include "platform_specific.h" +#include "securec.h" #include "value_hash_calc.h" using namespace testing::ext; @@ -163,6 +163,7 @@ void DistributedDBFilePackageTest::TearDownTestCase(void) void DistributedDBFilePackageTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); (void)OS::MakeDBDirectory(g_packageResultPath); (void)OS::MakeDBDirectory(g_unpackResultPath); } @@ -180,7 +181,7 @@ void DistributedDBFilePackageTest::TearDown(void) * @tc.require: AR000D4879 * @tc.author: liujialei */ -HWTEST_F(DistributedDBFilePackageTest, PackageFileTest001, TestSize.Level0) +HWTEST_F(DistributedDBFilePackageTest, PackageFileTest001, TestSize.Level1) { int errCode = PackageFile::PackageFiles(g_sourcePath, g_packageResultPath + PACKAGE_RESULT_FILE_NAME, g_fileInfo); ASSERT_EQ(errCode, E_OK); @@ -200,7 +201,7 @@ HWTEST_F(DistributedDBFilePackageTest, PackageFileTest001, TestSize.Level0) * @tc.require: AR000D4879 * @tc.author: liujialei */ -HWTEST_F(DistributedDBFilePackageTest, PackageFileTest002, TestSize.Level0) +HWTEST_F(DistributedDBFilePackageTest, PackageFileTest002, TestSize.Level1) { int errCode = PackageFile::PackageFiles(g_sourcePath + NON_EXIST_PATH, g_packageResultPath + PACKAGE_RESULT_FILE_NAME, g_fileInfo); @@ -215,7 +216,7 @@ HWTEST_F(DistributedDBFilePackageTest, PackageFileTest002, TestSize.Level0) * @tc.require: AR000D4879 * @tc.author: liujialei */ -HWTEST_F(DistributedDBFilePackageTest, PackageFileTest003, TestSize.Level0) +HWTEST_F(DistributedDBFilePackageTest, PackageFileTest003, TestSize.Level1) { int errCode = PackageFile::PackageFiles(g_sourcePath, g_packageResultPath + NON_EXIST_PATH + PACKAGE_RESULT_FILE_NAME, g_fileInfo); @@ -230,7 +231,7 @@ HWTEST_F(DistributedDBFilePackageTest, PackageFileTest003, TestSize.Level0) * @tc.require: AR000D4879 * @tc.author: liujialei */ -HWTEST_F(DistributedDBFilePackageTest, PackageFileTest004, TestSize.Level0) +HWTEST_F(DistributedDBFilePackageTest, PackageFileTest004, TestSize.Level1) { // Clear source files. RemovePath(g_sourcePath); @@ -257,7 +258,7 @@ HWTEST_F(DistributedDBFilePackageTest, PackageFileTest004, TestSize.Level0) * @tc.require: AR000D4879 * @tc.author: liujialei */ -HWTEST_F(DistributedDBFilePackageTest, PackageFileTest005, TestSize.Level0) +HWTEST_F(DistributedDBFilePackageTest, PackageFileTest005, TestSize.Level1) { FileInfo fileInfo; int errCode = PackageFile::UnpackFile(g_packageResultPath + PACKAGE_RESULT_FILE_NAME, g_unpackResultPath, fileInfo); @@ -272,7 +273,7 @@ HWTEST_F(DistributedDBFilePackageTest, PackageFileTest005, TestSize.Level0) * @tc.require: AR000D4879 * @tc.author: liujialei */ -HWTEST_F(DistributedDBFilePackageTest, PackageFileTest006, TestSize.Level0) +HWTEST_F(DistributedDBFilePackageTest, PackageFileTest006, TestSize.Level1) { int errCode = PackageFile::PackageFiles(g_sourcePath, g_packageResultPath + PACKAGE_RESULT_FILE_NAME, g_fileInfo); ASSERT_EQ(errCode, E_OK); @@ -290,7 +291,7 @@ HWTEST_F(DistributedDBFilePackageTest, PackageFileTest006, TestSize.Level0) * @tc.require: AR000D4879 * @tc.author: liujialei */ -HWTEST_F(DistributedDBFilePackageTest, PackageFileTest007, TestSize.Level0) +HWTEST_F(DistributedDBFilePackageTest, PackageFileTest007, TestSize.Level1) { int errCode = PackageFile::PackageFiles(g_sourcePath, g_packageResultPath + PACKAGE_RESULT_FILE_NAME, g_fileInfo); ASSERT_EQ(errCode, E_OK); @@ -315,7 +316,7 @@ HWTEST_F(DistributedDBFilePackageTest, PackageFileTest007, TestSize.Level0) * @tc.require: AR000D4879 * @tc.author: liujialei */ -HWTEST_F(DistributedDBFilePackageTest, PackageFileTest008, TestSize.Level0) +HWTEST_F(DistributedDBFilePackageTest, PackageFileTest008, TestSize.Level1) { int errCode = PackageFile::PackageFiles(g_sourcePath, g_packageResultPath + PACKAGE_RESULT_FILE_NAME, g_fileInfo); ASSERT_EQ(errCode, E_OK); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_multi_ver_vacuum_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_multi_ver_vacuum_test.cpp index d8b9e3e3e8658972015bb6a99092156bde36f432..f432a67d7ce6f7a15b6a4d8c6afbc185d9696225 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_multi_ver_vacuum_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_multi_ver_vacuum_test.cpp @@ -13,10 +13,11 @@ * limitations under the License. */ +#include #include #include -#include #include "db_errno.h" +#include "distributeddb_tools_unit_test.h" #include "log_print.h" #include "multi_ver_vacuum.h" #include "multi_ver_vacuum_executor_stub.h" @@ -77,6 +78,7 @@ void DistributedDBMultiVerVacuumTest::TearDownTestCase(void) void DistributedDBMultiVerVacuumTest::SetUp() { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); } void DistributedDBMultiVerVacuumTest::TearDown() @@ -692,9 +694,9 @@ HWTEST_F(DistributedDBMultiVerVacuumTest, MultipleTaskNormalStatusSwitch002, Tes EXPECT_EQ(errCode, E_OK); errCode = vacuum.Launch(DB_IDENTITY_C, &databaseC); EXPECT_EQ(errCode, E_OK); - bool stepOne = CheckVacuumTaskStatus(vacuum, DB_IDENTITY_A, VacuumTaskStatus::FINISH, 10, 100); // 10 time, 100 ms + bool stepOne = CheckVacuumTaskStatus(vacuum, DB_IDENTITY_A, VacuumTaskStatus::FINISH); EXPECT_EQ(stepOne, true); - stepOne = CheckVacuumTaskStatus(vacuum, DB_IDENTITY_B, VacuumTaskStatus::FINISH, 10, 500); // 10 time, 500 ms + stepOne = CheckVacuumTaskStatus(vacuum, DB_IDENTITY_B, VacuumTaskStatus::FINISH, 3, 1000); // 3 time, 1000 ms EXPECT_EQ(stepOne, true); stepOne = CheckVacuumTaskStatus(vacuum, DB_IDENTITY_C, VacuumTaskStatus::RUN_NING); EXPECT_EQ(stepOne, true); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_query_object_helper_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_query_object_helper_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..eabfbabd6c7bcb074812fd065581a1bac1d17393 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_query_object_helper_test.cpp @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "db_errno.h" +#include "get_query_info.h" +#include "log_print.h" +#include "query_object.h" + +using namespace testing::ext; +using namespace DistributedDB; + +namespace { +const std::string VALID_SCHEMA_FULL_DEFINE = "{\"SCHEMA_VERSION\":\"1.0\"," + "\"SCHEMA_MODE\":\"STRICT\"," + "\"SCHEMA_DEFINE\":{" + "\"field_name1\":\"BOOL\"," + "\"field_name2\":{" + "\"field_name3\":\"INTEGER, NOT NULL\"," + "\"field_name4\":\"LONG, DEFAULT 100\"," + "\"field_name5\":\"DOUBLE, NOT NULL, DEFAULT 3.14\"," + "\"field_name6\":\"STRING, NOT NULL, DEFAULT '3.1415'\"," + "\"field_name7\":[]," + "\"field_name8\":{}" + "}" + "}," + "\"SCHEMA_INDEXES\":[\"$.field_name1\", \"$.field_name2.field_name6\"]}"; +const std::string TEST_FIELD_NAME = "$.field_name2.field_name6"; + +static void GetQuerySql(const Query &query) +{ + QueryObject queryObj(query); + + SchemaObject schema; + schema.ParseFromSchemaString(VALID_SCHEMA_FULL_DEFINE); + queryObj.SetSchema(schema); + + int errCode = E_OK; + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); + ASSERT_EQ(errCode, E_OK); + EXPECT_EQ(errCode, E_OK); + std::string sql; + helper.GetQuerySql(sql, false); + LOGD("[UNITTEST][sql] = [%s]", sql.c_str()); +} +} + +class DistributedDBQueryObjectHelperTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBQueryObjectHelperTest::SetUpTestCase(void) +{ +} + +void DistributedDBQueryObjectHelperTest::TearDownTestCase(void) +{ +} + +void DistributedDBQueryObjectHelperTest::SetUp(void) +{ +} + +void DistributedDBQueryObjectHelperTest::TearDown(void) +{ +} + +/** + * @tc.name: Query001 + * @tc.desc: Check the legal single query operation to see if the generated container is correct + * @tc.type: FUNC + * @tc.require: AR000DR9K6 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBQueryObjectHelperTest, Query001, TestSize.Level1) +{ + Query query1 = Query::Select().NotEqualTo(TEST_FIELD_NAME, 123); // random test data + GetQuerySql(query1); + + Query query2 = Query::Select().EqualTo(TEST_FIELD_NAME, true); + GetQuerySql(query2); + + Query query3 = Query::Select().GreaterThan(TEST_FIELD_NAME, 0); + GetQuerySql(query3); + + Query query4 = Query::Select().LessThan(TEST_FIELD_NAME, INT_MAX); + GetQuerySql(query4); + + Query query5 = Query::Select().GreaterThanOrEqualTo(TEST_FIELD_NAME, 1.56); // random test data + GetQuerySql(query5); + + Query query6 = Query::Select().LessThanOrEqualTo(TEST_FIELD_NAME, 100); // random test data + GetQuerySql(query6); + + std::string testValue = "employee.sun.yong"; + Query query7 = Query::Select().Like(TEST_FIELD_NAME, testValue); + GetQuerySql(query7); + + Query query8 = Query::Select().NotLike(TEST_FIELD_NAME, "testValue"); + GetQuerySql(query8); + + std::vector fieldValues{1, 1, 1}; + Query query9 = Query::Select().In(TEST_FIELD_NAME, fieldValues); + GetQuerySql(query9); + + Query query10 = Query::Select().NotIn(TEST_FIELD_NAME, fieldValues); + GetQuerySql(query10); + + Query query11 = Query::Select().OrderBy(TEST_FIELD_NAME, false); + GetQuerySql(query11); + + Query query12 = Query::Select().Limit(1, 2); + GetQuerySql(query12); + + Query query13 = Query::Select().IsNull(TEST_FIELD_NAME); + GetQuerySql(query13); +} + +/** + * @tc.name: Query002 + * @tc.desc: Check for illegal query conditions can not get helper transfer to sql + * @tc.type: FUNC + * @tc.require: AR000DR9K6 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBQueryObjectHelperTest, Query002, TestSize.Level1) +{ + float testValue = 1.1; + Query query = Query::Select().NotEqualTo(".test", testValue); + QueryObject queryObj(query); + SchemaObject schema; + schema.ParseFromSchemaString(VALID_SCHEMA_FULL_DEFINE); + queryObj.SetSchema(schema); + int errCode = E_OK; + SqliteQueryHelper helper = queryObj.GetQueryHelper(errCode); // invalid field name + EXPECT_NE(errCode, E_OK); + + Query query1 = Query::Select().GreaterThan(TEST_FIELD_NAME, true); // bool compare + QueryObject queryObj1(query1); + queryObj1.SetSchema(schema); + SqliteQueryHelper helper1 = queryObj1.GetQueryHelper(errCode); + EXPECT_NE(errCode, E_OK); + + Query query2 = Query::Select().LessThan("$.field_name2.field_name4", true); + QueryObject queryObj2(query2); + queryObj2.SetSchema(schema); + SqliteQueryHelper helper2 = queryObj2.GetQueryHelper(errCode); + EXPECT_NE(errCode, E_OK); +} + +/** + * @tc.name: Query003 + * @tc.desc: Check combination condition transfer to sql + * @tc.type: FUNC + * @tc.require: AR000DR9K6 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBQueryObjectHelperTest, Query003, TestSize.Level1) +{ + Query query = Query::Select().EqualTo(TEST_FIELD_NAME, true).And().GreaterThan(TEST_FIELD_NAME, 1); + GetQuerySql(query); + + Query query1 = Query::Select().GreaterThan(TEST_FIELD_NAME, 1).OrderBy(TEST_FIELD_NAME); + GetQuerySql(query1); +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_sqlite_register_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_sqlite_register_test.cpp index fb6e5592748cf8bbc427864253a1de06d91628c0..6219f1edc31702f13be538998a03f5c1a1aa050b 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_sqlite_register_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_sqlite_register_test.cpp @@ -15,10 +15,9 @@ #include -#include "sqlite_import.h" -#include "platform_specific.h" #include "distributeddb_tools_unit_test.h" -#include "sqlite_local_kvdb_connection.h" +#include "platform_specific.h" +#include "sqlite_import.h" using namespace testing::ext; using namespace DistributedDB; @@ -30,21 +29,21 @@ namespace { char *g_errMsg = nullptr; sqlite3 *g_sqliteDb = nullptr; - const char *DB_NAME = "test.db"; - const char *SQL_HASH = "SELECT CALC_HASH_KEY(KEY) FROM ADDRESS_TEST"; + const char * const DB_NAME = "test.db"; + const char * const SQL_HASH = "SELECT CALC_HASH_KEY(KEY) FROM ADDRESS_TEST"; #ifndef OMIT_JSON - const char *SQL_JSON_RIGHT = "SELECT * FROM ADDRESS_TEST \ + const char * const SQL_JSON_RIGHT = "SELECT * FROM ADDRESS_TEST \ WHERE JSON_EXTRACT_BY_PATH(VALUE, '$.population', 0) > 800000"; - const char *SQL_JSON_WRONG_PATH = "SELECT * FROM ADDRESS_TEST \ + const char * const SQL_JSON_WRONG_PATH = "SELECT * FROM ADDRESS_TEST \ WHERE JSON_EXTRACT_BY_PATH(VALUE, '.populationWrong', 0) > 800000"; - const char *SQL_JSON_WRONG_ARGS = "SELECT * FROM ADDRESS_TEST \ + const char * const SQL_JSON_WRONG_ARGS = "SELECT * FROM ADDRESS_TEST \ WHERE JSON_EXTRACT_BY_PATH(VALUE, '$.population') > 800000"; #endif - const char *SQL_CREATE_TABLE = "CREATE TABLE IF NOT EXISTS ADDRESS_TEST(" \ + const char * const SQL_CREATE_TABLE = "CREATE TABLE IF NOT EXISTS ADDRESS_TEST(" \ "KEY BLOB NOT NULL PRIMARY KEY," \ "VALUE BLOB NOT NULL);"; - const char *SQL_INSERT = "INSERT INTO ADDRESS_TEST (KEY, VALUE)" \ + const char * const SQL_INSERT = "INSERT INTO ADDRESS_TEST (KEY, VALUE)" \ "VALUES ('1', '{\"province\":\"hunan\", \"city\":\"shaoyang\", \"population\":1000000}');" \ "INSERT INTO ADDRESS_TEST (KEY, VALUE)" \ "VALUES ('12', '{\"province\":\"jiangsu\", \"city\":\"nanjing\", \"population\":3500000}');" \ @@ -112,6 +111,7 @@ void DistributedDBSqliteRegisterTest::TearDownTestCase(void) void DistributedDBSqliteRegisterTest::SetUp() { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); } void DistributedDBSqliteRegisterTest::TearDown() diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_commit_storage_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_commit_storage_test.cpp index f572ea91fc99e20a776a05ac47f39b1112b1df7d..89df0e47077d098f0c1eb1cef0c37bffb3641d10 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_commit_storage_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_commit_storage_test.cpp @@ -15,11 +15,10 @@ #include +#include "db_errno.h" +#include "default_factory.h" #include "distributeddb_tools_unit_test.h" -#include "securec.h" #include "ikvdb_factory.h" -#include "default_factory.h" -#include "db_errno.h" using namespace testing::ext; using namespace DistributedDB; @@ -294,6 +293,7 @@ void DistributedDBStorageCommitStorageTest::TearDownTestCase(void) void DistributedDBStorageCommitStorageTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); IKvDBFactory *factory = IKvDBFactory::GetCurrent(); ASSERT_NE(factory, nullptr); if (factory == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_data_operation_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_data_operation_test.cpp index 08896c94bd0472fd47be1e319252e12396a4cb85..832cc32666fb8c57d3320d685dbc4d07e6615f0a 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_data_operation_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_data_operation_test.cpp @@ -17,12 +17,11 @@ #include "db_common.h" #include "db_constant.h" +#include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" #include "kvdb_manager.h" -#include "sqlite_local_kvdb_connection.h" -#include "distributeddb_data_generate_unit_test.h" #include "multi_ver_natural_store_transfer_data.h" -#include "multi_ver_natural_store_connection.h" +#include "sqlite_local_kvdb_connection.h" using namespace testing::ext; using namespace DistributedDB; @@ -56,6 +55,7 @@ void DistributedDBStorageDataOperationTest::TearDownTestCase(void) void DistributedDBStorageDataOperationTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); KvDBProperties properties; properties.SetBoolProp(KvDBProperties::CREATE_IF_NECESSARY, true); properties.SetIntProp(KvDBProperties::DATABASE_TYPE, KvDBProperties::LOCAL_TYPE); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_encrypt_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_encrypt_test.cpp index 6dbf07a3e0308d25534047d5ec2b6dfa710cd45b..5edc49d514a5faa7e60c14fa8cb0f8b6b6e15118 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_encrypt_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_encrypt_test.cpp @@ -13,19 +13,17 @@ * limitations under the License. */ -#include - +#include #include +#include +#include #include -#include #include -#include - #include "db_types.h" #include "log_print.h" -#include "sqlite3.h" #include "securec.h" +#include "sqlite_import.h" #ifndef OMIT_ENCRYPT using namespace testing::ext; @@ -53,10 +51,15 @@ namespace { const std::vector KEY_1 = {'A'}; const std::vector VALUE_1 = {'1'}; const std::vector VALUE_2 = {'2'}; - +#ifndef USE_SQLITE_CODEC_CIPHER + const std::string PRAGMA_CIPHER = "PRAGMA cipher="; + const std::string PRAGMA_KDF_ITER = "PRAGMA kdf_iter="; + const std::string EXPORT_STRING = "sqlcipher_export"; +#else const std::string PRAGMA_CIPHER = "PRAGMA codec_cipher="; const std::string PRAGMA_KDF_ITER = "PRAGMA codec_kdf_iter="; const std::string EXPORT_STRING = "export_database"; +#endif int Callback(void *data, int argc, char **argv, char **azColName) { @@ -85,11 +88,11 @@ namespace { int CreateTable() { - char *errMsg = nullptr; - int errCode = sqlite3_exec(g_db, CREATE_SQL.c_str(), nullptr, nullptr, &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + char *zErrMsg = nullptr; + int errCode = sqlite3_exec(g_db, CREATE_SQL.c_str(), nullptr, nullptr, &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } @@ -98,27 +101,27 @@ namespace { int SetEncryptParam(const char *passwd, int iterNumber, const string &algName) { - char *errMsg = nullptr; + char *zErrMsg = nullptr; int errCode = sqlite3_key(g_db, static_cast(passwd), strlen(passwd)); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } errCode = sqlite3_exec(g_db, (PRAGMA_KDF_ITER + to_string(iterNumber)).c_str(), nullptr, nullptr, - &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } errCode = sqlite3_exec(g_db, (PRAGMA_CIPHER + algName + ";").c_str(), nullptr, nullptr, - &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } @@ -185,14 +188,14 @@ namespace { int PutValue(const Key &key, const Value &value) { - char *errMsg = nullptr; + char *zErrMsg = nullptr; string keyStr(key.begin(), key.end()); string valueStr(value.begin(), value.end()); int errCode = sqlite3_exec(g_db, ("INSERT OR REPLACE INTO data VALUES('" + keyStr + "','" + valueStr + - "');").c_str(), nullptr, nullptr, &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + "');").c_str(), nullptr, nullptr, &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } return errCode; @@ -200,13 +203,13 @@ namespace { int DeleteValue(const Key &key) { - char *errMsg = nullptr; + char *zErrMsg = nullptr; string keyStr(key.begin(), key.end()); int errCode = sqlite3_exec(g_db, ("DELETE FROM data WHERE key='" + keyStr + "';").c_str(), nullptr, - nullptr, &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + nullptr, &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } return errCode; @@ -214,14 +217,14 @@ namespace { int UpdateValue(const Key &key, const Value &value) { - char *errMsg = nullptr; + char *zErrMsg = nullptr; string keyStr(key.begin(), key.end()); string valueStr(value.begin(), value.end()); int errCode = sqlite3_exec(g_db, ("INSERT OR REPLACE INTO data VALUES('" + keyStr + "','" + valueStr + - "');").c_str(), nullptr, nullptr, &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + "');").c_str(), nullptr, nullptr, &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } return errCode; @@ -229,13 +232,13 @@ namespace { int GetValue(const Key &key, Value &value) { - char *errMsg = nullptr; + char *zErrMsg = nullptr; string keyStr(key.begin(), key.end()); int errCode = sqlite3_exec(g_db, ("SELECT value from data WHERE key='" + keyStr + "';").c_str(), - Callback, static_cast(&value), &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + Callback, static_cast(&value), &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } return errCode; @@ -243,12 +246,12 @@ namespace { int Export(const string &dbName) { - char *errMsg = nullptr; + char *zErrMsg = nullptr; int errCode = sqlite3_exec(g_db, ("SELECT " + EXPORT_STRING + "('" + dbName + "');").c_str(), nullptr, nullptr, - &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } return errCode; @@ -256,12 +259,12 @@ namespace { int Attach(const string &dbName) { - char *errMsg = nullptr; + char *zErrMsg = nullptr; int errCode = sqlite3_exec(g_db, ("attach '" + dbName + ".db' as " + dbName + " key '';").c_str(), - nullptr, nullptr, &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + nullptr, nullptr, &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } return errCode; @@ -269,12 +272,12 @@ namespace { int AttachWithKey(const string &dbName, const char *passwd) { - char *errMsg = nullptr; + char *zErrMsg = nullptr; int errCode = sqlite3_exec(g_db, ("attach '" + dbName + ".db' as " + dbName + " key '" + passwd + "';").c_str(), - nullptr, nullptr, &errMsg); - if (errCode != SQLITE_OK && errMsg != nullptr) { - LOGE(" [SQLITE]: %s", errMsg); - sqlite3_free(errMsg); + nullptr, nullptr, &zErrMsg); + if (errCode != SQLITE_OK && zErrMsg != nullptr) { + LOGE(" [SQLITE]: %s", zErrMsg); + sqlite3_free(zErrMsg); return errCode; } return errCode; @@ -325,6 +328,11 @@ void DistributedDBStorageEncryptTest::TearDownTestCase(void) void DistributedDBStorageEncryptTest::SetUp(void) { + testing::UnitTest *test = testing::UnitTest::GetInstance(); + ASSERT_NE(test, nullptr); + const testing::TestInfo *testInfo = test->current_test_info(); + ASSERT_NE(testInfo, nullptr); + LOGI("Start unit test: %s.%s", testInfo->test_case_name(), testInfo->name()); /** * @tc.Clean DB files created from every test case. */ @@ -492,7 +500,7 @@ HWTEST_F(DistributedDBStorageEncryptTest, EncryptTest004, TestSize.Level1) */ EXPECT_EQ(sqlite3_close(g_db), SQLITE_OK); } - +#ifdef USE_SQLITE_CODEC_CIPHER /** * @tc.name: EncryptTest005 * @tc.desc: Check if rekeying possible with wrong password. @@ -526,7 +534,7 @@ HWTEST_F(DistributedDBStorageEncryptTest, EncryptTest005, TestSize.Level1) */ EXPECT_EQ(sqlite3_close(g_db), SQLITE_OK); } - +#endif /** * @tc.name: EncryptTest006 * @tc.desc: Check if rekeying possible with correct password. @@ -1290,8 +1298,14 @@ HWTEST_F(DistributedDBStorageEncryptTest, EncryptTest025, TestSize.Level1) * @tc.steps:step1. Open a database without password. * @tc.expected: step1. Return SQLITE_OK. */ - EXPECT_EQ(OpenWithKey(g_oldPasswd, ITERATION, ALG1, false), SQLITE_OK); + EXPECT_EQ(Open(g_db, STORE_ID), SQLITE_OK); + sqlite3_exec(g_db, ("PRAGMA cipher_default_attach_kdf_iter=5000;"), nullptr, nullptr, nullptr); + sqlite3_exec(g_db, ("PRAGMA cipher_default_attach_cipher='aes-256-gcm';"), nullptr, nullptr, nullptr); + + EXPECT_EQ(CreateTable(), SQLITE_OK); + EXPECT_EQ(sqlite3_close(g_db), SQLITE_OK); + EXPECT_EQ(Open(g_db, STORE_ID), SQLITE_OK); EXPECT_EQ(PutValue(KEY_1, VALUE_1), SQLITE_OK); /** @@ -1319,6 +1333,9 @@ HWTEST_F(DistributedDBStorageEncryptTest, EncryptTest025, TestSize.Level1) */ EXPECT_EQ(sqlite3_key(g_db, static_cast(g_oldPasswd), strlen(g_oldPasswd)), SQLITE_OK); + EXPECT_EQ(sqlite3_exec(g_db, (PRAGMA_CIPHER + "'aes-256-gcm';").c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + EXPECT_EQ(sqlite3_exec(g_db, (PRAGMA_KDF_ITER + "5000;").c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + /** * @tc.steps:step6. Get Value from exported DB and the value shall be the same as the original one. * @tc.expected: step6. Return SQLITE_OK. diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp index 344c62ba34930bda73b50f78895ab9dfa7a4cf76..50496bb84b42b6ca7ce06f88473972954df70f75 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_index_optimize_test.cpp @@ -16,14 +16,14 @@ #ifndef OMIT_JSON #include -#include "sqlite_import.h" -#include "platform_specific.h" -#include "types.h" #include "db_common.h" #include "db_constant.h" -#include "distributeddb_tools_unit_test.h" #include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" #include "log_print.h" +#include "platform_specific.h" +#include "sqlite_import.h" +#include "types.h" using namespace testing::ext; using namespace DistributedDB; @@ -129,6 +129,7 @@ void DistributedDBStorageIndexOptimizeTest::TearDownTestCase(void) void DistributedDBStorageIndexOptimizeTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); } void DistributedDBStorageIndexOptimizeTest::TearDown(void) @@ -259,8 +260,8 @@ HWTEST_F(DistributedDBStorageIndexOptimizeTest, SuggestIndexTest001, TestSize.Le Query query1 = Query::Select().GreaterThan("id", 1).SuggestIndex("id"); QueryObject queryObject1(query1); queryObject1.SetSchema(schemaObject1); - int errCode; - EXPECT_TRUE(queryObject1.IsValid(errCode)); + queryObject1.Init(); + EXPECT_TRUE(queryObject1.IsValid()); /** * @tc.steps: step2. Create a Query and call SuggestIndex().SuggestIndex(), then check the query2 @@ -271,7 +272,8 @@ HWTEST_F(DistributedDBStorageIndexOptimizeTest, SuggestIndexTest001, TestSize.Le Query query2 = Query::Select().SuggestIndex("id").SuggestIndex("id"); QueryObject queryObject2(query2); queryObject2.SetSchema(schemaObject2); - EXPECT_FALSE(queryObject2.IsValid(errCode)); + queryObject2.Init(); + EXPECT_FALSE(queryObject2.IsValid()); /** * @tc.steps: step3. Create a Query and call SuggestIndex().GreaterThan(), then check the query3 @@ -282,7 +284,8 @@ HWTEST_F(DistributedDBStorageIndexOptimizeTest, SuggestIndexTest001, TestSize.Le Query query3 = Query::Select().SuggestIndex("id").GreaterThan("id", 1); QueryObject queryObject3(query3); queryObject3.SetSchema(schemaObject3); - EXPECT_FALSE(queryObject3.IsValid(errCode)); + queryObject3.Init(); + EXPECT_FALSE(queryObject3.IsValid()); } /** @@ -313,11 +316,16 @@ HWTEST_F(DistributedDBStorageIndexOptimizeTest, SuggestIndexTest002, TestSize.Le */ QueryObject queryObject1(query1); queryObject1.SetSchema(schemaObject1); + int errCode = E_OK; + SqliteQueryHelper helper = queryObject1.GetQueryHelper(errCode); + ASSERT_EQ(errCode, E_OK); std::string sql1; - ASSERT_EQ(queryObject1.GetQuerySql(sql1, false), E_OK); + ASSERT_EQ(helper.GetQuerySql(sql1, false), E_OK); size_t pos = sql1.find("INDEXED BY '$.id'", 0); ASSERT_TRUE(pos != std::string::npos); + SqliteQueryHelper helper1 = queryObject1.GetQueryHelper(errCode); + ASSERT_EQ(errCode, E_OK); /** * @tc.steps: step4. Create a kvStore with the schema string. */ @@ -337,11 +345,14 @@ HWTEST_F(DistributedDBStorageIndexOptimizeTest, SuggestIndexTest002, TestSize.Le * @tc.steps: step6. GetEntries with the query1 * @tc.expected: step6. GetEntries return OK, and the out value is the given value. */ + ASSERT_EQ(errCode, E_OK); ASSERT_EQ(g_kvNbDelegatePtr->GetEntries(query1, entries), OK); EXPECT_TRUE(value == entries[0].value); + ASSERT_EQ(errCode, E_OK); EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); g_kvNbDelegatePtr = nullptr; - + SqliteQueryHelper helper2 = queryObject1.GetQueryHelper(errCode); + ASSERT_EQ(errCode, E_OK); /** * @tc.steps: step7. Create Query,call GreaterThan("id").SuggestIndex("car") */ @@ -355,8 +366,10 @@ HWTEST_F(DistributedDBStorageIndexOptimizeTest, SuggestIndexTest002, TestSize.Le */ QueryObject queryObject3(query3); queryObject3.SetSchema(schemaObject3); + SqliteQueryHelper helper3 = queryObject1.GetQueryHelper(errCode); + ASSERT_EQ(errCode, E_OK); std::string sql3; - ASSERT_EQ(queryObject3.GetQuerySql(sql3, false), E_OK); + ASSERT_EQ(helper3.GetQuerySql(sql3, false), E_OK); pos = sql3.find("INDEXED BY '$.car'", 0); ASSERT_TRUE(pos == std::string::npos); } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_memory_single_ver_naturall_store_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_memory_single_ver_naturall_store_test.cpp index 1c1ab8083c9ddfd99f965c96f52ab77bfaa57343..132e387aee1b2eeaa1363289c7acb854273fdccf 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_memory_single_ver_naturall_store_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_memory_single_ver_naturall_store_test.cpp @@ -50,6 +50,7 @@ void DistributedDBStorageMemorySingleVerNaturalStoreTest::TearDownTestCase(void) void DistributedDBStorageMemorySingleVerNaturalStoreTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); KvDBProperties property; property.SetStringProp(KvDBProperties::DATA_DIR, g_testDir); property.SetStringProp(KvDBProperties::STORE_ID, "TestGeneralNB"); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..641c9fe7706ea29ca986623082e19a8a8cf7712d --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_query_sync_test.cpp @@ -0,0 +1,1172 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "db_common.h" +#include "db_errno.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "generic_single_ver_kv_entry.h" +#include "kvdb_manager.h" +#include "query_sync_object.h" +#include "sqlite_single_ver_continue_token.h" +#include "sqlite_single_ver_natural_store.h" +#include "sqlite_single_ver_natural_store_connection.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { + DistributedDB::KvStoreConfig g_config; + std::string g_testDir; + DistributedDB::SQLiteSingleVerNaturalStore *g_store = nullptr; + DistributedDB::SQLiteSingleVerNaturalStore *g_schemaStore = nullptr; + DistributedDB::SQLiteSingleVerNaturalStoreConnection *g_connection = nullptr; + DistributedDB::SQLiteSingleVerNaturalStoreConnection *g_schemaConnect = nullptr; + + KvStoreDelegateManager g_mgr(APP_ID, USER_ID); + // define the g_kvDelegateCallback, used to get some information when open a kv store. + DBStatus g_kvDelegateStatus = INVALID_ARGS; + KvStoreNbDelegate *g_kvNbDelegatePtr = nullptr; + auto g_kvNbDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, + placeholders::_1, placeholders::_2, std::ref(g_kvDelegateStatus), std::ref(g_kvNbDelegatePtr)); + + const std::string REMOTE_DEVICE_ID = "remote_device_id"; + const Key PREFIX_KEY = { 'k' }; + const Key KEY1 = { 'k', '1' }; + const Key KEY2 = { 'k', '2' }; + const Key KEY3 = { 'k', '3' }; + const Value VALUE1 = { 'v', '1' }; + const Value VALUE2 = { 'v', '2' }; + const Value VALUE3 = { 'v', '3' }; + + void ReleaseKvEntries(std::vector &entries) + { + for (auto &itemEntry : entries) { + delete itemEntry; + itemEntry = nullptr; + } + entries.clear(); + } + + const string SCHEMA_STRING = + "{\"SCHEMA_VERSION\":\"1.0\"," + "\"SCHEMA_MODE\":\"STRICT\"," + "\"SCHEMA_DEFINE\":{" + "\"field_name1\":\"BOOL\"," + "\"field_name2\":\"BOOL\"," + "\"field_name3\":\"INTEGER, NOT NULL\"," + "\"field_name4\":\"LONG, DEFAULT 100\"," + "\"field_name5\":\"DOUBLE, NOT NULL, DEFAULT 3.14\"," + "\"field_name6\":\"STRING, NOT NULL, DEFAULT '3.1415'\"," + "\"field_name7\":\"LONG, DEFAULT 100\"," + "\"field_name8\":\"LONG, DEFAULT 100\"," + "\"field_name9\":\"LONG, DEFAULT 100\"," + "\"field_name10\":\"LONG, DEFAULT 100\"" + "}," + "\"SCHEMA_INDEXES\":[\"$.field_name1\", \"$.field_name2\"]}"; + + const std::string SCHEMA_VALUE1 = + "{\"field_name1\":true," + "\"field_name2\":false," + "\"field_name3\":10," + "\"field_name4\":20," + "\"field_name5\":3.14," + "\"field_name6\":\"3.1415\"," + "\"field_name7\":100," + "\"field_name8\":100," + "\"field_name9\":100," + "\"field_name10\":100}"; +} + +class DistributedDBStorageQuerySyncTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBStorageQuerySyncTest::SetUpTestCase(void) +{ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + LOGD("Test dir is %s", g_testDir.c_str()); + DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir + "/TestQuerySync/" + DBConstant::SINGLE_SUB_DIR); + + g_config.dataDir = g_testDir; + g_mgr.SetKvStoreConfig(g_config); + // Create schema database + KvStoreNbDelegate::Option option = {true, false, false}; + option.schema = SCHEMA_STRING; + g_mgr.GetKvStore("QuerySyncSchema", option, g_kvNbDelegateCallback); + ASSERT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + Value value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end()); + g_kvNbDelegatePtr->Put(KEY_1, value); + + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); +} + +void DistributedDBStorageQuerySyncTest::TearDownTestCase(void) +{ + DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir); +} + +void DistributedDBStorageQuerySyncTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + KvDBProperties property; + property.SetStringProp(KvDBProperties::DATA_DIR, g_testDir); + property.SetStringProp(KvDBProperties::STORE_ID, "31"); + property.SetStringProp(KvDBProperties::IDENTIFIER_DIR, "TestQuerySync"); + property.SetBoolProp(KvDBProperties::MEMORY_MODE, false); + property.SetIntProp(KvDBProperties::DATABASE_TYPE, KvDBProperties::SINGLE_VER_TYPE); + property.SetIntProp(KvDBProperties::CONFLICT_RESOLVE_POLICY, ConflictResolvePolicy::DEVICE_COLLABORATION); + g_store = new (std::nothrow) SQLiteSingleVerNaturalStore; + ASSERT_NE(g_store, nullptr); + ASSERT_EQ(g_store->Open(property), E_OK); + + int erroCode = E_OK; + g_connection = static_cast(g_store->GetDBConnection(erroCode)); + ASSERT_NE(g_connection, nullptr); + g_store->DecObjRef(g_store); + EXPECT_EQ(erroCode, E_OK); + + std::string oriIdentifier = USER_ID + "-" + APP_ID + "-" + "QuerySyncSchema"; + std::string identifier = DBCommon::TransferHashString(oriIdentifier); + property.SetStringProp(KvDBProperties::IDENTIFIER_DATA, identifier); + std::string identifierHex = DBCommon::TransferStringToHex(identifier); + property.SetStringProp(KvDBProperties::DATA_DIR, g_testDir); + property.SetStringProp(KvDBProperties::STORE_ID, "QuerySyncSchema"); + property.SetStringProp(KvDBProperties::IDENTIFIER_DIR, identifierHex); + + g_schemaStore = new (std::nothrow) SQLiteSingleVerNaturalStore; + ASSERT_NE(g_schemaStore, nullptr); + ASSERT_EQ(g_schemaStore->Open(property), E_OK); + g_schemaConnect = static_cast(g_schemaStore->GetDBConnection(erroCode)); + ASSERT_NE(g_schemaConnect, nullptr); + + std::vector entries; + IOption option; + option.dataType = IOption::SYNC_DATA; + g_schemaConnect->GetEntries(option, Query::Select(), entries); + ASSERT_FALSE(entries.empty()); + + g_schemaStore->DecObjRef(g_schemaStore); +} + +void DistributedDBStorageQuerySyncTest::TearDown(void) +{ + if (g_connection != nullptr) { + g_connection->Close(); + } + + if (g_schemaConnect != nullptr) { + g_schemaConnect->Close(); + } + + g_store = nullptr; + DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir + "/TestQuerySync/" + DBConstant::SINGLE_SUB_DIR); +} + +/** + * @tc.name: GetSyncData001 + * @tc.desc: To test the function of querying the data in the time stamp range in the database. + * @tc.type: FUNC + * @tc.require: AR000CRAKO + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQuerySyncData001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Obtain the data within the time stamp range + * through the GetSyncData(A, C) interface of the NaturalStore, where APut(option, key, value), E_OK); + + Query query = Query::Select().PrefixKey(key); + QueryObject queryObj(query); + DataSizeSpecInfo specInfo = {4 * 1024 * 1024, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; + std::vector entries; + ContinueToken token = nullptr; + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 1UL); + ReleaseKvEntries(entries); + + Key keyOther = key; + keyOther.push_back('1'); + g_store->ReleaseContinueToken(token); + EXPECT_EQ(g_connection->Put(option, keyOther, value), E_OK); + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 2UL); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); +} + +/** + * @tc.name: GetQuerySyncData002 + * @tc.desc: To test GetSyncData function is available and check the boundary value. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQuerySyncData002, TestSize.Level1) +{ + /** + * @tc.steps: step1. Put k1 k2. k1's timestamp is 0 and k2's timestamp is INT64_MAX-1. + * @tc.expected: step1. Put k1 k2 successfully. + */ + DataItem data1{KEY1, VALUE1, 0, DataItem::LOCAL_FLAG, REMOTE_DEVICE_ID}; + DataItem data2{KEY2, VALUE2, INT64_MAX - 1, DataItem::LOCAL_FLAG, REMOTE_DEVICE_ID}; + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(g_store, vector{data1, data2}, REMOTE_DEVICE_ID), E_OK); + + /** + * @tc.steps: step2. Get k1 k2. SyncTimeRange is default(all time range). + * @tc.expected: step2. Get k1 k2 successfully. + */ + Query query = Query::Select().PrefixKey(PREFIX_KEY); + QueryObject queryObj(query); + DataSizeSpecInfo specInfo = {4 * 1024 * 1024, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; + std::vector entries; + ContinueToken token = nullptr; + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 2UL); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); + + /** + * @tc.steps: step3. Put k3. k3's timestamp t3 is random. + * @tc.expected: step3. Put k3. + */ + auto time3 = static_cast(DistributedDBToolsUnitTest::GetRandInt64(0, g_store->GetCurrentTimeStamp())); + DataItem data3{KEY3, VALUE3, time3, DataItem::LOCAL_FLAG, REMOTE_DEVICE_ID}; + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(g_store, vector{data3}, REMOTE_DEVICE_ID), E_OK); + + /** + * @tc.steps: step4. Get k3. SyncTimeRange is between t3 and t3 + 1. + * @tc.expected: step4. Get k3 successfully. + */ + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{time3, 0, time3 + 1, 0}, + specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 1UL); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); + + /** + * @tc.steps: step5. Delete k1 k3. + * @tc.expected: step5. Delete k1 k3 successfully. + */ + IOption option{ IOption::SYNC_DATA }; + TimeStamp deleteBeginTime = g_store->GetCurrentTimeStamp(); + g_connection->DeleteBatch(option, vector{KEY1, KEY3}); + + /** + * @tc.steps: step6. Get deleted data. + * @tc.expected: step6. Get k1 k3. + */ + TimeStamp deleteEndTime = g_store->GetCurrentTimeStamp(); + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{0, deleteBeginTime, 0, deleteEndTime}, specInfo, token, + entries), E_OK); + EXPECT_EQ(entries.size(), 2UL); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); +} + +/** + * @tc.name: GetQuerySyncData004 + * @tc.desc: To test GetSyncDataNext function is available and check the boundary value. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQuerySyncData004, TestSize.Level1) +{ + /** + * @tc.steps: step1. Put k1 k2. k1's timestamp is 0 and k2's timestamp is INT64_MAX-1. + * @tc.expected: step1. Put k1 k2 successfully. + */ + DataItem data1{KEY1, VALUE1, 0, DataItem::LOCAL_FLAG, REMOTE_DEVICE_ID}; + DataItem data2{KEY2, VALUE2, INT64_MAX - 1, DataItem::LOCAL_FLAG, REMOTE_DEVICE_ID}; + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(g_store, vector{data1, data2}, REMOTE_DEVICE_ID), E_OK); + + /** + * @tc.steps: step2. Get k1 k2. SyncTimeRange is default(all time range). + * @tc.expected: step2. Get k1 k2 successfully. + */ + Query query = Query::Select().PrefixKey(PREFIX_KEY); + QueryObject queryObj(query); + DataSizeSpecInfo specInfo = {4 * 1024 * 1024, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; + std::vector entries; + ContinueToken token = new (std::nothrow) SQLiteSingleVerContinueToken{SyncTimeRange{}, queryObj}; + EXPECT_EQ(g_store->GetSyncDataNext(entries, token, specInfo), E_OK); + EXPECT_EQ(entries.size(), 2UL); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); + + /** + * @tc.steps: step3. Put k3. k3's timestamp t3 is random. + * @tc.expected: step3. Put k3. + */ + auto time3 = static_cast(DistributedDBToolsUnitTest::GetRandInt64(0, g_store->GetCurrentTimeStamp())); + DataItem data3{KEY3, VALUE3, time3, DataItem::LOCAL_FLAG, REMOTE_DEVICE_ID}; + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(g_store, vector{data3}, REMOTE_DEVICE_ID), E_OK); + + /** + * @tc.steps: step4. Get k3. SyncTimeRange is between t3 and t3 + 1. + * @tc.expected: step4. Get k3 successfully. + */ + token = new (std::nothrow) SQLiteSingleVerContinueToken{SyncTimeRange{time3, 0, time3 + 1}, queryObj}; + EXPECT_EQ(g_store->GetSyncDataNext(entries, token, specInfo), E_OK); + EXPECT_EQ(entries.size(), 1UL); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); + + /** + * @tc.steps: step5. Delete k1 k3. + * @tc.expected: step5. Delete k1 k3 successfully. + */ + IOption option{ IOption::SYNC_DATA }; + TimeStamp deleteBeginTime = g_store->GetCurrentTimeStamp(); + g_connection->DeleteBatch(option, vector{KEY1, KEY3}); + + /** + * @tc.steps: step6. Get deleted data. + * @tc.expected: step6. Get k1 k3. + */ + TimeStamp deleteEndTime = g_store->GetCurrentTimeStamp(); + token = new (std::nothrow) SQLiteSingleVerContinueToken{SyncTimeRange{0, deleteBeginTime, 0, deleteEndTime}, + queryObj}; + EXPECT_EQ(g_store->GetSyncDataNext(entries, token, specInfo), E_OK); + EXPECT_EQ(entries.size(), 2UL); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); +} + +/** + * @tc.name: GetQuerySyncData006 + * @tc.desc: To test if parameter is invalid, GetSyncData function return an E_INVALID_ARGS code. If no data found, + GetSyncData will return E_OK but entries will be empty. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQuerySyncData006, TestSize.Level1) +{ + /** + * @tc.steps: step1. Get sync data when no data exists in DB. + * @tc.expected: step1. GetSyncData return E_OK and entries is empty. + */ + Query query = Query::Select().PrefixKey(PREFIX_KEY); + QueryObject queryObj(query); + DataSizeSpecInfo specInfo = {4 * 1024 * 1024, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; + std::vector entries; + ContinueToken token = nullptr; + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_TRUE(entries.empty()); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); + + /** + * @tc.steps: step2. Get sync data with invalid SyncTimeRange(beginTime is greater than endTime). + * @tc.expected: step2. GetSyncData return E_INVALID_ARGS. + */ + auto time = static_cast(DistributedDBToolsUnitTest::GetRandInt64(0, INT64_MAX)); + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{time, 0, 0}, specInfo, token, entries), + -E_INVALID_ARGS); +} + +/** + * @tc.name: GetQuerySyncData006 + * @tc.desc: To test QUERY_SYNC_THRESHOLD works. When all query data is found in one get sync data operation, + if the size of query data is greater than QUERY_SYNC_THRESHOLD*MAX_ITEM_SIZE , will not get deleted data next. + Otherwise, will get deleted data next. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQuerySyncData008, TestSize.Level1) +{ + const size_t maxItemSize = 200; + const float querySyncThreshold = 0.50; + + Key key; + Value value; + string str; + IOption option{ IOption::SYNC_DATA }; + + /** + * @tc.steps: step1. Put MAX_ITEM_SIZE / 2 + 1 entries from k0 to k100. + * @tc.expected: step1. Put data successfully. + */ + for (unsigned i = 0; i <= maxItemSize * querySyncThreshold; i++) { + str = "k" + to_string(i); + key = Key(str.begin(), str.end()); + str[0] = 'v'; + value = Value(str.begin(), str.end()); + EXPECT_EQ(g_connection->Put(option, key, value), E_OK); + } + + DataItem item{key, value}; + auto oneBlockSize = SQLiteSingleVerStorageExecutor::GetDataItemSerialSize(item, Parcel::GetAppendedLen()); + + /** + * @tc.steps: step2. Delete k0. + * @tc.expected: step2. Delete k0 successfully. + */ + str = "k0"; + g_connection->Delete(option, Key(str.begin(), str.end())); + + /** + * @tc.steps: step3. Get sync data when 100 query data and 1 deleted data exists in DB. + * @tc.expected: step3. Get 100 query data and no deleted data. + */ + Query query = Query::Select().PrefixKey(PREFIX_KEY); + QueryObject queryObj(query); + + DataSizeSpecInfo specInfo = {static_cast((maxItemSize) * oneBlockSize), maxItemSize}; + std::vector entries; + ContinueToken token = nullptr; + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries), -E_UNFINISHED); + EXPECT_EQ(entries.size(), 100UL); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); + + /** + * @tc.steps: step4. Delete k1. + * @tc.expected: step4. Delete k1 successfully. + */ + str = "k1"; + g_connection->Delete(option, Key(str.begin(), str.end())); + + /** + * @tc.steps: step5. Get sync data when 99 query data and 2 deleted data exists in DB. + * @tc.expected: step5. Get 99 query data and 2 deleted data. + */ + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 101UL); + ReleaseKvEntries(entries); + g_store->ReleaseContinueToken(token); +} + +/** + * @tc.name: GetQuerySyncData009 + * @tc.desc: To test GetSyncData and GetSyncDataNext function works with large amounts of data. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQuerySyncData009, TestSize.Level2) +{ + /** + * @tc.steps: step1. Put 500 entries from k0 to k499. + * @tc.expected: step1. Put data successfully. + */ + Key key; + Value value; + string str; + IOption option{ IOption::SYNC_DATA }; + const uint64_t totalSize = 500; // 500 data in DB. + for (unsigned i = 0; i < totalSize; i++) { + str = "k" + to_string(i); + key = Key(str.begin(), str.end()); + str[0] = 'v'; + value = Value(str.begin(), str.end()); + EXPECT_EQ(g_connection->Put(option, key, value), E_OK); + } + + /** + * @tc.steps: step2. Delete 150 entries from k150 to k299. + * @tc.expected: step2. Delete data successfully. + */ + for (unsigned i = 150; i < 300; i++) { + str = "k" + to_string(i); + g_connection->Delete(option, Key(str.begin(), str.end())); + } + + /** + * @tc.steps: step3. Get all sync data; + * @tc.expected: step3. Get 500 data. + */ + Query query = Query::Select().PrefixKey(PREFIX_KEY); + QueryObject queryObj(query); + + uint64_t getSize = 0; + std::vector entries; + const size_t maxItemSize = 100; // Get 100 data at most in one GetSyncData operation. + DataSizeSpecInfo specInfo = {MTU_SIZE, maxItemSize}; + ContinueToken token = nullptr; + g_store->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries); + getSize += entries.size(); + ReleaseKvEntries(entries); + + while (token != nullptr) { + g_store->GetSyncDataNext(entries, token, specInfo); + getSize += entries.size(); + ReleaseKvEntries(entries); + } + + EXPECT_EQ(getSize, totalSize); +} + +/** + * @tc.name: GetQuerySyncData010 + * @tc.desc: To test GetSyncData when Query with limit. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: lidongwei + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQuerySyncData010, TestSize.Level1) +{ + /** + * @tc.steps: step1. Put 100 entries from k100 to k1. + * @tc.expected: step1. Put data successfully. + */ + Key key; + Value value; + string str; + IOption option{ IOption::SYNC_DATA }; + const uint64_t totalSize = 100; // 100 data in DB. + for (unsigned i = totalSize; i > 0; i--) { + str = "k" + to_string(i); + key = Key(str.begin(), str.end()); + str[0] = 'v'; + value = Value(str.begin(), str.end()); + EXPECT_EQ(g_connection->Put(option, key, value), E_OK); + } + + /** + * @tc.steps: step3. Get half of sync data; + * @tc.expected: step3. Get half of sync data successfully. + */ + Query query = Query::Select().PrefixKey(PREFIX_KEY).Limit(totalSize / 2); + QueryObject queryObj(query); + + uint64_t getSize = 0; + std::vector entries; + const size_t maxItemSize = 10; // Get 10 data at most in one GetSyncData operation. + DataSizeSpecInfo specInfo = {MTU_SIZE, maxItemSize}; + ContinueToken token = nullptr; + g_store->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries); + getSize += entries.size(); + ReleaseKvEntries(entries); + + while (token != nullptr) { + g_store->GetSyncDataNext(entries, token, specInfo); + getSize += entries.size(); + ReleaseKvEntries(entries); + } + + EXPECT_EQ(getSize, totalSize / 2); +} + +/** + * @tc.name: GetQueryID001 + * @tc.desc: To test the function of generating query identity. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQueryID001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Get illegal query object, get this object identify + * @tc.expected: step1. GetIdentify will get empty string + */ + Query errQuery = Query::Select().GreaterThan("$.test", true); + QuerySyncObject querySync(errQuery); + EXPECT_EQ(querySync.GetIdentify().empty(), true); + + /** + * @tc.steps:step2. use illegal query object to serialized + * @tc.expected: step2. SerializeData will not return E_OK + */ + vector buffer(querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT), 0); + Parcel writeParcel(buffer.data(), querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + Parcel readParcel(buffer.data(), querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + EXPECT_NE(querySync.SerializeData(writeParcel, SOFTWARE_VERSION_CURRENT), E_OK); +} + +/** + * @tc.name: GetQueryID002 + * @tc.desc: To test the function of generating query identity. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQueryID002, TestSize.Level1) +{ + /** + * @tc.steps:step1. Get empty condition query object, get this object identify + * @tc.expected: step1. GetIdentify result not change + */ + Query query1 = Query::Select(); + + QuerySyncObject querySync(query1); + EXPECT_EQ(querySync.GetIdentify().empty(), false); + // same object identify is same + EXPECT_EQ(querySync.GetIdentify(), querySync.GetIdentify()); + + IOption option; + option.dataType = IOption::SYNC_DATA; + Key key; + Value value; + DistributedDBToolsUnitTest::GetRandomKeyValue(key); + DistributedDBToolsUnitTest::GetRandomKeyValue(value); + EXPECT_EQ(g_connection->Put(option, key, value), E_OK); + EXPECT_EQ(g_connection->Put(option, KEY_1, VALUE_1), E_OK); + EXPECT_EQ(g_connection->Put(option, KEY_2, VALUE_2), E_OK); + + /** + * @tc.steps:step2. Get prefix key condition query object, get this object identify + * @tc.expected: step2. GetIdentify result not same as other condition query object + */ + Query query2 = Query::Select().PrefixKey(key); + QuerySyncObject querySync1(query2); + EXPECT_EQ(querySync1.GetIdentify().empty(), false); + // same object identify is not same + EXPECT_NE(querySync.GetIdentify(), querySync1.GetIdentify()); + + /** + * @tc.steps:step3. empty condition query object can serialized and deserialized normally + * @tc.expected: step3. after deserialized, can get all key value in database + */ + vector buffer(querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT), 0); + Parcel writeParcel(buffer.data(), querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + Parcel readParcel(buffer.data(), querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + EXPECT_EQ(querySync.SerializeData(writeParcel, SOFTWARE_VERSION_CURRENT), E_OK); + + QuerySyncObject queryObj2; + EXPECT_EQ(QuerySyncObject::DeSerializeData(readParcel, queryObj2), E_OK); + LOGD("Query obj after serialize!"); + + DataSizeSpecInfo specInfo = {4 * 1024 * 1024, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; + std::vector entries; + ContinueToken token = nullptr; + EXPECT_EQ(g_store->GetSyncData(queryObj2, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 3UL); + SingleVerKvEntry::Release(entries); +} + +/** + * @tc.name: GetQueryID003 + * @tc.desc: To test the function of generating query identity ignore limit, orderby, suggestion. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, GetQueryID003, TestSize.Level1) +{ + /** + * @tc.steps:step1. Get empty condition query object, get this object identify + */ + Query query1 = Query::Select().PrefixKey({}); + QuerySyncObject querySync1(query1); + std::string id1 = querySync1.GetIdentify(); + EXPECT_EQ(id1.empty(), false); + + /** + * @tc.steps:step2. Get limit condition query object, get this object identify + * @tc.expected: step2. GetIdentify result same as no contain limit condition + */ + Query query2 = query1.Limit(1, 1); + QuerySyncObject querySync2(query2); + std::string id2 = querySync2.GetIdentify(); + EXPECT_EQ(id2, id1); + + /** + * @tc.steps:step3. Get orderby condition query object, get this object identify + * @tc.expected: step3. GetIdentify result same as no contain orderby condition + */ + Query query3 = query2.OrderBy("$.test"); + QuerySyncObject querySync3(query3); + std::string id3 = querySync3.GetIdentify(); + EXPECT_EQ(id2, id3); +} + +/** + * @tc.name: Serialize001 + * @tc.desc: To test the function of querying the data after serialized and deserialized. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, Serialize001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Put K1V1 K2V2 and rand KV for query + */ + IOption option; + option.dataType = IOption::SYNC_DATA; + Key key; + Value value; + DistributedDBToolsUnitTest::GetRandomKeyValue(key); + DistributedDBToolsUnitTest::GetRandomKeyValue(value); + EXPECT_EQ(g_connection->Put(option, key, value), E_OK); + EXPECT_EQ(g_connection->Put(option, KEY_1, VALUE_1), E_OK); + EXPECT_EQ(g_connection->Put(option, KEY_2, VALUE_2), E_OK); + + /** + * @tc.steps:step2. Put K1V1 K2V2 and rand KV for query + * @tc.expected: step2. GetIdentify result same as no contain limit condition + */ + Query query = Query::Select().PrefixKey(key); + QueryObject queryObj(query); + DataSizeSpecInfo specInfo = {4 * 1024 * 1024, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; + std::vector entries; + ContinueToken token = nullptr; + LOGD("Ori query obj!"); + EXPECT_EQ(g_store->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 1UL); + SingleVerKvEntry::Release(entries); + + /** + * @tc.steps:step3. query result after serialized and deserialized + * @tc.expected: step3. Get same result + */ + QuerySyncObject querySync(query); + vector buffer(querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT), 0); + Parcel writeParcel(buffer.data(), querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + Parcel readParcel(buffer.data(), querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + EXPECT_EQ(querySync.SerializeData(writeParcel, SOFTWARE_VERSION_CURRENT), E_OK); + + QuerySyncObject queryObj1; + EXPECT_EQ(QuerySyncObject::DeSerializeData(readParcel, queryObj1), E_OK); + + LOGD("Query obj after serialize!"); + EXPECT_EQ(g_store->GetSyncData(queryObj1, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 1UL); + SingleVerKvEntry::Release(entries); + + std::string id = querySync.GetIdentify().c_str(); + EXPECT_EQ(id.size(), 64u); + LOGD("query identify [%s] [%zu]", id.c_str(), id.size()); +} + +/** + * @tc.name: Serialize002 + * @tc.desc: To test the function of serialized illegal query object. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, Serialize002, TestSize.Level1) +{ + /** + * @tc.steps:step1. Serialized illegal query object + * @tc.expected: step1. return not E_OK + */ + Query query = Query::Select().PrefixKey({}).GreaterThan("$.test", true); // bool can not compare + QuerySyncObject querySync(query); + vector buffer(querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT), 0); + Parcel writeParcel(buffer.data(), querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + EXPECT_NE(querySync.SerializeData(writeParcel, SOFTWARE_VERSION_CURRENT), E_OK); +} + +/** + * @tc.name: DeSerialize001 + * @tc.desc: To test the function of deserialized illegal query object. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, DeSerialize001, TestSize.Level1) +{ + /** + * @tc.steps:step1. deserialized empty content query object + * @tc.expected: step1. return not E_OK + */ + QuerySyncObject querySync; + vector buffer(querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT), 0); + Parcel readParcel(buffer.data(), querySync.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + + QuerySyncObject queryObj; + EXPECT_NE(QuerySyncObject::DeSerializeData(readParcel, queryObj), E_OK); + + /** + * @tc.steps:step2. deserialized empty parcel + * @tc.expected: step2. return not E_OK + */ + buffer.resize(0); + Parcel readParcel1(buffer.data(), 0); + EXPECT_NE(QuerySyncObject::DeSerializeData(readParcel1, queryObj), E_OK); + + /** + * @tc.steps:step3. deserialized error size parcel + * @tc.expected: step3. return not E_OK + */ + uint8_t simSize = 0; + RAND_bytes(&simSize, 1); + buffer.resize(simSize); + Parcel readParcel2(buffer.data(), simSize); + EXPECT_NE(QuerySyncObject::DeSerializeData(readParcel2, queryObj), E_OK); +} + +/** + * @tc.name: SameQueryObjectIdInDiffVer001 + * @tc.desc: Same query object have same id in different version. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, SameQueryObjectIdInDiffVer001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Record the fixed id of the query object of the current version, + * and keep it unchanged in subsequent versions + * @tc.expected: step1. Never change in diff version + */ + Query query1 = Query::Select().PrefixKey({}); + QuerySyncObject querySync1(query1); + EXPECT_EQ(querySync1.GetIdentify(), "A9AB721457C4CA98726EECC7CB16F94E31B9752BEE6D08569CFE797B4A64A304"); + + Query query2 = Query::Select(); + QuerySyncObject querySync2(query2); + EXPECT_EQ(querySync2.GetIdentify(), "AF5570F5A1810B7AF78CAF4BC70A660F0DF51E42BAF91D4DE5B2328DE0E83DFC"); + + Query query3 = Query::Select().NotLike("$.test", "testValue"); + QuerySyncObject querySync3(query3); + EXPECT_EQ(querySync3.GetIdentify(), "F2BAC2B53FE81F9928E5F8DCDF502F2419E8CEB5DFC157EEBDDB955A66C0148B"); + + vector fieldValues{1, 1, 1}; + Query query4 = Query::Select().In("$.test", fieldValues); + QuerySyncObject querySync4(query4); + EXPECT_EQ(querySync4.GetIdentify(), "EEAECCD0E1A7217574ED3092C8DAA39469388FA1B8B7B210185B4257B785FE4D"); + + Query query5 = Query::Select().OrderBy("$.test.test_child", false); + QuerySyncObject querySync5(query5); + EXPECT_EQ(querySync5.GetIdentify(), "AF5570F5A1810B7AF78CAF4BC70A660F0DF51E42BAF91D4DE5B2328DE0E83DFC"); + + Query query6 = Query::Select().Limit(1, 2); + QuerySyncObject querySync6(query6); + EXPECT_EQ(querySync6.GetIdentify(), "AF5570F5A1810B7AF78CAF4BC70A660F0DF51E42BAF91D4DE5B2328DE0E83DFC"); + + Query query7 = Query::Select().IsNull("$.test.test_child"); + QuerySyncObject querySync7(query7); + EXPECT_EQ(querySync7.GetIdentify(), "762AB5FDF9B1433D6F398269D4DDD6DE6444953F515E87C6796654180A7FF422"); + + Query query8 = Query::Select().EqualTo("$.test.test_child", true).And().GreaterThan("$.test.test_child", 1); + QuerySyncObject querySync8(query8); + EXPECT_EQ(querySync8.GetIdentify(), "B97FBFFBC690DAF25031FD4EE8ADC92F4698B9E81FD4877CD54EDEA122F6A6E0"); + + Query query9 = Query::Select().GreaterThan("$.test", 1).OrderBy("$.test"); + QuerySyncObject querySync9(query9); + EXPECT_EQ(querySync9.GetIdentify(), "77480E3EE04EB1500BB2F1A31704EE5676DC81F088A7A300F6D30E3FABA7D0A3"); + + Query query = Query::Select().GreaterThan("$.test1", 1).OrderBy("$.test1"); + QuerySyncObject querySync(query); + EXPECT_EQ(querySync.GetIdentify(), "170F5137C0BB49011D7415F706BD96B86F5FAFADA356374981362B1E177263B9"); +} + +/** + * @tc.name: querySyncByField + * @tc.desc: Test for illegal query conditions, use GetSyncData + * @tc.type: FUNC + * @tc.require: + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, querySyncByField, TestSize.Level1) +{ + Query queryInvalidField = Query::Select().EqualTo("$.field_name11", 1); + Query queryInvalidCombine = Query::Select().EqualTo("$.field_name3", 1).BeginGroup(); + Query queryAll = Query::Select(); + Query queryPrefixKeyLimit = Query::Select().PrefixKey({}).Limit(1, 0); + + DataSizeSpecInfo specInfo = {4 * 1024 * 1024, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; + std::vector entries; + ContinueToken token = nullptr; + + QueryObject queryObj(queryInvalidCombine); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries), -E_INVALID_QUERY_FORMAT); + + QueryObject queryObj2(queryAll); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj2, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 1UL); + ReleaseKvEntries(entries); + + QueryObject queryObj1(queryInvalidField); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj1, SyncTimeRange{}, specInfo, token, entries), -E_INVALID_QUERY_FIELD); + + QueryObject queryObj3(queryPrefixKeyLimit); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj3, SyncTimeRange{}, specInfo, token, entries), E_OK); + ReleaseKvEntries(entries); +} + +/** + * @tc.name: IsQueryOnlyByKey + * @tc.desc: The test can correctly determine whether the value is used for query + * @tc.type: FUNC + * @tc.require: + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, IsQueryOnlyByKey, TestSize.Level1) +{ + Query queryAll = Query::Select(); + Query queryPrefixKeyLimit = Query::Select().PrefixKey({}).Limit(1, 0); + Query queryPrefix = Query::Select().PrefixKey({}); + Query queryPrefixKeyLimitIndex = Query::Select().PrefixKey({}).Limit(1, 0).SuggestIndex("$.field_name3"); + Query queryPrefixKeyLimitEQ = Query::Select().PrefixKey({}).Limit(1, 0).EqualTo("$.field_name3", 1); + Query queryEQ = Query::Select().EqualTo("$.field_name3", 1); + Query queryLimitEQ = Query::Select().Limit(1, 0).EqualTo("$.field_name3", 1); + + QueryObject queryObj(queryAll); + EXPECT_TRUE(queryObj.IsQueryOnlyByKey()); + + QueryObject queryObj1(queryPrefixKeyLimit); + EXPECT_TRUE(queryObj1.IsQueryOnlyByKey()); + + QueryObject queryObj2(queryPrefix); + EXPECT_TRUE(queryObj2.IsQueryOnlyByKey()); + + QueryObject queryObj3(queryPrefixKeyLimitIndex); + EXPECT_FALSE(queryObj3.IsQueryOnlyByKey()); + + QueryObject queryObj4(queryPrefixKeyLimitEQ); + EXPECT_FALSE(queryObj4.IsQueryOnlyByKey()); + + QueryObject queryObj5(queryEQ); + EXPECT_FALSE(queryObj5.IsQueryOnlyByKey()); + + QueryObject queryObj6(queryLimitEQ); + EXPECT_FALSE(queryObj6.IsQueryOnlyByKey()); +} + +/** + * @tc.name: MultiQueryParcel + * @tc.desc: Mix multiple conditions for simultaneous query can be serialize + * @tc.type: FUNC + * @tc.require: + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, MultiQueryParcel, TestSize.Level1) +{ + Query queryInvalidField = Query::Select().LessThan("$.field_name1", 1); + Query queryInvalidCombine = Query::Select().EqualTo("$.field_name3", 1).BeginGroup(); + Query queryPrefixKeyLimit = Query::Select().PrefixKey({}).Limit(1, 0); + + DataSizeSpecInfo specInfo = {4 * 1024 * 1024, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; + std::vector entries; + ContinueToken token = nullptr; + + QuerySyncObject querySyncObj(queryInvalidField); + vector buffer(querySyncObj.CalculateParcelLen(SOFTWARE_VERSION_CURRENT), 0); + Parcel writeParcel(buffer.data(), querySyncObj.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + Parcel readParcel(buffer.data(), querySyncObj.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + EXPECT_EQ(querySyncObj.SerializeData(writeParcel, SOFTWARE_VERSION_CURRENT), E_OK); + QuerySyncObject queryObjAfterSer; + EXPECT_EQ(QuerySyncObject::DeSerializeData(readParcel, queryObjAfterSer), E_OK); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObjAfterSer, SyncTimeRange{}, specInfo, token, entries), + -E_INVALID_QUERY_FORMAT); + + QuerySyncObject querySyncObj1(queryInvalidCombine); + vector buffer1(querySyncObj1.CalculateParcelLen(SOFTWARE_VERSION_CURRENT), 0); + Parcel writeParcel1(buffer1.data(), querySyncObj1.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + Parcel readParcel1(buffer1.data(), querySyncObj1.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + EXPECT_EQ(querySyncObj1.SerializeData(writeParcel1, SOFTWARE_VERSION_CURRENT), E_OK); + QuerySyncObject queryObjAfterSer1; + EXPECT_EQ(QuerySyncObject::DeSerializeData(readParcel1, queryObjAfterSer1), E_OK); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObjAfterSer1, SyncTimeRange{}, specInfo, token, entries), + -E_INVALID_QUERY_FORMAT); + + QuerySyncObject querySyncObj2(queryPrefixKeyLimit); + vector buffer2(querySyncObj2.CalculateParcelLen(SOFTWARE_VERSION_CURRENT), 0); + Parcel writeParcel2(buffer2.data(), querySyncObj2.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + Parcel readParcel2(buffer2.data(), querySyncObj2.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + EXPECT_EQ(querySyncObj2.SerializeData(writeParcel2, SOFTWARE_VERSION_CURRENT), E_OK); + QuerySyncObject queryObjAfterSer2; + EXPECT_EQ(QuerySyncObject::DeSerializeData(readParcel2, queryObjAfterSer2), E_OK); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObjAfterSer2, SyncTimeRange{}, specInfo, token, entries), + E_OK); + EXPECT_FALSE(entries.empty()); + ReleaseKvEntries(entries); +} + + +/** + * @tc.name: QueryParcel001 + * @tc.desc: Query object should has same attribute(Limit, OrderBy) after deserialized + * @tc.type: FUNC + * @tc.require: + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, QueryParcel001, TestSize.Level1) +{ + Query query = Query::Select().OrderBy("$.field_name1").Limit(10, 5); + + QuerySyncObject querySyncObj(query); + vector buffer(querySyncObj.CalculateParcelLen(SOFTWARE_VERSION_CURRENT), 0); + Parcel writeParcel(buffer.data(), querySyncObj.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + Parcel readParcel(buffer.data(), querySyncObj.CalculateParcelLen(SOFTWARE_VERSION_CURRENT)); + EXPECT_EQ(querySyncObj.SerializeData(writeParcel, SOFTWARE_VERSION_CURRENT), E_OK); + QuerySyncObject queryObjAfterSer; + EXPECT_EQ(QuerySyncObject::DeSerializeData(readParcel, queryObjAfterSer), E_OK); + EXPECT_EQ(queryObjAfterSer.HasLimit(), true); + int limit = 0; + int offset = 0; + queryObjAfterSer.GetLimitVal(limit, offset); + EXPECT_EQ(limit, 10); + EXPECT_EQ(offset, 5); + EXPECT_EQ(queryObjAfterSer.HasOrderBy(), true); +} + +/** + * @tc.name: MultiQueryGetSyncData001 + * @tc.desc: Mix multiple conditions for simultaneous query + * @tc.type: FUNC + * @tc.require: + * @tc.author: sunpeng + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, MultiQueryGetSyncData001, TestSize.Level1) +{ + Query query = Query::Select(); + Query query1 = Query::Select().EqualTo("$.field_name1", true); + Query query2 = Query::Select().BeginGroup().GreaterThan("$.field_name3", 1).EndGroup(); + Query query3 = Query::Select().Like("field_name7", ""); + Query query4 = Query::Select().PrefixKey({}).OrderBy("$.field_name6"); + Query query5 = Query::Select().PrefixKey({}).IsNull("field_name10"); + + DataSizeSpecInfo specInfo = {4 * 1024 * 1024, DBConstant::MAX_HPMODE_PACK_ITEM_SIZE}; + std::vector entries; + ContinueToken token = nullptr; + + QueryObject queryObj(query); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj, SyncTimeRange{}, specInfo, token, entries), E_OK); + ReleaseKvEntries(entries); + + QueryObject queryObj1(query1); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj1, SyncTimeRange{}, specInfo, token, entries), E_OK); + EXPECT_EQ(entries.size(), 1UL); + ReleaseKvEntries(entries); + + QueryObject queryObj2(query2); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj2, SyncTimeRange{}, specInfo, token, entries), E_OK); + ReleaseKvEntries(entries); + + QueryObject queryObj3(query3); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj3, SyncTimeRange{}, specInfo, token, entries), E_OK); + ReleaseKvEntries(entries); + + QueryObject queryObj4(query4); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj4, SyncTimeRange{}, specInfo, token, entries), E_OK); + ReleaseKvEntries(entries); + + QueryObject queryObj5(query5); + EXPECT_EQ(g_schemaStore->GetSyncData(queryObj5, SyncTimeRange{}, specInfo, token, entries), E_OK); + ReleaseKvEntries(entries); +} + +/** + * @tc.name: QueryPredicateValidation001 + * @tc.desc: check query object is query only by key and has orderBy or not + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, QueryPredicateValidation001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Create a query object with prefixKey only, check it's predicate + * @tc.expected: step1. check IsQueryOnlyByKey true; check HasOrderBy false + */ + Query query1 = Query::Select().PrefixKey({}); + QuerySyncObject querySync1(query1); + EXPECT_EQ(querySync1.IsQueryOnlyByKey(), true); + EXPECT_EQ(querySync1.HasOrderBy(), false); + + /** + * @tc.steps:step2. Create a query object with prefixKey and equalTo, check it's predicate + * @tc.expected: step2. check IsQueryOnlyByKey false; check HasOrderBy false + */ + Query query2 = Query::Select().PrefixKey({}).EqualTo("$.testField", 0); + QuerySyncObject querySync2(query2); + EXPECT_EQ(querySync2.IsQueryOnlyByKey(), false); + EXPECT_EQ(querySync2.HasOrderBy(), false); + + /** + * @tc.steps:step3. Create a query object with orderBy only, check it's predicate + * @tc.expected: step3. check IsQueryOnlyByKey false; check HasOrderBy true + */ + Query query3 = Query::Select().OrderBy("$.testField"); + QuerySyncObject querySync3(query3); + EXPECT_EQ(querySync3.IsQueryOnlyByKey(), false); + EXPECT_EQ(querySync3.HasOrderBy(), true); +} + +/** + * @tc.name: RelationalQuerySyncTest001 + * @tc.desc: Test querySyncObject serialize with table name is specified + * @tc.type: FUNC + * @tc.require: + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Create a query object with table name is specified + * @tc.expected: ok + */ + Query qeury1 = Query::Select("Relatonal_table").EqualTo("field1", "abc"); + QuerySyncObject obj1(qeury1); + + /** + * @tc.steps:step2. Serialize the object + * @tc.expected: ok + */ + uint32_t buffLen = obj1.CalculateParcelLen(SOFTWARE_VERSION_CURRENT); + vector buffer(buffLen, 0); + Parcel writeParcel(buffer.data(), buffLen); + EXPECT_EQ(obj1.SerializeData(writeParcel, SOFTWARE_VERSION_CURRENT), E_OK); + + /** + * @tc.steps:step3. DeSerialize the data + * @tc.expected: ok, And the queryId is same + */ + Parcel readParcel(buffer.data(), buffLen); + QuerySyncObject queryObj2; + EXPECT_EQ(QuerySyncObject::DeSerializeData(readParcel, queryObj2), E_OK); + EXPECT_EQ(obj1.GetIdentify(), queryObj2.GetIdentify()); +} + +/** + * @tc.name: RelationalQuerySyncTest002 + * @tc.desc: Test querySyncObject with different table name has different identity + * @tc.type: FUNC + * @tc.require: + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageQuerySyncTest, RelationalQuerySyncTest002, TestSize.Level1) +{ + Query qeury1 = Query::Select("Relatonal_table1").EqualTo("field1", "abc"); + QuerySyncObject obj1(qeury1); + + Query qeury2 = Query::Select("Relatonal_table2").EqualTo("field1", "abc"); + QuerySyncObject obj2(qeury2); + + /** + * @tc.steps:step1. check object identity + * @tc.expected: identity should be different. + */ + EXPECT_NE(obj1.GetIdentify(), obj2.GetIdentify()); +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_register_conflict_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_register_conflict_test.cpp index 87024a6a91ebb85c325a685fc61ea05276287ed4..31a3c0008962e905e332c25c0cd8e6e929b1b017 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_register_conflict_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_register_conflict_test.cpp @@ -16,17 +16,17 @@ #include #include +#include "db_common.h" +#include "db_constant.h" +#include "db_errno.h" #include "distributeddb_data_generate_unit_test.h" #include "kv_store_nb_conflict_data.h" #include "kv_store_nb_delegate_impl.h" +#include "kvdb_conflict_entry.h" +#include "platform_specific.h" #include "sqlite_single_ver_natural_store.h" #include "sqlite_single_ver_natural_store_connection.h" #include "time_helper.h" -#include "kvdb_conflict_entry.h" -#include "db_errno.h" -#include "db_common.h" -#include "db_constant.h" -#include "platform_specific.h" using namespace testing::ext; using namespace DistributedDB; @@ -168,6 +168,7 @@ void DistributedDBStorageRegisterConflictTest::TearDownTestCase(void) void DistributedDBStorageRegisterConflictTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /* * Here, we create STORE_ID before test, * and it will be closed in TearDown(). diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_register_observer_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_register_observer_test.cpp index cb83b493721c6f38908be232c014b38a7f9c8db5..614a9751c5152f5e3bacce17eb1fd53b75b1a947 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_register_observer_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_register_observer_test.cpp @@ -15,15 +15,15 @@ #include #include -#include "distributeddb_tools_unit_test.h" -#include "distributeddb_data_generate_unit_test.h" -#include "ikvdb_factory.h" -#include "default_factory.h" +#include "db_common.h" #include "db_constant.h" #include "db_errno.h" -#include "sqlite_single_ver_natural_store.h" -#include "db_common.h" +#include "default_factory.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "ikvdb_factory.h" #include "log_print.h" +#include "sqlite_single_ver_natural_store.h" using namespace testing::ext; using namespace DistributedDB; @@ -283,6 +283,7 @@ void DistributedDBStorageRegisterObserverTest::TearDownTestCase(void) void DistributedDBStorageRegisterObserverTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); IKvDBFactory *factory = IKvDBFactory::GetCurrent(); ASSERT_NE(factory, nullptr); if (factory == nullptr) { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_resultset_and_json_optimize.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_resultset_and_json_optimize.cpp index f763d3af766151ea80790aff8fdff5c3fcedede9..b74fa08abec0582c0a83077f8ac864863121a8ab 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_resultset_and_json_optimize.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_resultset_and_json_optimize.cpp @@ -12,21 +12,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +#ifndef OMIT_JSON #include -#include "types.h" -#include "sqlite_utils.h" -#include "distributeddb_tools_unit_test.h" -#include "log_print.h" -#include "sqlite_single_ver_natural_store_connection.h" -#include "distributeddb_data_generate_unit_test.h" -#include "res_finalizer.h" #include "db_common.h" #include "db_constant.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "kvdb_manager.h" +#include "log_print.h" #include "platform_specific.h" +#include "res_finalizer.h" +#include "sqlite_single_ver_natural_store_connection.h" #include "sqlite_single_ver_result_set.h" -#include "kvdb_manager.h" +#include "sqlite_utils.h" +#include "types.h" using namespace testing::ext; using namespace DistributedDB; @@ -80,6 +80,7 @@ void DistributedDBStorageResultAndJsonOptimizeTest::TearDownTestCase(void) void DistributedDBStorageResultAndJsonOptimizeTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: 1. Create a SQLiteSingleVerNaturalStore. * 2. Set the ResultSet cache mode to CACHE_ENTRY_ID_ONLY. @@ -311,3 +312,4 @@ HWTEST_F(DistributedDBStorageResultAndJsonOptimizeTest, ResultSetGetEntry001, Te */ resultSet->Close(); } +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_natural_store_testcase.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_natural_store_testcase.cpp index 3297365b32742906c3f03a0c8568432fb2e17baf..ebe1e5420802b21286a92379db4efb0914fcecb2 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_natural_store_testcase.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_natural_store_testcase.cpp @@ -15,8 +15,8 @@ #include "distributeddb_storage_single_ver_natural_store_testcase.h" -#include "time_helper.h" #include "generic_single_ver_kv_entry.h" +#include "time_helper.h" using namespace DistributedDB; using namespace DistributedDBUnitTest; @@ -583,6 +583,24 @@ void DistributedDBStorageSingleVerNaturalStoreTestCase::GetMetaData001(SQLiteSin TestMetaDataPutAndGet(store, connection); } +/** + * @tc.name: DeleteMetaData001 + * @tc.desc: To test the function of deleting the metadata with prefix key in the database. + * @tc.type: FUNC + * @tc.require: AR000CCPOM + * @tc.author: wangbingquan + */ +void DistributedDBStorageSingleVerNaturalStoreTestCase::DeleteMetaData001(SQLiteSingleVerNaturalStore *&store, + SQLiteSingleVerNaturalStoreConnection *&connection) +{ + /** + * @tc.steps:step1. Put 2 mete data with prefix key 'a', 2 meta data with prefix key 'b'. + And delete meta data with prefix key 'b'. + * @tc.expected: step1. Get all meta data and will get 2 data with prefix key 'a'. + */ + TestMetaDataDeleteByPrefixKey(store, connection); +} + /** * @tc.name: GetCurrentMaxTimeStamp001 * @tc.desc: To test the function of obtaining the maximum timestamp when a record exists in the database. @@ -1650,6 +1668,14 @@ void DistributedDBStorageSingleVerNaturalStoreTestCase::TestMetaDataPutAndGet(SQ sizeKey.pop_back(); sizeValue.push_back(174); // random EXPECT_EQ(store->PutMetaData(sizeKey, sizeValue), -E_INVALID_ARGS); + + /** + * @tc.steps:step11. Delete key1 and key2 successfully. + * @tc.expected: step11. Cannot find key1 and key2 in DB anymore. + */ + EXPECT_EQ(store->DeleteMetaData(std::vector {key1, key2}), E_OK); + EXPECT_EQ(store->GetMetaData(key1, valueRead), -E_NOT_FOUND); + EXPECT_EQ(store->GetMetaData(key2, valueRead), -E_NOT_FOUND); } void DistributedDBStorageSingleVerNaturalStoreTestCase::DataBaseCommonPutOperate(SQLiteSingleVerNaturalStore *&store, @@ -1845,3 +1871,32 @@ void DistributedDBStorageSingleVerNaturalStoreTestCase::DataBaseCommonGetOperate EXPECT_EQ(DistributedDBToolsUnitTest::IsValueEqual(valueRead, value2), true); } +void DistributedDBStorageSingleVerNaturalStoreTestCase::TestMetaDataDeleteByPrefixKey( + SQLiteSingleVerNaturalStore *&store, SQLiteSingleVerNaturalStoreConnection *&connection) +{ + /** + * @tc.steps:step1. Put a1, b1, a2, b2. + * @tc.expected: step1. Return OK. + */ + ASSERT_EQ(store->PutMetaData(Key {'a', '1'}, Value {'a', '1'}), E_OK); + ASSERT_EQ(store->PutMetaData(Key {'b', '1'}, Value {'b', '1'}), E_OK); + ASSERT_EQ(store->PutMetaData(Key {'a', '2'}, Value {'a', '2'}), E_OK); + ASSERT_EQ(store->PutMetaData(Key {'b', '2'}, Value {'b', '2'}), E_OK); + + /** + * @tc.steps:step2. Delete meta data with prefix key 'b'. + * @tc.expected: step2. Return OK. + */ + ASSERT_EQ(store->DeleteMetaDataByPrefixKey(Key {'b'}), E_OK); + ASSERT_EQ(store->DeleteMetaDataByPrefixKey(Key {'c'}), E_OK); + + /** + * @tc.steps:step3. Get a1, b1, a2, b2. + * @tc.expected: step3. Get a1, a2 successfully, and get b1, b2 failed. + */ + Value value; + EXPECT_EQ(store->GetMetaData(Key {'a', '1'}, value), E_OK); + EXPECT_EQ(store->GetMetaData(Key {'a', '2'}, value), E_OK); + EXPECT_EQ(store->GetMetaData(Key {'b', '1'}, value), -E_NOT_FOUND); + EXPECT_EQ(store->GetMetaData(Key {'b', '2'}, value), -E_NOT_FOUND); +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_natural_store_testcase.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_natural_store_testcase.h index b925c7c9fd117112ff963ae6a632c8ff52a98ead..f5461f27dbd3b71bfda9562e42d3c80b78a340e9 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_natural_store_testcase.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_natural_store_testcase.h @@ -14,14 +14,14 @@ */ #include +#include "db_errno.h" #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" -#include "sqlite_utils.h" +#include "multi_ver_natural_store.h" +#include "sqlite_local_kvdb.h" #include "sqlite_single_ver_natural_store.h" #include "sqlite_single_ver_natural_store_connection.h" -#include "sqlite_local_kvdb.h" -#include "multi_ver_natural_store.h" -#include "db_errno.h" +#include "sqlite_utils.h" #ifndef DISTRIBUTEDDB_STORAGE_SINGLE_VER_NATURAL_STORE_TESTCASE_H #define DISTRIBUTEDDB_STORAGE_SINGLE_VER_NATURAL_STORE_TESTCASE_H @@ -72,6 +72,9 @@ public: static void GetMetaData001(DistributedDB::SQLiteSingleVerNaturalStore *&store, DistributedDB::SQLiteSingleVerNaturalStoreConnection *&connection); + static void DeleteMetaData001(DistributedDB::SQLiteSingleVerNaturalStore *&store, + DistributedDB::SQLiteSingleVerNaturalStoreConnection *&connection); + static void GetCurrentMaxTimeStamp001(DistributedDB::SQLiteSingleVerNaturalStore *&store, DistributedDB::SQLiteSingleVerNaturalStoreConnection *&connection); @@ -143,6 +146,9 @@ private: static void TestMetaDataPutAndGet(DistributedDB::SQLiteSingleVerNaturalStore *&store, DistributedDB::SQLiteSingleVerNaturalStoreConnection *&connection); + static void TestMetaDataDeleteByPrefixKey(DistributedDB::SQLiteSingleVerNaturalStore *&store, + DistributedDB::SQLiteSingleVerNaturalStoreConnection *&connection); + static void DataBaseCommonPutOperate(DistributedDB::SQLiteSingleVerNaturalStore *&store, DistributedDB::SQLiteSingleVerNaturalStoreConnection *&connection, DistributedDB::IOption option); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_upgrade_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_upgrade_test.cpp index 4112fba82befcdbee290f89ae0e47a0dfeec22cf..3068c1332b6a4dd20dfebeeeb7819c470d4bbef0 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_upgrade_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_single_ver_upgrade_test.cpp @@ -15,20 +15,19 @@ #include +#include "db_common.h" +#include "db_constant.h" +#include "db_errno.h" +#include "distributeddb_tools_unit_test.h" +#include "iprocess_system_api_adapter.h" #include "kv_store_delegate_manager.h" #include "kv_store_nb_delegate.h" -#include "distributeddb_tools_unit_test.h" -#include "sqlite_utils.h" -#include "sqlite_single_ver_natural_store.h" -#include "db_errno.h" #include "log_print.h" -#include "db_common.h" -#include "db_constant.h" -#include "time_helper.h" #include "platform_specific.h" -#include "iprocess_system_api_adapter.h" #include "process_system_api_adapter_impl.h" #include "runtime_context.h" +#include "sqlite_single_ver_natural_store.h" +#include "sqlite_utils.h" using namespace testing::ext; using namespace DistributedDB; @@ -305,6 +304,7 @@ void DistributedDBStorageSingleVerUpgradeTest::TearDownTestCase(void) void DistributedDBStorageSingleVerUpgradeTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); std::string identifier = DBCommon::TransferStringToHex(g_identifier); DBCommon::CreateDirectory(g_testDir + "/" + identifier); DBCommon::CreateDirectory(g_testDir + "/" + identifier + "/" + DBConstant::SINGLE_SUB_DIR); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_sqlite_single_ver_natural_store_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_sqlite_single_ver_natural_store_test.cpp index 175e5bcce266f06e7fbc34e278d9fa9b8e3eb1bb..f941f0ef4b1688c9b257ae33e1688a613f4c781b 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_sqlite_single_ver_natural_store_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_sqlite_single_ver_natural_store_test.cpp @@ -50,6 +50,7 @@ void DistributedDBStorageSQLiteSingleVerNaturalStoreTest::TearDownTestCase(void) void DistributedDBStorageSQLiteSingleVerNaturalStoreTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); DistributedDBToolsUnitTest::TestDirInit(g_testDir); LOGD("DistributedDBStorageSQLiteSingleVerNaturalStoreTest dir is %s", g_testDir.c_str()); std::string oriIdentifier = APP_ID + "-" + USER_ID + "-" + "TestGeneralNB"; @@ -424,6 +425,31 @@ HWTEST_F(DistributedDBStorageSQLiteSingleVerNaturalStoreTest, GetMetaData001, Te DistributedDBStorageSingleVerNaturalStoreTestCase::GetMetaData001(g_store, g_connection); } +/** + * @tc.name: DeleteMetaData001 + * @tc.desc: * @tc.name: To test the function of deleting the metadata with prefix key in the database. + * @tc.type: FUNC + * @tc.require: AR000CCPOM + * @tc.author: wangbingquan + */ +HWTEST_F(DistributedDBStorageSQLiteSingleVerNaturalStoreTest, DeleteMetaData001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Put a1, b1, a2, b2. + * @tc.expected: step1. Return OK. + */ + /** + * @tc.steps:step2. Delete meta data with prefix key 'b'. + * @tc.expected: step2. Return OK. + */ + /** + * @tc.steps:step3. Get a1, b1, a2, b2. + * @tc.expected: step3. Get a1, a2 successfully, and get b1, b2 failed. + */ + DistributedDBStorageSingleVerNaturalStoreTestCase::DeleteMetaData001(g_store, g_connection); +} + + /** * @tc.name: GetCurrentMaxTimeStamp001 * @tc.desc: To test the function of obtaining the maximum timestamp when a record exists in the database. diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_subscribe_query_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_subscribe_query_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..80318eef5028f32a6e181545a432456541548884 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_subscribe_query_test.cpp @@ -0,0 +1,670 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "db_common.h" +#include "db_errno.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "generic_single_ver_kv_entry.h" +#include "kvdb_manager.h" +#include "process_communicator_test_stub.h" +#include "process_system_api_adapter_impl.h" +#include "query_sync_object.h" +#include "sqlite_single_ver_natural_store.h" +#include "sqlite_single_ver_natural_store_connection.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { +DistributedDB::KvStoreConfig g_config; +std::string g_testDir; +string g_resourceDir; + +KvStoreDelegateManager g_mgr(APP_ID, USER_ID); +// define the g_kvDelegateCallback, used to get some information when open a kv store. +DBStatus g_kvDelegateStatus = INVALID_ARGS; +KvStoreNbDelegate *g_kvNbDelegatePtr = nullptr; +auto g_kvNbDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, + placeholders::_1, placeholders::_2, std::ref(g_kvDelegateStatus), std::ref(g_kvNbDelegatePtr)); + +const uint8_t PRESET_DATA_SIZE = 2; +const std::string SUBSCRIBE_ID = "680A20600517073AE306B11FEA8306C57DC5102CD33E322F7C513176AA707F0C"; + +const std::string REMOTE_DEVICE_ID = "remote_device_id"; +const std::string REMOTE_DEVICE_A = "remote_device_A"; +const std::string REMOTE_DEVICE_B = "remote_device_B"; +const Key PREFIX_KEY = { 'k' }; +const Key KEY1 = { 'k', '1' }; +const Key KEY2 = { 'k', '2' }; +const Key KEY3 = { 'k', '3' }; +const Value VALUE1 = { 'v', '1' }; +const Value VALUE2 = { 'v', '2' }; +const Value VALUE3 = { 'v', '3' }; + +const std::string NORMAL_FBS_FILE_NAME = "normal_fbs.bfbs"; +const string SCHEMA_STRING = + "{\"SCHEMA_VERSION\":\"1.0\"," + "\"SCHEMA_MODE\":\"STRICT\"," + "\"SCHEMA_DEFINE\":{" + "\"field_name1\":\"BOOL\"," + "\"field_name2\":\"BOOL\"," + "\"field_name3\":\"INTEGER, NOT NULL\"," + "\"field_name4\":\"LONG, DEFAULT 100\"," + "\"field_name5\":\"DOUBLE, DEFAULT 3.14\"," + "\"field_name6\":\"STRING, DEFAULT '3.1415'\"," + "\"field_name7\":\"LONG, DEFAULT 100\"," + "\"field_name8\":\"LONG, DEFAULT 100\"," + "\"field_name9\":\"LONG, DEFAULT 100\"," + "\"field_name10\":\"LONG, DEFAULT 100\"" + "}," + "\"SCHEMA_INDEXES\":[\"$.field_name1\", \"$.field_name2\"]}"; + +void PreSetData(uint8_t dataNum) +{ + EXPECT_GE(dataNum, 0); // 0 No preset data + EXPECT_LT(dataNum, 128); // 128 Max preset data size + for (uint8_t i = 0; i < dataNum; i++) { + Key keyA = {'K', i}; + Value value; + std::string validJsonData; + if (i % 2 == 0) { // 2 : for data construct + validJsonData = R"({"field_name1":false,"field_name2":true,"field_name3":100})"; + } else { + validJsonData = R"({"field_name1":false,"field_name2":false,"field_name3":100})"; + } + value.assign(validJsonData.begin(), validJsonData.end()); + EXPECT_EQ(g_kvNbDelegatePtr->Put(keyA, value), E_OK); + } +} + +void CreateAndGetStore(const std::string &storeId, const std::string &schemaString, + SQLiteSingleVerNaturalStoreConnection *&conn, SQLiteSingleVerNaturalStore *&store, uint8_t preSetDataNum = 0) +{ + KvStoreNbDelegate::Option option = {true, false, false}; + option.schema = schemaString; + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + EXPECT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + PreSetData(preSetDataNum); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); + + std::string oriIdentifier = USER_ID + "-" + APP_ID + "-" + storeId; + std::string identifier = DBCommon::TransferHashString(oriIdentifier); + KvDBProperties property; + property.SetStringProp(KvDBProperties::IDENTIFIER_DATA, identifier); + std::string identifierHex = DBCommon::TransferStringToHex(identifier); + property.SetStringProp(KvDBProperties::DATA_DIR, g_testDir); + property.SetStringProp(KvDBProperties::STORE_ID, storeId); + property.SetBoolProp(KvDBProperties::MEMORY_MODE, false); + property.SetIntProp(KvDBProperties::DATABASE_TYPE, KvDBProperties::SINGLE_VER_TYPE); + property.SetStringProp(KvDBProperties::IDENTIFIER_DIR, identifierHex); + property.SetIntProp(KvDBProperties::CONFLICT_RESOLVE_POLICY, ConflictResolvePolicy::LAST_WIN); + + if (!schemaString.empty()) { + SchemaObject schemaObj; + schemaObj.ParseFromSchemaString(schemaString); + EXPECT_EQ(schemaObj.IsSchemaValid(), true); + property.SetSchema(schemaObj); + } + + int errCode = E_OK; + conn = static_cast(KvDBManager::GetDatabaseConnection(property, errCode)); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(conn, nullptr); + store = static_cast(KvDBManager::OpenDatabase(property, errCode)); + EXPECT_EQ(errCode, E_OK); + ASSERT_NE(store, nullptr); +} + +std::string FbfFileToSchemaString(const std::string &fileName) +{ + std::string filePath = g_resourceDir + "fbs_files_for_ut/" + fileName; + std::ifstream is(filePath, std::ios::binary | std::ios::ate); + if (!is.is_open()) { + LOGE("[FbfFileToSchemaString] open file failed name : %s", filePath.c_str()); + return ""; + } + + auto size = is.tellg(); + LOGE("file size %d", size); + std::string schema(size, '\0'); + is.seekg(0); + if (is.read(&schema[0], size)) { + return schema; + } + LOGE("[FbfFileToSchemaString] read file failed path : %s", filePath.c_str()); + return ""; +} + +void CheckDataNumByKey(const std::string &storeId, const Key& key, size_t expSize) +{ + KvStoreNbDelegate::Option option = {true, false, false}; + option.schema = SCHEMA_STRING; + g_mgr.GetKvStore(storeId, option, g_kvNbDelegateCallback); + EXPECT_TRUE(g_kvNbDelegatePtr != nullptr); + EXPECT_TRUE(g_kvDelegateStatus == OK); + std::vector entries; + EXPECT_EQ(g_kvNbDelegatePtr->GetEntries(key, entries), E_OK); + EXPECT_TRUE(entries.size() == expSize); + EXPECT_EQ(g_mgr.CloseKvStore(g_kvNbDelegatePtr), OK); +} +} + +class DistributedDBStorageSubscribeQueryTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp() override; + void TearDown() override; +}; + +static std::shared_ptr g_adapter; +void DistributedDBStorageSubscribeQueryTest::SetUpTestCase(void) +{ + g_mgr.SetProcessLabel("DistributedDBStorageSubscribeQueryTest", "test"); + g_mgr.SetProcessCommunicator(std::make_shared()); // export and import get devID + + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + ASSERT_EQ(DistributedDBToolsUnitTest::GetResourceDir(g_resourceDir), E_OK); + LOGD("Test dir is %s", g_testDir.c_str()); + DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir + "/TestQuerySync/" + DBConstant::SINGLE_SUB_DIR); + + g_config.dataDir = g_testDir; + g_mgr.SetKvStoreConfig(g_config); + + g_adapter = std::make_shared(); + EXPECT_TRUE(g_adapter != nullptr); + RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(g_adapter); +} + +void DistributedDBStorageSubscribeQueryTest::TearDownTestCase(void) +{ + RuntimeContext::GetInstance()->SetProcessSystemApiAdapter(nullptr); +} + +void DistributedDBStorageSubscribeQueryTest::SetUp() +{ + Test::SetUp(); + DistributedDBToolsUnitTest::PrintTestCaseInfo(); +} + +void DistributedDBStorageSubscribeQueryTest::TearDown() +{ + Test::TearDown(); + DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir); +} + +/** + * @tc.name: CheckAndInitQueryCondition001 + * @tc.desc: Check the condition is legal or not with json schema + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageSubscribeQueryTest, CheckAndInitQueryCondition001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Create a json schema db, get the natural store instance. + * @tc.expected: step1. Get results OK and non-null store. + */ + SQLiteSingleVerNaturalStoreConnection *conn = nullptr; + SQLiteSingleVerNaturalStore *store = nullptr; + CreateAndGetStore("SchemaCondition01", SCHEMA_STRING, conn, store); + + /** + * @tc.steps:step2. Create a query with prefixKey only, check it as condition. + * @tc.expected: step2. Check condition return E_OK. + */ + Query query1 = Query::Select().PrefixKey({}); + QueryObject queryObject1(query1); + int errCode = store->CheckAndInitQueryCondition(queryObject1); + EXPECT_EQ(errCode, E_OK); + + /** + * @tc.steps:step3. Create a query with predicate, check it as condition. + * @tc.expected: step3. Check condition return E_OK. + */ + Query query2 = Query::Select().GreaterThan("field_name3", 10); + QueryObject queryObject2(query2); + errCode = store->CheckAndInitQueryCondition(queryObject2); + EXPECT_EQ(errCode, E_OK); + + /** + * @tc.steps:step4. Create a query with invalid field, check it as condition. + * @tc.expected: step4. Check condition return E_INVALID_QUERY_FIELD. + */ + Query query3 = Query::Select().GreaterThan("field_name11", 10); + QueryObject queryObject3(query3); + errCode = store->CheckAndInitQueryCondition(queryObject3); + EXPECT_EQ(errCode, -E_INVALID_QUERY_FIELD); + + /** + * @tc.steps:step5. Create a query with invalid format, check it as condition. + * @tc.expected: step5. Check condition return E_INVALID_QUERY_FORMAT. + */ + Query query4 = Query::Select().GreaterThan("field_name3", 10).And().BeginGroup(). + LessThan("field_name3", 100).OrderBy("field_name3").EndGroup(); + QueryObject queryObject4(query4); + errCode = store->CheckAndInitQueryCondition(queryObject4); + EXPECT_EQ(errCode, -E_INVALID_QUERY_FORMAT); + + /** + * @tc.steps:step6. Close natural store + * @tc.expected: step6. Close ok + */ + RefObject::KillAndDecObjRef(store); + KvDBManager::ReleaseDatabaseConnection(conn); +} + +/** + * @tc.name: CheckAndInitQueryCondition002 + * @tc.desc: Check the condition always illegal with flatbuffer schema + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageSubscribeQueryTest, CheckAndInitQueryCondition002, TestSize.Level1) +{ + /** + * @tc.steps:step1. Create a flatbuffer schema db, get the natural store instance. + * @tc.expected: step1. Get results OK and non-null store. + */ + std::string fbSchema = FbfFileToSchemaString(NORMAL_FBS_FILE_NAME); + EXPECT_FALSE(fbSchema.empty()); + SQLiteSingleVerNaturalStoreConnection *conn = nullptr; + SQLiteSingleVerNaturalStore *store = nullptr; + CreateAndGetStore("SchemaCondition02", fbSchema, conn, store); + + /** + * @tc.steps:step2. Create a query, check it as condition. + * flatbuffer schema is not support with querySync and subscribe. + * @tc.expected: step2. Check condition return E_NOT_SUPPORT. + */ + Query query1 = Query::Select().PrefixKey({}); + QueryObject queryObject1(query1); + int errCode = store->CheckAndInitQueryCondition(queryObject1); + EXPECT_EQ(errCode, -E_NOT_SUPPORT); + + /** + * @tc.steps:step3. Close natural store + * @tc.expected: step3. Close ok + */ + RefObject::KillAndDecObjRef(store); + KvDBManager::ReleaseDatabaseConnection(conn); +} + +/** + * @tc.name: CheckAndInitQueryCondition003 + * @tc.desc: Check the condition always illegal with flatbuffer schema + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageSubscribeQueryTest, CheckAndInitQueryCondition003, TestSize.Level1) +{ + /** + * @tc.steps:step1. Create a kv db, get the natural store instance. + * @tc.expected: step1. Get results OK and non-null store. + */ + SQLiteSingleVerNaturalStoreConnection *conn = nullptr; + SQLiteSingleVerNaturalStore *store = nullptr; + CreateAndGetStore("SchemaCondition03", "", conn, store); + + /** + * @tc.steps:step2. Create a prefixKey query, check it as condition. + * @tc.expected: step2. Check condition return E_OK. + */ + Query query1 = Query::Select().PrefixKey({}); + QueryObject queryObject1(query1); + int errCode = store->CheckAndInitQueryCondition(queryObject1); + EXPECT_EQ(errCode, E_OK); + + /** + * @tc.steps:step2. Create a predicate query, check it as condition. + * @tc.expected: step2. Check condition return E_NOT_SUPPORT. + */ + Query query2 = Query::Select().GreaterThan("field_name3", 10); + QueryObject queryObject2(query2); + errCode = store->CheckAndInitQueryCondition(queryObject2); + EXPECT_EQ(errCode, -E_NOT_SUPPORT); + + /** + * @tc.steps:step3. Close natural store + * @tc.expected: step3. Close ok + */ + RefObject::KillAndDecObjRef(store); + KvDBManager::ReleaseDatabaseConnection(conn); +} + +/** + * @tc.name: PutSyncDataTestWithQuery + * @tc.desc: put remote devices sync data(get by query sync or subscribe) with query. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageSubscribeQueryTest, PutSyncDataTestWithQuery, TestSize.Level1) +{ + /** + * @tc.steps:step1. create and open a schema store, preset some data; + * @tc.expected: step1. open success + */ + const std::string storeId = "PutSyncData01"; + SQLiteSingleVerNaturalStoreConnection *conn = nullptr; + SQLiteSingleVerNaturalStore *store = nullptr; + CreateAndGetStore(storeId, SCHEMA_STRING, conn, store, PRESET_DATA_SIZE); + + /** + * @tc.steps:step2. Construct sync data + * @tc.expected: OK + */ + Key key; + Value value; + TimeStamp now = store->GetCurrentTimeStamp(); + LOGD("now time is : %ld", now); + std::vector data; + for (uint8_t i = 0; i < PRESET_DATA_SIZE; i++) { + DataItem item{key, value, now, DataItem::REMOTE_DEVICE_DATA_MISS_QUERY, REMOTE_DEVICE_ID, now}; + item.key.clear(); + DBCommon::CalcValueHash({'K', i}, item.key); + EXPECT_EQ(item.key.empty(), false); + data.push_back(item); + } + + /** + * @tc.steps:step3. put sync data with query + * @tc.expected: step3. data put success + */ + Query query = Query::Select().EqualTo("field_name2", true); + QueryObject queryObj(query); + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(store, data, REMOTE_DEVICE_ID, queryObj), E_OK); + + RefObject::KillAndDecObjRef(store); + KvDBManager::ReleaseDatabaseConnection(conn); + + /** + * @tc.steps:step4. Check sync data + * @tc.expected: step4. check data ok + */ + CheckDataNumByKey(storeId, {'K'}, PRESET_DATA_SIZE / 2); +} + +/** + * @tc.name: PutSyncDataTestWithQuery002 + * @tc.desc: put remote devices sync data(timestamp is smaller then DB data) with query. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageSubscribeQueryTest, PutSyncDataTestWithQuery002, TestSize.Level1) +{ + /** + * @tc.steps:step1. create and open a schema store, preset some data; + * @tc.expected: step1. open success + */ + const std::string storeId = "PutSyncData02"; + + SQLiteSingleVerNaturalStoreConnection *conn = nullptr; + SQLiteSingleVerNaturalStore *store = nullptr; + CreateAndGetStore(storeId, SCHEMA_STRING, conn, store); + + Key key({'K', 'e', 'y'}); + Value value; + TimeStamp now = store->GetCurrentTimeStamp(); + /** + * @tc.steps:step2. put sync data + * @tc.expected: OK + */ + std::string validJsonData(R"({"field_name1":false,"field_name2":true,"field_name3":100})"); + value.assign(validJsonData.begin(), validJsonData.end()); + std::vector data; + DataItem item{key, value, now, DataItem::LOCAL_FLAG, REMOTE_DEVICE_ID, now}; + data.push_back(item); + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(store, data, REMOTE_DEVICE_ID), E_OK); + + /** + * @tc.steps:step3. put sync miss query data with smaller timestamp + * @tc.expected: OK + */ + data.clear(); + value.clear(); + DataItem itemMiss{key, value, now - 1, DataItem::REMOTE_DEVICE_DATA_MISS_QUERY, REMOTE_DEVICE_ID, now - 1}; + itemMiss.key.clear(); + DBCommon::CalcValueHash({'K', 'e', 'y'}, itemMiss.key); + EXPECT_EQ(itemMiss.key.empty(), false); + data.push_back(itemMiss); + Query query = Query::Select().EqualTo("field_name2", true); + QueryObject queryObj(query); + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(store, data, REMOTE_DEVICE_ID, queryObj), E_OK); + + RefObject::KillAndDecObjRef(store); + KvDBManager::ReleaseDatabaseConnection(conn); + + /** + * @tc.steps:step4. Check sync data + * @tc.expected: check data ok, data {key} is not erased. + */ + CheckDataNumByKey(storeId, {'K', 'e', 'y'}, 1); +} + +/** + * @tc.name: PutSyncDataTestWithQuery003 + * @tc.desc: put remote devices sync data(with same timestamp in DB data but different devices) with query. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageSubscribeQueryTest, PutSyncDataTestWithQuery003, TestSize.Level1) +{ + /** + * @tc.steps:step1. create and open a schema store, preset some data; + * @tc.expected: step1. open success + */ + const std::string storeId = "PutSyncData03"; + + SQLiteSingleVerNaturalStoreConnection *conn = nullptr; + SQLiteSingleVerNaturalStore *store = nullptr; + CreateAndGetStore(storeId, SCHEMA_STRING, conn, store); + + Key key({'K', 'e', 'y'}); + Value value; + TimeStamp now = store->GetCurrentTimeStamp(); + /** + * @tc.steps:step2. put sync data + * @tc.expected: OK + */ + std::string validJsonData(R"({"field_name1":false,"field_name2":true,"field_name3":100})"); + value.assign(validJsonData.begin(), validJsonData.end()); + std::vector data; + DataItem item{key, value, now, DataItem::LOCAL_FLAG, REMOTE_DEVICE_ID, now}; + data.push_back(item); + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(store, data, REMOTE_DEVICE_ID), E_OK); + + /** + * @tc.steps:step3. put sync miss query data with same timestamp + * @tc.expected: OK + */ + data.clear(); + DataItem itemMiss{key, {}, now, DataItem::REMOTE_DEVICE_DATA_MISS_QUERY, REMOTE_DEVICE_ID, now}; + itemMiss.key.clear(); + DBCommon::CalcValueHash({'K', 'e', 'y'}, itemMiss.key); + EXPECT_EQ(itemMiss.key.empty(), false); + data.push_back(itemMiss); + Query query = Query::Select().EqualTo("field_name2", true); + QueryObject queryObj(query); + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(store, data, REMOTE_DEVICE_ID, queryObj), E_OK); + + RefObject::KillAndDecObjRef(store); + KvDBManager::ReleaseDatabaseConnection(conn); + + /** + * @tc.steps:step4. Check sync data + * @tc.expected: check data ok, data {key} is not erased. + */ + CheckDataNumByKey(storeId, {'K', 'e', 'y'}, 1); +} + +/** + * @tc.name: PutSyncDataTestWithQuery004 + * @tc.desc: put remote devices sync data(with same timestamp in DB data but different devices) with query. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageSubscribeQueryTest, PutSyncDataTestWithQuery004, TestSize.Level1) +{ + /** + * @tc.steps:step1. create and open a schema store, preset some data; + * @tc.expected: step1. open success + */ + const std::string storeId = "PutSyncData04"; + SQLiteSingleVerNaturalStoreConnection *conn = nullptr; + SQLiteSingleVerNaturalStore *store = nullptr; + CreateAndGetStore(storeId, SCHEMA_STRING, conn, store); + + Key key({'K', 'e', 'y'}); + Value value; + TimeStamp now = store->GetCurrentTimeStamp(); + /** + * @tc.steps:step2. put sync data + * @tc.expected: OK + */ + std::string validJsonData(R"({"field_name1":false,"field_name2":true,"field_name3":100})"); + value.assign(validJsonData.begin(), validJsonData.end()); + std::vector data; + DataItem item{key, value, now, DataItem::LOCAL_FLAG, REMOTE_DEVICE_A, now}; + data.push_back(item); + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(store, data, REMOTE_DEVICE_ID), E_OK); + + /** + * @tc.steps:step3. put sync miss query data with same timestamp + * @tc.expected: OK + */ + data.clear(); + DataItem itemMiss{key, {}, now, DataItem::REMOTE_DEVICE_DATA_MISS_QUERY, REMOTE_DEVICE_B, now}; + itemMiss.key.clear(); + DBCommon::CalcValueHash({'K', 'e', 'y'}, itemMiss.key); + EXPECT_EQ(itemMiss.key.empty(), false); + data.push_back(itemMiss); + Query query = Query::Select().EqualTo("field_name2", true); + QueryObject queryObj(query); + EXPECT_EQ(DistributedDBToolsUnitTest::PutSyncDataTest(store, data, REMOTE_DEVICE_B, queryObj), E_OK); + + RefObject::KillAndDecObjRef(store); + KvDBManager::ReleaseDatabaseConnection(conn); + + /** + * @tc.steps:step4. Check sync data + * @tc.expected: check data ok, data {key} is not erased. + */ + CheckDataNumByKey(storeId, {'K', 'e', 'y'}, 1); +} + +/** + * @tc.name: AddSubscribeTest001 + * @tc.desc: Add subscribe with query + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageSubscribeQueryTest, AddSubscribeTest001, TestSize.Level1) +{ + /** + * @tc.steps:step1. Create a json schema db, get the natural store instance. + * @tc.expected: Get results OK and non-null store. + */ + SQLiteSingleVerNaturalStoreConnection *conn = nullptr; + SQLiteSingleVerNaturalStore *store = nullptr; + CreateAndGetStore("SubscribeTest01", SCHEMA_STRING, conn, store); + + std::vector queryList; + queryList.push_back(Query::Select().PrefixKey({10, 20})); + queryList.push_back(Query::Select().EqualTo("field_name3", 30)); + queryList.push_back(Query::Select().NotEqualTo("field_name3", 30)); + queryList.push_back(Query::Select().GreaterThan("field_name3", 10)); + queryList.push_back(Query::Select().LessThan("field_name3", 30)); + queryList.push_back(Query::Select().GreaterThanOrEqualTo("field_name3", 30)); + queryList.push_back(Query::Select().LessThanOrEqualTo("field_name3", 30)); + queryList.push_back(Query::Select().Like("field_name6", "Abc%")); + queryList.push_back(Query::Select().NotLike("field_name6", "Asd%")); + std::vector set = {1, 2, 3, 4}; + queryList.push_back(Query::Select().In("field_name3", set)); + queryList.push_back(Query::Select().NotIn("field_name3", set)); + queryList.push_back(Query::Select().IsNull("field_name4")); + queryList.push_back(Query::Select().IsNotNull("field_name5")); + queryList.push_back(Query::Select().EqualTo("field_name3", 30).And().EqualTo("field_name1", true)); + queryList.push_back(Query::Select().EqualTo("field_name3", 30).Or().EqualTo("field_name1", true)); + queryList.push_back(Query::Select().EqualTo("field_name2", false).Or(). + BeginGroup().EqualTo("field_name3", 30).Or().EqualTo("field_name1", true).EndGroup()); + + /** + * @tc.steps:step2. Add subscribe with query, remove subscribe. + * @tc.expected: success. + */ + for (const auto &query : queryList) { + QueryObject queryObj(query); + EXPECT_EQ(store->AddSubscribe(SUBSCRIBE_ID, queryObj, false), E_OK); + EXPECT_EQ(store->RemoveSubscribe(SUBSCRIBE_ID), E_OK); + } + + /** + * @tc.steps:step6. Close natural store + * @tc.expected: step6. Close ok + */ + RefObject::KillAndDecObjRef(store); + KvDBManager::ReleaseDatabaseConnection(conn); +} + +/** + * @tc.name: AddSubscribeTest002 + * @tc.desc: Add subscribe with same query not failed + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xulianhui + */ +HWTEST_F(DistributedDBStorageSubscribeQueryTest, AddSubscribeTest002, TestSize.Level1) +{ + /** + * @tc.steps:step1. Create a json schema db, get the natural store instance. + * @tc.expected: Get results OK and non-null store. + */ + SQLiteSingleVerNaturalStoreConnection *conn = nullptr; + SQLiteSingleVerNaturalStore *store = nullptr; + CreateAndGetStore("SubscribeTest02", SCHEMA_STRING, conn, store); + + Query query = Query::Select().EqualTo("field_name2", false).Or(). + BeginGroup().EqualTo("field_name3", 30).Or().EqualTo("field_name1", true).EndGroup(); + /** + * @tc.steps:step2. Add subscribe with same query + * @tc.expected: step2. add success + */ + QueryObject queryObj(query); + int errCode = store->AddSubscribe(SUBSCRIBE_ID, queryObj, false); + EXPECT_EQ(errCode, E_OK); + errCode = store->AddSubscribe(SUBSCRIBE_ID, queryObj, false); + EXPECT_EQ(errCode, E_OK); + EXPECT_EQ(store->RemoveSubscribe(SUBSCRIBE_ID), E_OK); + /** + * @tc.steps:step3. Close natural store + * @tc.expected: step3. Close ok + */ + RefObject::KillAndDecObjRef(store); + KvDBManager::ReleaseDatabaseConnection(conn); +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_transaction_data_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_transaction_data_test.cpp index 38a80df55657e9363fdb1416e162e0a383a122e9..ab414c0d204e1d00f4b239257447153c0ccf4a86 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_transaction_data_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_transaction_data_test.cpp @@ -13,26 +13,25 @@ * limitations under the License. */ -#include #include #include -#include +#include #include +#include #include #include "db_common.h" +#include "db_constant.h" #include "db_errno.h" +#include "default_factory.h" #include "distributeddb_data_generate_unit_test.h" #include "distributeddb_tools_unit_test.h" #include "log_print.h" #include "multi_ver_natural_store.h" #include "multi_ver_natural_store_commit_storage.h" #include "multi_ver_natural_store_connection.h" -#include "default_factory.h" -#include "sqlite_multi_ver_data_storage.h" -#include "sqlite_utils.h" -#include "db_constant.h" #include "process_communicator_test_stub.h" +#include "sqlite_multi_ver_data_storage.h" using namespace testing::ext; using namespace DistributedDB; @@ -249,6 +248,7 @@ void DistributedDBStorageTransactionDataTest::TearDownTestCase(void) void DistributedDBStorageTransactionDataTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); DistributedDBToolsUnitTest::TestDirInit(g_testDir); // KvDBProperties prop; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_transaction_record_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_transaction_record_test.cpp index 471040d3dee61a48941b71cf6b38c4e711ec9f82..2b25f36625397b1b2f20975f67a33bde33ddef94 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_transaction_record_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/storage/distributeddb_storage_transaction_record_test.cpp @@ -15,11 +15,11 @@ #include +#include "db_constant.h" #include "distributeddb_tools_unit_test.h" #include "sqlite_local_kvdb_connection.h" #include "sqlite_multi_ver_data_storage.h" #include "sqlite_utils.h" -#include "db_constant.h" using namespace testing::ext; using namespace DistributedDB; @@ -58,6 +58,7 @@ void DistributedDBStorageTransactionRecordTest::TearDownTestCase(void) void DistributedDBStorageTransactionRecordTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); g_transaction = new (std::nothrow) SQLiteMultiVerTransaction(); ASSERT_NE(g_transaction, nullptr); CipherPassword passwd; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp index 7eace5ee1b8cb0346e886c68e0d739e270752265..d1c463efd1ad1c81412219394051312857f51e7a 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_ability_sync_test.cpp @@ -16,12 +16,12 @@ #include #include "ability_sync.h" -#include "version.h" +#include "distributeddb_tools_unit_test.h" +#include "single_ver_sync_task_context.h" #include "sync_types.h" -#include "vitural_communicator_aggregator.h" -#include "vitural_communicator.h" +#include "version.h" +#include "virtual_communicator_aggregator.h" #include "virtual_single_ver_sync_db_Interface.h" -#include "single_ver_sync_task_context.h" using namespace std; using namespace testing::ext; @@ -35,8 +35,10 @@ namespace { VirtualSingleVerSyncDBInterface *g_syncInterface = nullptr; VirtualCommunicatorAggregator *g_communicatorAggregator = nullptr; + ICommunicator *g_communicatorA = nullptr; ICommunicator *g_communicatorB = nullptr; + std::shared_ptr g_meta = nullptr; } class DistributedDBAbilitySyncTest : public testing::Test { @@ -63,6 +65,7 @@ void DistributedDBAbilitySyncTest::TearDownTestCase(void) void DistributedDBAbilitySyncTest::SetUp(void) { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create the instance for virtual communicator, virtual storage */ @@ -76,6 +79,8 @@ void DistributedDBAbilitySyncTest::SetUp(void) ASSERT_TRUE(g_communicatorA != nullptr); g_communicatorB = g_communicatorAggregator->AllocCommunicator(DEVICE_B, errCode); ASSERT_TRUE(g_communicatorB != nullptr); + g_meta = std::make_shared(); + g_meta->Initialize(g_syncInterface); } void DistributedDBAbilitySyncTest::TearDown(void) @@ -115,10 +120,15 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest001, TestSize.Level0) * @tc.steps: step2. set version = ABILITY_SYNC_VERSION_V1. schema = TEST_SCHEMA. */ AbilitySyncRequestPacket packet1; + DbAbility ability1; +#ifndef OMIT_ZLIB + ability1.SetAbilityItem(DATABASE_COMPRESSION_ZLIB, SUPPORT_MARK); +#endif packet1.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); - packet1.SetSoftwareVersion(SOFTWARE_VERSION_BASE); + packet1.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); packet1.SetSchema(TEST_SCHEMA); packet1.SetSendCode(E_OK); + packet1.SetDbAbility(ability1); Message msg1(ABILITY_SYNC_MESSAGE); msg1.SetMessageType(TYPE_REQUEST); msg1.SetCopiedObject(packet1); @@ -146,10 +156,10 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest001, TestSize.Level0) * @tc.expected: step5. packet1 == packet2 */ EXPECT_TRUE(packet2->GetProtocolVersion() == ABILITY_SYNC_VERSION_V1); - EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_BASE); + EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_CURRENT); EXPECT_TRUE(packet2->GetSendCode() == E_OK); - std::string schema; - packet2->GetSchema(schema); + EXPECT_TRUE(packet2->GetDbAbility() == ability1); + std::string schema = packet2->GetSchema(); EXPECT_EQ(schema, TEST_SCHEMA); } @@ -214,7 +224,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest003, TestSize.Level0) */ AbilitySyncRequestPacket packet1; packet1.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); - packet1.SetSoftwareVersion(SOFTWARE_VERSION_RELEASE_3_0); + packet1.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); packet1.SetSchema(TEST_SCHEMA); packet1.SetSendCode(E_OK); int secLabel = 3; // label 3 @@ -248,10 +258,9 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest003, TestSize.Level0) * @tc.expected: step5. packet1 == packet2 */ EXPECT_TRUE(packet2->GetProtocolVersion() == ABILITY_SYNC_VERSION_V1); - EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_RELEASE_3_0); + EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_CURRENT); EXPECT_TRUE(packet2->GetSendCode() == E_OK); - std::string schema; - packet2->GetSchema(schema); + std::string schema = packet2->GetSchema(); EXPECT_EQ(schema, TEST_SCHEMA); EXPECT_TRUE(packet2->GetSecFlag() == secFlag); EXPECT_TRUE(packet2->GetSecLabel() == secLabel); @@ -308,8 +317,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestPacketTest004, TestSize.Level0) EXPECT_TRUE(packet2->GetProtocolVersion() == ABILITY_SYNC_VERSION_V1); EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_RELEASE_2_0); EXPECT_TRUE(packet2->GetSendCode() == E_OK); - std::string schema; - packet2->GetSchema(schema); + std::string schema = packet2->GetSchema(); EXPECT_EQ(schema, TEST_SCHEMA); EXPECT_TRUE(packet2->GetSecFlag() == 0); EXPECT_TRUE(packet2->GetSecLabel() == 0); @@ -329,10 +337,15 @@ HWTEST_F(DistributedDBAbilitySyncTest, AckPacketTest001, TestSize.Level0) * @tc.steps: step2. set version = ABILITY_SYNC_VERSION_V1. schema = TEST_SCHEMA. */ AbilitySyncAckPacket packet1; + DbAbility ability1; +#ifndef OMIT_ZLIB + ability1.SetAbilityItem(DATABASE_COMPRESSION_ZLIB, SUPPORT_MARK); +#endif packet1.SetProtocolVersion(ABILITY_SYNC_VERSION_V1); packet1.SetSoftwareVersion(SOFTWARE_VERSION_CURRENT); packet1.SetSchema(TEST_SCHEMA); packet1.SetAckCode(E_VERSION_NOT_SUPPORT); + packet1.SetDbAbility(ability1); Message msg1(ABILITY_SYNC_MESSAGE); msg1.SetMessageType(TYPE_RESPONSE); msg1.SetCopiedObject(packet1); @@ -362,8 +375,8 @@ HWTEST_F(DistributedDBAbilitySyncTest, AckPacketTest001, TestSize.Level0) EXPECT_TRUE(packet2->GetProtocolVersion() == ABILITY_SYNC_VERSION_V1); EXPECT_TRUE(packet2->GetSoftwareVersion() == SOFTWARE_VERSION_CURRENT); EXPECT_TRUE(packet2->GetAckCode() == E_VERSION_NOT_SUPPORT); - std::string schema; - packet2->GetSchema(schema); + EXPECT_TRUE(packet2->GetDbAbility() == ability1); + std::string schema = packet2->GetSchema(); ASSERT_TRUE(schema == TEST_SCHEMA); } @@ -380,7 +393,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, SyncStart001, TestSize.Level0) * @tc.steps: step1. create a AbilitySync */ AbilitySync async; - async.Initialize(g_communicatorB, g_syncInterface, DEVICE_A); + async.Initialize(g_communicatorB, g_syncInterface, g_meta, DEVICE_A); /** * @tc.steps: step2. call SyncStart @@ -413,7 +426,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, RequestReceiveTest001, TestSize.Level0) * @tc.steps: step1. create a AbilitySync */ AbilitySync async; - async.Initialize(g_communicatorB, g_syncInterface, DEVICE_A); + async.Initialize(g_communicatorB, g_syncInterface, g_meta, DEVICE_A); /** * @tc.steps: step2. call RequestRecv, set inMsg nullptr or set context nullptr @@ -484,7 +497,7 @@ HWTEST_F(DistributedDBAbilitySyncTest, AckReceiveTest001, TestSize.Level0) * @tc.steps: step1. create a AbilitySync */ AbilitySync async; - async.Initialize(g_communicatorB, g_syncInterface, DEVICE_A); + async.Initialize(g_communicatorB, g_syncInterface, g_meta, DEVICE_A); /** * @tc.steps: step2. call AckRecv, set inMsg nullptr or set context nullptr diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_anti_dos_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_anti_dos_sync_test.cpp index e3b5f9d86e9ea910d373110db85017bcf9a94f90..13e1b2835e1a81c3e4711ae435ebb45c6edbf0fa 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_anti_dos_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_anti_dos_sync_test.cpp @@ -17,15 +17,15 @@ #include #include "distributeddb_data_generate_unit_test.h" +#include "generic_single_ver_kv_entry.h" #include "message.h" #include "meta_data.h" #include "ref_object.h" -#include "single_ver_sync_engine.h" #include "single_ver_data_sync.h" -#include "vitural_communicator_aggregator.h" -#include "virtual_single_ver_sync_db_Interface.h" +#include "single_ver_sync_engine.h" #include "version.h" -#include "generic_single_ver_kv_entry.h" +#include "virtual_communicator_aggregator.h" +#include "virtual_single_ver_sync_db_Interface.h" using namespace testing::ext; using namespace DistributedDB; @@ -35,7 +35,11 @@ using namespace std; namespace { string g_testDir; const string ANTI_DOS_STORE_ID = "anti_dos_sync_test"; +#ifndef RELATIONAL_STORE const int NUM = 108; +#else + const int NUM = 120; +#endif const int WAIT_LONG_TIME = 26000; const int WAIT_SHORT_TIME = 18000; const int TEST_ONE = 2; @@ -94,6 +98,7 @@ void DistributeddbAntiDosSyncTest::TearDownTestCase(void) void DistributeddbAntiDosSyncTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create VirtualCommunicator, VirtualSingleVerSyncDBInterface, SyncEngine, * and set maximum cache of queue. @@ -110,7 +115,7 @@ void DistributeddbAntiDosSyncTest::SetUp(void) ASSERT_TRUE(errCodeMetaData == E_OK); g_syncEngine = new (std::nothrow) SingleVerSyncEngine(); ASSERT_TRUE(g_syncEngine != nullptr); - int errCodeSyncEngine = g_syncEngine->Initialize(g_syncInterface, g_metaData, nullptr); + int errCodeSyncEngine = g_syncEngine->Initialize(g_syncInterface, g_metaData, nullptr, nullptr, nullptr); ASSERT_TRUE(errCodeSyncEngine == E_OK); g_communicator = static_cast(g_communicatorAggregator->GetCommunicator(remoteDeviceId)); ASSERT_TRUE(g_communicator != nullptr); diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_communicator_proxy_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_communicator_proxy_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0d4f923ba6fe8864e2d79ca00e698e884271912b --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_communicator_proxy_test.cpp @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "communicator_proxy.h" +#include "db_constant.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "kv_store_nb_delegate.h" +#include "mock_communicator.h" +#include "platform_specific.h" +#include "virtual_communicator_aggregator.h" + +using namespace testing::ext; +using namespace testing; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { + string g_testDir; + const string STORE_ID = "kv_store_sync_test"; + const std::string DEVICE_B = "deviceB"; + const std::string DEVICE_C = "deviceC"; + const std::string DEVICE_D = "deviceD"; + const std::string DEVICE_E = "deviceE"; + + + KvStoreDelegateManager g_mgr(APP_ID, USER_ID); + KvStoreConfig g_config; + DistributedDBToolsUnitTest g_tool; + DBStatus g_kvDelegateStatus = INVALID_ARGS; + KvStoreNbDelegate* g_kvDelegatePtr = nullptr; + + // the type of g_kvDelegateCallback is function + auto g_kvDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, + placeholders::_1, placeholders::_2, std::ref(g_kvDelegateStatus), std::ref(g_kvDelegatePtr)); +} + +class DistributedDBCommunicatorProxyTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); + +protected: + MockCommunicator extComm_; + MockCommunicator mainComm_; + CommunicatorProxy *commProxy_ = nullptr; +}; + +void DistributedDBCommunicatorProxyTest::SetUpTestCase(void) +{ + /** + * @tc.setup: Init datadir and Virtual Communicator. + */ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + g_config.dataDir = g_testDir; + g_mgr.SetKvStoreConfig(g_config); + + string dir = g_testDir + "/single_ver"; + DIR* dirTmp = opendir(dir.c_str()); + if (dirTmp == nullptr) { + OS::MakeDBDirectory(dir); + } else { + closedir(dirTmp); + } + + auto communicatorAggregator = new (std::nothrow) VirtualCommunicatorAggregator(); + ASSERT_TRUE(communicatorAggregator != nullptr); + RuntimeContext::GetInstance()->SetCommunicatorAggregator(communicatorAggregator); +} + +void DistributedDBCommunicatorProxyTest::TearDownTestCase(void) +{ + /** + * @tc.teardown: Release virtual Communicator and clear data dir. + */ + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error!"); + } + RuntimeContext::GetInstance()->SetCommunicatorAggregator(nullptr); +} + +void DistributedDBCommunicatorProxyTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + /** + * @tc.setup: Get a KvStoreNbDelegate and init the CommunicatorProxy + */ + KvStoreNbDelegate::Option option; + g_mgr.GetKvStore(STORE_ID, option, g_kvDelegateCallback); + ASSERT_TRUE(g_kvDelegateStatus == OK); + ASSERT_TRUE(g_kvDelegatePtr != nullptr); + commProxy_ = new (std::nothrow) CommunicatorProxy(); + ASSERT_TRUE(commProxy_ != nullptr); + commProxy_->SetMainCommunicator(&mainComm_); + commProxy_->SetEqualCommunicator(&extComm_, { DEVICE_C }); +} + +void DistributedDBCommunicatorProxyTest::TearDown(void) +{ + /** + * @tc.teardown: Release the KvStoreNbDelegate and CommunicatorProxy + */ + if (g_kvDelegatePtr != nullptr) { + ASSERT_EQ(g_mgr.CloseKvStore(g_kvDelegatePtr), OK); + g_kvDelegatePtr = nullptr; + DBStatus status = g_mgr.DeleteKvStore(STORE_ID); + LOGD("delete kv store status %d", status); + ASSERT_TRUE(status == OK); + } + if (commProxy_ != nullptr) { + RefObject::DecObjRef(commProxy_); + } + commProxy_ = nullptr; +} + +/** + * @tc.name: Interface set equal 001 + * @tc.desc: Test set equal identifier from interface. + * @tc.type: FUNC + * @tc.require: AR000F4GVG + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBCommunicatorProxyTest, InterfaceSetEqualId001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Call GetKvStoreIdentifier to make a store identifier. + */ + std::string identifier = g_mgr.GetKvStoreIdentifier("default", APP_ID, STORE_ID); + + /** + * @tc.steps: step2. Call SetEqualIdentifier to set the store identifier B, D, E. + * @tc.expected: step2. SetEqualIdentifier return OK. + */ + DBStatus status = g_kvDelegatePtr->SetEqualIdentifier(identifier, { DEVICE_B, DEVICE_D, DEVICE_E }); + EXPECT_EQ(status, DBStatus::OK); + + /** + * @tc.steps: step2. Call SetEqualIdentifier to set the store identifier B. + * @tc.expected: step2. SetEqualIdentifier return OK and D, E will offline. + */ + status = g_kvDelegatePtr->SetEqualIdentifier(identifier, { DEVICE_B }); + EXPECT_EQ(status, DBStatus::OK); +} + +/** + * @tc.name: Register callback 001 + * @tc.desc: Test register callback from CommunicatorProxy. + * @tc.type: FUNC + * @tc.require: AR000F4GVG + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBCommunicatorProxyTest, RegCallBack001, TestSize.Level1) +{ + OnMessageCallback msgCallback; + OnConnectCallback connCallback; + std::function sendableCallback; + Finalizer finalizer; + + /** + * @tc.steps: step1. Call RegOnMessageCallback from CommProxy. + * @tc.expected: step1. mainComm and extComm's RegOnMessageCallback should be called once. + */ + EXPECT_CALL(extComm_, RegOnMessageCallback(_, _)).Times(1); + EXPECT_CALL(mainComm_, RegOnMessageCallback(_, _)).Times(1); + commProxy_->RegOnMessageCallback(msgCallback, finalizer); + + /** + * @tc.steps: step2. Call RegOnConnectCallback from CommProxy. + * @tc.expected: step2. mainComm and extComm's RegOnConnectCallback should be called once. + */ + EXPECT_CALL(extComm_, RegOnConnectCallback(_, _)).Times(1); + EXPECT_CALL(mainComm_, RegOnConnectCallback(_, _)).Times(1); + commProxy_->RegOnConnectCallback(connCallback, finalizer); + + /** + * @tc.steps: step3. Call RegOnSendableCallback from CommProxy. + * @tc.expected: step3. mainComm and extComm's RegOnSendableCallback should be called once. + */ + EXPECT_CALL(extComm_, RegOnSendableCallback(_, _)).Times(1); + EXPECT_CALL(mainComm_, RegOnSendableCallback(_, _)).Times(1); + commProxy_->RegOnSendableCallback(sendableCallback, finalizer); +} + +/** + * @tc.name: Activate 001 + * @tc.desc: Test Activate called from CommunicatorProxy. + * @tc.type: FUNC + * @tc.require: AR000F4GVG + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBCommunicatorProxyTest, Activate001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Call Activate from CommProxy. + * @tc.expected: step1. mainComm and extComm's Activate should be called once. + */ + EXPECT_CALL(extComm_, Activate()).Times(1); + EXPECT_CALL(mainComm_, Activate()).Times(1); + commProxy_->Activate(); +} + +/** + * @tc.name: Get mtu 001 + * @tc.desc: Test mtu called from CommunicatorProxy. + * @tc.type: FUNC + * @tc.require: AR000F4GVG + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBCommunicatorProxyTest, GetMtu001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Call GetCommunicatorMtuSize from CommProxy with no param. + * @tc.expected: step1. GetCommunicatorMtuSize return DBConstant::MIN_MTU_SIZE. + */ + EXPECT_CALL(mainComm_, GetCommunicatorMtuSize()).WillOnce(Return(DBConstant::MIN_MTU_SIZE)); + EXPECT_EQ(commProxy_->GetCommunicatorMtuSize(), DBConstant::MIN_MTU_SIZE); + + /** + * @tc.steps: step2. Call GetCommunicatorMtuSize from CommProxy with param DEVICE_C. + * @tc.expected: step2. GetCommunicatorMtuSize return DBConstant::MAX_MTU_SIZE. + */ + EXPECT_CALL(extComm_, GetCommunicatorMtuSize(DEVICE_C)).WillOnce(Return(DBConstant::MAX_MTU_SIZE)); + EXPECT_EQ(commProxy_->GetCommunicatorMtuSize(DEVICE_C), DBConstant::MAX_MTU_SIZE); +} + +/** + * @tc.name: Get local identify 001 + * @tc.desc: Test Get local identify from CommunicatorProxy. + * @tc.type: FUNC + * @tc.require: AR000F4GVG + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBCommunicatorProxyTest, GetLocalIdentity001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Call GetLocalIdentity from CommProxy, and set mainComm return DEVICE_B. + * @tc.expected: step1. GetCommunicatorMtuSize return DEVICE_B and function call return E_OK. + */ + EXPECT_CALL(mainComm_, GetLocalIdentity(_)).WillOnce(DoAll(SetArgReferee<0>(DEVICE_B), Return(E_OK))); + std::string localId; + EXPECT_EQ(commProxy_->GetLocalIdentity(localId), E_OK); + EXPECT_EQ(localId, DEVICE_B); +} + +/** + * @tc.name: Get remote version 001 + * @tc.desc: Test Get remote version from CommunicatorProxy. + * @tc.type: FUNC + * @tc.require: AR000F4GVG + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBCommunicatorProxyTest, GetRemoteVersion001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Set mainComm called GetRemoteCommunicatorVersion will return SOFTWARE_VERSION_BASE. + */ + EXPECT_CALL(mainComm_, GetRemoteCommunicatorVersion(DEVICE_B, _)) + .WillOnce(DoAll(SetArgReferee<1>(SOFTWARE_VERSION_BASE), Return(E_OK))); + + /** + * @tc.steps: step2. Call GetRemoteCommunicatorVersion from CommProxy with param DEVICE_B. + * @tc.expected: step2. GetRemoteCommunicatorVersion return SOFTWARE_VERSION_BASE and function call return E_OK. + */ + uint16_t version = 0; + EXPECT_EQ(commProxy_->GetRemoteCommunicatorVersion(DEVICE_B, version), E_OK); + EXPECT_EQ(version, SOFTWARE_VERSION_BASE); + + /** + * @tc.steps: step3. Set extComm called GetRemoteCommunicatorVersion will return SOFTWARE_VERSION_CURRENT. + */ + EXPECT_CALL(extComm_, GetRemoteCommunicatorVersion(DEVICE_C, _)) + .WillOnce(DoAll(SetArgReferee<1>(SOFTWARE_VERSION_CURRENT), Return(E_OK))); + + /** + * @tc.steps: step4. Call GetRemoteCommunicatorVersion from CommProxy with param DEVICE_C. + * @tc.expected: step4. GetRemoteCommunicatorVersion return SOFTWARE_VERSION_CURRENT and function call return E_OK. + */ + EXPECT_EQ(commProxy_->GetRemoteCommunicatorVersion(DEVICE_C, version), E_OK); + EXPECT_EQ(version, SOFTWARE_VERSION_CURRENT); +} + +/** + * @tc.name: Send message 001 + * @tc.desc: Test Send message from CommunicatorProxy. + * @tc.type: FUNC + * @tc.require: AR000F4GVG + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBCommunicatorProxyTest, SendMessage001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Call SendMessage from CommProxy with param DEVICE_B. + * @tc.expected: step1. MainComm's SendMessage willed called and return E_OK. + */ + EXPECT_CALL(mainComm_, SendMessage(DEVICE_B, _, _, _, _)).WillOnce(Return(E_OK)); + EXPECT_EQ(commProxy_->SendMessage(DEVICE_B, nullptr, true, 0, nullptr), E_OK); + + /** + * @tc.steps: step1. Call SendMessage from CommProxy with param DEVICE_C. + * @tc.expected: step1. ExtComm's SendMessage willed called and return E_OK. + */ + EXPECT_CALL(extComm_, SendMessage(DEVICE_C, _, _, _, _)).WillOnce(Return(E_OK)); + EXPECT_EQ(commProxy_->SendMessage(DEVICE_C, nullptr, true, 0, nullptr), E_OK); +} + +/** + * @tc.name: Get timeout time 001 + * @tc.desc: Test get timout called from CommunicatorProxy. + * @tc.type: FUNC + * @tc.require: AR000F4GVG + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBCommunicatorProxyTest, GetTimeout001, TestSize.Level1) +{ + /** + * @tc.steps: step1. Call GetTimeout from CommProxy with no param. + * @tc.expected: step1. GetTimeout return DBConstant::MIN_TIMEOUT. + */ + EXPECT_CALL(mainComm_, GetTimeout()).WillOnce(Return(DBConstant::MIN_TIMEOUT)); + EXPECT_EQ(commProxy_->GetTimeout(), DBConstant::MIN_TIMEOUT); + + /** + * @tc.steps: step2. Call GetTimeout from CommProxy with param DEVICE_C. + * @tc.expected: step2. GetTimeout return DBConstant::MAX_MTU_SIZE. + */ + EXPECT_CALL(extComm_, GetTimeout(DEVICE_C)).WillOnce(Return(DBConstant::MAX_TIMEOUT)); + EXPECT_EQ(commProxy_->GetTimeout(DEVICE_C), DBConstant::MAX_TIMEOUT); +} diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_multi_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_multi_ver_p2p_sync_test.cpp index 920bd7feea86540e9ba9da581e75938ac5f76aaf..5b6605ab90c575c283d0ceef967db392772de547 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_multi_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_multi_ver_p2p_sync_test.cpp @@ -16,26 +16,22 @@ #include #include -#include "distributeddb_tools_unit_test.h" +#include "commit_history_sync.h" +#include "db_common.h" #include "distributeddb_data_generate_unit_test.h" -#include "kv_store_observer.h" +#include "distributeddb_tools_unit_test.h" +#include "ikvdb_connection.h" #include "kv_store_delegate.h" -#include "vitural_communicator_aggregator.h" -#include "vitural_communicator.h" -#include "vitural_device.h" -#include "isyncer.h" -#include "virtual_multi_ver_sync_db_interface.h" -#include "time_sync.h" -#include "meta_data.h" +#include "kv_virtual_device.h" #include "kvdb_manager.h" #include "kvdb_pragma.h" -#include "ikvdb_connection.h" -#include "sync_types.h" -#include "commit_history_sync.h" #include "log_print.h" +#include "meta_data.h" #include "multi_ver_data_sync.h" #include "platform_specific.h" -#include "db_common.h" +#include "sync_types.h" +#include "time_sync.h" +#include "virtual_multi_ver_sync_db_interface.h" using namespace testing::ext; using namespace DistributedDB; @@ -45,9 +41,9 @@ using namespace std; #ifndef LOW_LEVEL_MEM_DEV namespace { string g_testDir; - const string STORE_ID = "kv_stroe_sync_test"; - const string STORE_ID_A = "kv_stroe_sync_test_a"; - const string STORE_ID_B = "kv_stroe_sync_test_b"; + const string STORE_ID = "kv_store_sync_test"; + const string STORE_ID_A = "kv_store_sync_test_a"; + const string STORE_ID_B = "kv_store_sync_test_b"; const int WAIT_TIME_1 = 1000; const int WAIT_TIME_2 = 2000; const int WAIT_LONG_TIME = 10000; @@ -67,8 +63,8 @@ namespace { MultiVerNaturalStoreConnection *g_connectionA; MultiVerNaturalStoreConnection *g_connectionB; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; - VituralDevice* g_deviceB = nullptr; - VituralDevice* g_deviceC = nullptr; + KvVirtualDevice *g_deviceB = nullptr; + KvVirtualDevice *g_deviceC = nullptr; // the type of g_kvDelegateCallback is function auto g_kvDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreDelegateCallback, @@ -153,18 +149,19 @@ void DistributedDBMultiVerP2PSyncTest::TearDownTestCase(void) void DistributedDBMultiVerP2PSyncTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create virtual device B and C */ g_communicatorAggregator->Disable(); - g_deviceB = new (std::nothrow) VituralDevice(DEVICE_B); + g_deviceB = new (std::nothrow) KvVirtualDevice(DEVICE_B); ASSERT_TRUE(g_deviceB != nullptr); VirtualMultiVerSyncDBInterface *syncInterfaceB = new (std::nothrow) VirtualMultiVerSyncDBInterface; ASSERT_TRUE(syncInterfaceB != nullptr); ASSERT_EQ(syncInterfaceB->Initialize(DEVICE_B), E_OK); ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, syncInterfaceB), E_OK); - g_deviceC = new (std::nothrow) VituralDevice(DEVICE_C); + g_deviceC = new (std::nothrow) KvVirtualDevice(DEVICE_C); ASSERT_TRUE(g_deviceC != nullptr); VirtualMultiVerSyncDBInterface *syncInterfaceC = new (std::nothrow) VirtualMultiVerSyncDBInterface; ASSERT_TRUE(syncInterfaceC != nullptr); @@ -506,7 +503,7 @@ HWTEST_F(DistributedDBMultiVerP2PSyncTest, IsolationSync001, TestSize.Level2) * @tc.expected: step3. Pragma OK, connectionA have {k1, v1} , connectionB don't have k1. */ PragmaSync pragmaData(devices, SYNC_MODE_PULL_ONLY, nullptr); - ASSERT_TRUE(g_connectionA->Pragma(PRAGMA_SYNC_DEVICES, &pragmaData) > 0); + ASSERT_TRUE(g_connectionA->Pragma(PRAGMA_SYNC_DEVICES, &pragmaData) == E_OK); std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_TIME_2)); Value value; ASSERT_EQ(GetDataFromConnection(g_connectionA, DistributedDBUnitTest::KEY_1, value), E_OK); @@ -550,7 +547,7 @@ HWTEST_F(DistributedDBMultiVerP2PSyncTest, IsolationSync002, TestSize.Level2) * @tc.expected: step3. Pragma OK, connectionA have {k1, v2} , connectionB don't have k1. */ PragmaSync pragmaData(devices, SYNC_MODE_PULL_ONLY, nullptr); - ASSERT_TRUE(g_connectionA->Pragma(PRAGMA_SYNC_DEVICES, &pragmaData) > 0); + ASSERT_TRUE(g_connectionA->Pragma(PRAGMA_SYNC_DEVICES, &pragmaData) == E_OK); std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_TIME_2)); Value value; @@ -598,7 +595,7 @@ HWTEST_F(DistributedDBMultiVerP2PSyncTest, IsolationSync003, TestSize.Level2) */ LOGD("[DistributeddbMultiVerP2PSyncTes] start sync"); PragmaSync pragmaData(devices, SYNC_MODE_PULL_ONLY, nullptr); - ASSERT_TRUE(g_connectionA->Pragma(PRAGMA_SYNC_DEVICES, &pragmaData) > 0); + ASSERT_TRUE(g_connectionA->Pragma(PRAGMA_SYNC_DEVICES, &pragmaData) == E_OK); std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_TIME_2)); Value value; @@ -633,7 +630,7 @@ static bool IsTimeSyncPacketEqual(const TimeSyncPacket &inPacketA, const TimeSyn * @tc.require: AR000BVRNU AR000CQE0J * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBMultiVerP2PSyncTest, TimesyncPacket001, TestSize.Level0) +HWTEST_F(DistributedDBMultiVerP2PSyncTest, TimesyncPacket001, TestSize.Level1) { /** * @tc.steps: step1. create TimeSyncPacket packetA aand packetB @@ -799,7 +796,7 @@ static bool IsCommitHistorySyncRequestPacketEqual(const CommitHistorySyncRequest * @tc.require: AR000BVRNU AR000CQE0J * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBMultiVerP2PSyncTest, CommitHistorySyncRequestPacket001, TestSize.Level0) +HWTEST_F(DistributedDBMultiVerP2PSyncTest, CommitHistorySyncRequestPacket001, TestSize.Level1) { /** * @tc.steps: step1. create CommitHistorySyncRequestPacket packetA aand packetB @@ -915,7 +912,7 @@ static bool IsCommitHistorySyncAckPacketEqual(const CommitHistorySyncAckPacket & * @tc.require: AR000BVRNU AR000CQE0J * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBMultiVerP2PSyncTest, CommitHistorySyncAckPacket001, TestSize.Level0) +HWTEST_F(DistributedDBMultiVerP2PSyncTest, CommitHistorySyncAckPacket001, TestSize.Level1) { /** * @tc.steps: step1. create CommitHistorySyncAckPacket packetA aand packetB @@ -996,7 +993,7 @@ static bool IsMultiVerRequestPacketEqual(const MultiVerRequestPacket &inPacketA, * @tc.require: AR000BVRNU AR000CQE0J * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBMultiVerP2PSyncTest, MultiVerRequestPacket001, TestSize.Level0) +HWTEST_F(DistributedDBMultiVerP2PSyncTest, MultiVerRequestPacket001, TestSize.Level1) { /** * @tc.steps: step1. create CommitHistorySyncAckPacket packetA aand packetB @@ -1106,7 +1103,7 @@ static bool IsMultiVerAckPacketEqual(const MultiVerAckPacket &inPacketA, const M * @tc.require: AR000BVRNU AR000CQE0J * @tc.author: xiaozhenjian */ -HWTEST_F(DistributedDBMultiVerP2PSyncTest, MultiVerAckPacket001, TestSize.Level0) +HWTEST_F(DistributedDBMultiVerP2PSyncTest, MultiVerAckPacket001, TestSize.Level1) { /** * @tc.steps: step1. create MultiVerAckPacket packetA aand packetB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c24f02a1e3bd7619a22ff6498271455bb5342c71 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_relational_ver_p2p_sync_test.cpp @@ -0,0 +1,396 @@ +/* +* Copyright (c) 2021 Huawei Device Co., Ltd. +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +#ifdef RELATIONAL_STORE +#include + +#include "db_common.h" +#include "db_constant.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "isyncer.h" +#include "kv_virtual_device.h" +#include "platform_specific.h" +#include "relational_schema_object.h" +#include "relational_store_manager.h" +#include "relational_virtual_device.h" +#include "runtime_config.h" +#include "virtual_relational_ver_sync_db_interface.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; + +namespace { + const std::string DEVICE_B = "deviceB"; + const std::string g_tableName = "TEST_TABLE"; + + RelationalStoreManager g_mgr(APP_ID, USER_ID); + std::string g_testDir; + std::string g_dbDir; + DistributedDBToolsUnitTest g_tool; + DBStatus g_kvDelegateStatus = INVALID_ARGS; + RelationalStoreDelegate* g_kvDelegatePtr = nullptr; + VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; + RelationalVirtualDevice *g_deviceB = nullptr; + + // the type of g_kvDelegateCallback is function + auto g_kvDelegateCallback = [](DBStatus status, RelationalStoreDelegate *delegatePtr) { + g_kvDelegateStatus = status; + g_kvDelegatePtr = delegatePtr; + }; + + int GetDB(sqlite3 *&db) + { + int flag = SQLITE_OPEN_URI | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + const auto &dbPath = g_dbDir; + int rc = sqlite3_open_v2(dbPath.c_str(), &db, flag, nullptr); + if (rc != SQLITE_OK) { + return rc; + } + EXPECT_EQ(SQLiteUtils::RegisterCalcHash(db), E_OK); + EXPECT_EQ(SQLiteUtils::RegisterGetSysTime(db), E_OK); + return rc; + } + + int CreateTable(sqlite3 *db) + { + char *pErrMsg = nullptr; + + const char *sql1 = "CREATE TABLE TEST_TABLE(" \ + "ID INT PRIMARY KEY NOT NULL," \ + "NAME TEXT ," \ + "AGE INT);"; + return sqlite3_exec(db, sql1, nullptr, nullptr, &pErrMsg); + } + + int PrepareInsert(sqlite3 *db, sqlite3_stmt *&statement) + { + const char *sql = "INSERT OR REPLACE INTO TEST_TABLE (ID,NAME,AGE) " \ + "VALUES (?, ?, ?);"; + return sqlite3_prepare_v2(db, sql, -1, &statement, nullptr); + } + + int SimulateCommitData(sqlite3 *db, sqlite3_stmt *&statement) + { + sqlite3_exec(db, "BEGIN IMMEDIATE TRANSACTION", nullptr, nullptr, nullptr); + + int rc = sqlite3_step(statement); + + sqlite3_exec(db, "COMMIT TRANSACTION", nullptr, nullptr, nullptr); + return rc; + } + + void InsertValue(sqlite3 *db, RowData &rowData) + { + sqlite3_stmt *stmt = nullptr; + EXPECT_EQ(PrepareInsert(db, stmt), SQLITE_OK); + for (int i = 0; i < (int)rowData.size(); ++i) { + const auto &item = rowData[i]; + const int index = i + 1; + int errCode; + if (item.GetType() == StorageType::STORAGE_TYPE_INTEGER) { + int64_t val; + (void)item.GetInt64(val); + errCode = SQLiteUtils::BindInt64ToStatement(stmt, index, val); + } else if (item.GetType() == StorageType::STORAGE_TYPE_TEXT) { + std::string val; + (void)item.GetText(val); + errCode = SQLiteUtils::BindTextToStatement(stmt, index, val); + } + EXPECT_EQ(errCode, E_OK); + } + EXPECT_EQ(SimulateCommitData(db, stmt), SQLITE_DONE); + } + + void GenerateValue(RowData &rowData, std::map &dataMap) + { + int64_t id = 0; + dataMap["ID"] = id; + rowData.push_back(dataMap["ID"]); + std::string val = "test"; + dataMap["NAME"] = val; + rowData.push_back(dataMap["NAME"]); + dataMap["AGE"] = INT64_MAX; + rowData.push_back(dataMap["AGE"]); + } + + void InsertFieldInfo() + { + std::vector localFieldInfo; + FieldInfo columnFirst; + columnFirst.SetFieldName("ID"); + columnFirst.SetStorageType(StorageType::STORAGE_TYPE_INTEGER); + FieldInfo columnSecond; + columnSecond.SetFieldName("NAME"); + columnSecond.SetStorageType(StorageType::STORAGE_TYPE_TEXT); + FieldInfo columnThird; + columnThird.SetFieldName("AGE"); + columnThird.SetStorageType(StorageType::STORAGE_TYPE_INTEGER); + localFieldInfo.push_back(columnFirst); + localFieldInfo.push_back(columnSecond); + localFieldInfo.push_back(columnThird); + g_deviceB->SetLocalFieldInfo(localFieldInfo); + } + + void BlockSync(SyncMode syncMode, DBStatus exceptStatus) + { + std::vector devices = {DEVICE_B}; + Query query = Query::Select(g_tableName); + std::mutex syncLock; + std::condition_variable syncCondVar; + std::map> statusMap; + std::mutex syncMutex; + std::condition_variable syncCv; + SyncStatusCallback callBack = [&statusMap, &syncCv]( + const std::map> &devicesMap) { + statusMap = devicesMap; + syncCv.notify_one(); + }; + DBStatus callStatus = g_kvDelegatePtr->Sync(devices, syncMode, callBack, query, false); + std::unique_lock uniqueLock(syncMutex); + syncCv.wait(uniqueLock, [&statusMap]() { + return !statusMap.empty(); + }); + EXPECT_EQ(callStatus, OK); + for (const auto &tablesRes : statusMap) { + for (const auto &tableStatus : tablesRes.second) { + EXPECT_EQ(tableStatus.status, exceptStatus); + } + } + } + + void InsertValueToDB(RowData &rowData) + { + sqlite3 *db = nullptr; + EXPECT_EQ(GetDB(db), SQLITE_OK); + InsertValue(db, rowData); + sqlite3_close(db); + } + + void PrepareEnvironment(RowData &rowData) + { + sqlite3 *db = nullptr; + EXPECT_EQ(GetDB(db), SQLITE_OK); + EXPECT_EQ(CreateTable(db), SQLITE_OK); + + EXPECT_EQ(g_kvDelegatePtr->CreateDistributedTable(g_tableName, {}), OK); + + sqlite3_close(db); + + InsertFieldInfo(); + std::map dataMap; + GenerateValue(rowData, dataMap); + InsertValueToDB(rowData); + + rowData.clear(); + for (const auto &item : dataMap) { + rowData.push_back(item.second); + } + } + + void CheckVirtualData(RowData &rowData) + { + std::vector targetData; + g_deviceB->GetAllSyncData(g_tableName, targetData); + + for (auto &item : targetData) { + for (int j = 0; j < (int)item.rowData.size(); ++j) { + LOGD("index %d actual_val[%s] except_val[%s]", + j, item.rowData[j].ToString().c_str(), rowData[j].ToString().c_str()); + EXPECT_TRUE(item.rowData[j] == rowData[j]); + } + } + } +} + +class DistributedDBRelationalVerP2PSyncTest : public testing::Test { +public: + static void SetUpTestCase(); + static void TearDownTestCase(); + void SetUp(); + void TearDown(); +}; + +void DistributedDBRelationalVerP2PSyncTest::SetUpTestCase() +{ + /** + * @tc.setup: Init datadir and Virtual Communicator. + */ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + g_dbDir = g_testDir + "/test.db"; + + g_communicatorAggregator = new (std::nothrow) VirtualCommunicatorAggregator(); + ASSERT_TRUE(g_communicatorAggregator != nullptr); + RuntimeContext::GetInstance()->SetCommunicatorAggregator(g_communicatorAggregator); +} + +void DistributedDBRelationalVerP2PSyncTest::TearDownTestCase() +{ + /** + * @tc.teardown: Release virtual Communicator and clear data dir. + */ + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error!"); + } + RuntimeContext::GetInstance()->SetCommunicatorAggregator(nullptr); + LOGD("TearDownTestCase FINISH"); +} + +void DistributedDBRelationalVerP2PSyncTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + /** + * @tc.setup: create virtual device B, and get a KvStoreNbDelegate as deviceA + */ + g_mgr.OpenStore(g_dbDir, {true}, g_kvDelegateCallback); + ASSERT_TRUE(g_kvDelegateStatus == OK); + ASSERT_TRUE(g_kvDelegatePtr != nullptr); + g_deviceB = new (std::nothrow) RelationalVirtualDevice(DEVICE_B); + ASSERT_TRUE(g_deviceB != nullptr); + auto *syncInterfaceB = new (std::nothrow) VirtualRelationalVerSyncDBInterface(); + ASSERT_TRUE(syncInterfaceB != nullptr); + ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, syncInterfaceB), E_OK); + + auto permissionCheckCallback = [] (const std::string &userId, const std::string &appId, const std::string &storeId, + const std::string &deviceId, uint8_t flag) -> bool { + return true; + }; + EXPECT_EQ(RuntimeConfig::SetPermissionCheckCallback(permissionCheckCallback), OK); +} + +void DistributedDBRelationalVerP2PSyncTest::TearDown(void) +{ + /** + * @tc.teardown: Release device A, B, C + */ + if (g_kvDelegatePtr != nullptr) { + LOGD("CloseStore Start"); + ASSERT_EQ(g_mgr.CloseStore(g_kvDelegatePtr), OK); + g_kvDelegatePtr = nullptr; + LOGD("DeleteStore Start"); + DBStatus status = g_mgr.DeleteStore(g_dbDir); + LOGD("delete kv store status %d", status); + ASSERT_TRUE(status == OK); + } + if (g_deviceB != nullptr) { + delete g_deviceB; + g_deviceB = nullptr; + } + PermissionCheckCallbackV2 nullCallback; + EXPECT_EQ(RuntimeConfig::SetPermissionCheckCallback(nullCallback), OK); + LOGD("TearDown FINISH"); +} + +/** +* @tc.name: Normal Sync 001 +* @tc.desc: Test normal push sync for add data. +* @tc.type: FUNC +* @tc.require: +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync001, TestSize.Level1) +{ + RowData rowData; + PrepareEnvironment(rowData); + BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); + + CheckVirtualData(rowData); +} + +/** +* @tc.name: Normal Sync 002 +* @tc.desc: Test normal pull sync for add data. +* @tc.type: FUNC +* @tc.require: +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync002, TestSize.Level1) +{ + RowData rowData; + PrepareEnvironment(rowData); + + std::vector devices = {DEVICE_B}; + Query query = Query::Select(g_tableName); + g_deviceB->GenericVirtualDevice::Sync(DistributedDB::SYNC_MODE_PULL_ONLY, query, true); + + CheckVirtualData(rowData); +} + +/** +* @tc.name: Normal Sync 003 +* @tc.desc: Test normal push sync for update data. +* @tc.type: FUNC +* @tc.require: +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync003, TestSize.Level1) +{ + RowData rowData; + PrepareEnvironment(rowData); + + BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); + + CheckVirtualData(rowData); + + int64_t val = 0; + + std::map dataMap; + rowData.clear(); + GenerateValue(rowData, dataMap); + EXPECT_EQ(rowData[rowData.size() - 1].GetInt64(val), E_OK); + rowData[rowData.size() - 1] = static_cast(1); + InsertValueToDB(rowData); + BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); + + rowData.clear(); + dataMap["AGE"] = static_cast(1); + for (const auto &item : dataMap) { + rowData.push_back(item.second); + } + CheckVirtualData(rowData); +} + +/** +* @tc.name: Normal Sync 004 +* @tc.desc: Test normal push sync for delete data. +* @tc.type: FUNC +* @tc.require: +* @tc.author: zhangqiquan +*/ +HWTEST_F(DistributedDBRelationalVerP2PSyncTest, NormalSync004, TestSize.Level1) +{ + RowData rowData; + PrepareEnvironment(rowData); + + BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); + + CheckVirtualData(rowData); + + sqlite3 *db = nullptr; + EXPECT_EQ(GetDB(db), SQLITE_OK); + std::string sql = "DELETE FROM TEST_TABLE WHERE ID = 0"; + EXPECT_EQ(SQLiteUtils::ExecuteRawSQL(db, sql), E_OK); + sqlite3_close(db); + + BlockSync(SyncMode::SYNC_MODE_PUSH_ONLY, OK); + + std::vector dataList; + EXPECT_EQ(g_deviceB->GetAllSyncData(g_tableName, dataList), E_OK); + EXPECT_EQ(static_cast(dataList.size()), 1); + for (const auto &item : dataList) { + EXPECT_EQ(item.logInfo.flag, DataItem::DELETE_FLAG); + } +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e9c5bc4afaadb2fd78d42a72b200442f812f6260 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_query_sync_test.cpp @@ -0,0 +1,1570 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "db_common.h" +#include "db_constant.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "generic_single_ver_kv_entry.h" +#include "kv_store_nb_delegate.h" +#include "kv_virtual_device.h" +#include "platform_specific.h" +#include "query.h" +#include "query_sync_object.h" +#include "single_ver_data_sync.h" +#include "single_ver_serialize_manager.h" +#include "sync_types.h" +#include "virtual_communicator.h" +#include "virtual_communicator_aggregator.h" +#include "virtual_single_ver_sync_db_Interface.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { + string g_testDir; + const string STORE_ID = "kv_store_sync_test"; + const string SCHEMA_STORE_ID = "kv_store_sync_schema_test"; + const std::string DEVICE_B = "deviceB"; + + KvStoreDelegateManager g_mgr(APP_ID, USER_ID); + KvStoreDelegateManager g_schemaMgr(SCHEMA_APP_ID, USER_ID); + KvStoreConfig g_config; + DistributedDBToolsUnitTest g_tool; + DBStatus g_kvDelegateStatus = INVALID_ARGS; + DBStatus g_schemaKvDelegateStatus = INVALID_ARGS; + KvStoreNbDelegate* g_kvDelegatePtr = nullptr; + KvStoreNbDelegate* g_schemaKvDelegatePtr = nullptr; + VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; + KvVirtualDevice *g_deviceB = nullptr; + + // the type of g_kvDelegateCallback is function + auto g_kvDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, + placeholders::_1, placeholders::_2, std::ref(g_kvDelegateStatus), std::ref(g_kvDelegatePtr)); + auto g_schemaKvDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, + placeholders::_1, placeholders::_2, std::ref(g_schemaKvDelegateStatus), std::ref(g_schemaKvDelegatePtr)); + const string SCHEMA_STRING = + "{\"SCHEMA_VERSION\":\"1.0\"," + "\"SCHEMA_MODE\":\"STRICT\"," + "\"SCHEMA_DEFINE\":{" + "\"field_name1\":\"BOOL\"," + "\"field_name2\":\"BOOL\"," + "\"field_name3\":\"INTEGER, NOT NULL\"," + "\"field_name4\":\"LONG, DEFAULT 100\"," + "\"field_name5\":\"DOUBLE, NOT NULL, DEFAULT 3.14\"," + "\"field_name6\":\"STRING, NOT NULL, DEFAULT '3.1415'\"," + "\"field_name7\":\"LONG, DEFAULT 100\"," + "\"field_name8\":\"LONG, DEFAULT 100\"," + "\"field_name9\":\"LONG, DEFAULT 100\"," + "\"field_name10\":\"LONG, DEFAULT 100\"" + "}," + "\"SCHEMA_INDEXES\":[\"$.field_name1\", \"$.field_name2\"]}"; + + const std::string SCHEMA_VALUE1 = + "{\"field_name1\":true," + "\"field_name2\":false," + "\"field_name3\":10," + "\"field_name4\":20," + "\"field_name5\":3.14," + "\"field_name6\":\"3.1415\"," + "\"field_name7\":100," + "\"field_name8\":100," + "\"field_name9\":100," + "\"field_name10\":100}"; + + const std::string SCHEMA_VALUE2 = + "{\"field_name1\":false," + "\"field_name2\":true," + "\"field_name3\":100," + "\"field_name4\":200," + "\"field_name5\":3.14," + "\"field_name6\":\"3.1415\"," + "\"field_name7\":100," + "\"field_name8\":100," + "\"field_name9\":100," + "\"field_name10\":100}"; +} + +class DistributedDBSingleVerP2PQuerySyncTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBSingleVerP2PQuerySyncTest::SetUpTestCase(void) +{ + /** + * @tc.setup: Init datadir and Virtual Communicator. + */ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + string dir = g_testDir + "/single_ver"; + DIR* dirTmp = opendir(dir.c_str()); + if (dirTmp == nullptr) { + OS::MakeDBDirectory(dir); + } else { + closedir(dirTmp); + } + + g_communicatorAggregator = new (std::nothrow) VirtualCommunicatorAggregator(); + ASSERT_TRUE(g_communicatorAggregator != nullptr); + RuntimeContext::GetInstance()->SetCommunicatorAggregator(g_communicatorAggregator); +} + +void DistributedDBSingleVerP2PQuerySyncTest::TearDownTestCase(void) +{ + /** + * @tc.teardown: Release virtual Communicator and clear data dir. + */ + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error!"); + } + RuntimeContext::GetInstance()->SetCommunicatorAggregator(nullptr); +} + +void DistributedDBSingleVerP2PQuerySyncTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + /** + * @tc.setup: create virtual device B and get a KvStoreNbDelegate as deviceA + */ + g_deviceB = new (std::nothrow) KvVirtualDevice(DEVICE_B); + ASSERT_TRUE(g_deviceB != nullptr); + VirtualSingleVerSyncDBInterface *syncInterfaceB = new (std::nothrow) VirtualSingleVerSyncDBInterface(); + ASSERT_TRUE(syncInterfaceB != nullptr); + ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, syncInterfaceB), E_OK); +} + +void DistributedDBSingleVerP2PQuerySyncTest::TearDown(void) +{ + /** + * @tc.teardown: Release device A, B + */ + if (g_kvDelegatePtr != nullptr) { + ASSERT_EQ(g_mgr.CloseKvStore(g_kvDelegatePtr), OK); + g_kvDelegatePtr = nullptr; + DBStatus status = g_mgr.DeleteKvStore(STORE_ID); + LOGD("delete kv store status %d", status); + ASSERT_TRUE(status == OK); + } + if (g_schemaKvDelegatePtr != nullptr) { + ASSERT_EQ(g_schemaMgr.CloseKvStore(g_schemaKvDelegatePtr), OK); + g_schemaKvDelegatePtr = nullptr; + DBStatus status = g_schemaMgr.DeleteKvStore(SCHEMA_STORE_ID); + LOGD("delete kv store status %d", status); + ASSERT_TRUE(status == OK); + } + if (g_deviceB != nullptr) { + delete g_deviceB; + g_deviceB = nullptr; + } + PermissionCheckCallbackV2 nullCallback; + EXPECT_EQ(g_mgr.SetPermissionCheckCallback(nullCallback), OK); +} + +void InitNormalDb() +{ + g_config.dataDir = g_testDir; + g_mgr.SetKvStoreConfig(g_config); + KvStoreNbDelegate::Option option; + g_mgr.GetKvStore(STORE_ID, option, g_kvDelegateCallback); + ASSERT_TRUE(g_kvDelegateStatus == OK); + ASSERT_TRUE(g_kvDelegatePtr != nullptr); +} + +void InitSchemaDb() +{ + g_config.dataDir = g_testDir; + g_schemaMgr.SetKvStoreConfig(g_config); + KvStoreNbDelegate::Option option; + option.schema = SCHEMA_STRING; + g_schemaMgr.GetKvStore(SCHEMA_STORE_ID, option, g_schemaKvDelegateCallback); + ASSERT_TRUE(g_schemaKvDelegateStatus == OK); + ASSERT_TRUE(g_schemaKvDelegatePtr != nullptr); +} + +/** + * @tc.name: Normal Sync 001 + * @tc.desc: Test normal push sync for keyprefix data. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, NormalSync001, TestSize.Level1) +{ + InitNormalDb(); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step1. deviceA put {k0, v0} - {k9, v9} + */ + Key key = {'1'}; + Value value = {'1'}; + const int dataSize = 10; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + value.push_back(i); + status = g_kvDelegatePtr->Put(key, value); + ASSERT_TRUE(status == OK); + key.pop_back(); + value.pop_back(); + } + Key key2 = {'2'}; + Value value2 = {'2'}; + status = g_kvDelegatePtr->Put(key2, value2); + ASSERT_TRUE(status == OK); + + /** + * @tc.steps: step2. deviceA call query sync and wait + * @tc.expected: step2. sync should return OK. + */ + Query query = Query::Select().PrefixKey(key); + std::map result; + status = g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result, query); + ASSERT_TRUE(status == OK); + + /** + * @tc.expected: step3. onComplete should be called, DeviceB have {k1,v1} - {k9, v9} + */ + ASSERT_TRUE(result.size() == devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + value.push_back(i); + g_deviceB->GetData(key, item); + EXPECT_TRUE(item.value == value); + key.pop_back(); + value.pop_back(); + } + EXPECT_TRUE(g_deviceB->GetData(key2, item) != E_OK); +} + +/** + * @tc.name: Normal Sync 002 + * @tc.desc: Test normal push sync for limit and offset. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: xushaohua + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, NormalSync002, TestSize.Level1) +{ + InitNormalDb(); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step1. deviceA put {k0, v0} - {k9, v9} + */ + Key key = {'1'}; + Value value = {'1'}; + const int dataSize = 10; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + value.push_back(i); + status = g_kvDelegatePtr->Put(key, value); + ASSERT_TRUE(status == OK); + key.pop_back(); + value.pop_back(); + } + + /** + * @tc.steps: step2. deviceA call sync and wait + * @tc.expected: step2. sync should return OK. + */ + const int limit = 5; + const int offset = 4; + Query query = Query::Select().PrefixKey(key).Limit(limit, offset); + std::map result; + status = g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result, query); + ASSERT_TRUE(status == OK); + + /** + * @tc.expected: step3. onComplete should be called, DeviceB have {k4,v4} {k8, v8} + */ + ASSERT_TRUE(result.size() == devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + + VirtualDataItem item; + for (int i = limit - 1; i < limit + offset; i++) { + key.push_back(i); + value.push_back(i); + g_deviceB->GetData(key, item); + EXPECT_TRUE(item.value == value); + key.pop_back(); + value.pop_back(); + } +} + +/** + * @tc.name: Normal Sync 001 + * @tc.desc: Test normal push_and_pull sync for keyprefix data. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, NormalSync003, TestSize.Level1) +{ + InitNormalDb(); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step1. deviceA put {k, v}, {b, v} + */ + Key key = {'1'}; + Value value = {'1'}; + const int dataSize = 10; + status = g_kvDelegatePtr->Put(key, value); + Key key2 = {'2'}; + Value value2 = {'2'}; + status = g_kvDelegatePtr->Put(key2, value2); + + /** + * @tc.steps: step2. deviceB put {b0, v0} - {b9, v9}, {c, v} + */ + for (int i = 0; i < dataSize; i++) { + key2.push_back(i); + value2.push_back(i); + g_deviceB->PutData(key2, value2, 10 + i, 0); + key2.pop_back(); + value2.pop_back(); + } + Key key3 = {'3'}; + Value value3 = {'3'}; + g_deviceB->PutData(key3, value3, 20, 0); + + /** + * @tc.steps: step2. deviceA call query sync and wait + * @tc.expected: step2. sync should return OK. + */ + Query query = Query::Select().PrefixKey(key2); + std::map result; + status = g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_PULL, result, query); + ASSERT_TRUE(status == OK); + + /** + * @tc.expected: step3. onComplete should be called, DeviceA have {b0, v0} - {b9, v9}, DeviceB have {b, v} + */ + ASSERT_TRUE(result.size() == devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + Value tmpValue; + for (int i = 0; i < dataSize; i++) { + key2.push_back(i); + value2.push_back(i); + g_kvDelegatePtr->Get(key2, tmpValue); + EXPECT_TRUE(tmpValue == value2); + key2.pop_back(); + value2.pop_back(); + } + EXPECT_TRUE(g_deviceB->GetData(key, item) != E_OK); + EXPECT_TRUE(g_deviceB->GetData(key2, item) == E_OK); + g_kvDelegatePtr->Get(key3, tmpValue); + EXPECT_TRUE(tmpValue != value3); +} + +/** + * @tc.name: Normal Sync 001 + * @tc.desc: Test normal pull sync for keyprefix data. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, NormalSync004, TestSize.Level1) +{ + InitNormalDb(); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + /** + * @tc.steps: step1. deviceB put {k1, v1} - {k9, k9}, {b0, v0} - {b9, v9} + */ + Key key = {'1'}; + Value value = {'1'}; + const int dataSize = 10; + Key key2 = {'2'}; + Value value2 = {'2'}; + vector> key1Vec; + vector> key2Vec; + for (int i = 0; i < dataSize; i++) { + Key tmpKey(key); + Value tmpValue(value); + tmpKey.push_back(i); + tmpValue.push_back(i); + key1Vec.push_back(pair {tmpKey, tmpValue}); + } + for (int i = 0; i < dataSize; i++) { + Key tmpKey(key2); + Value tmpValue(value2); + tmpKey.push_back(i); + tmpValue.push_back(i); + key2Vec.push_back(pair {tmpKey, tmpValue}); + } + for (int i = 0; i < dataSize; i++) { + g_deviceB->PutData(key2Vec[i].first, key2Vec[i].second, 20 + i, 0); + g_deviceB->PutData(key1Vec[i].first, key1Vec[i].second, 10 + i, 0); + } + + /** + * @tc.steps: step2. deviceA call query sync and wait + * @tc.expected: step2. sync should return OK. + */ + Query query = Query::Select().PrefixKey(key2); + std::map result; + status = g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PULL_ONLY, result, query); + ASSERT_TRUE(status == OK); + + /** + * @tc.expected: step3. onComplete should be called, DeviceA have {b0, v0} - {b9, v9} + */ + ASSERT_TRUE(result.size() == devices.size()); + for (const auto &pair : result) { + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + Value tmpValue; + for (int i = 0; i < dataSize; i++) { + g_kvDelegatePtr->Get(key2Vec[i].first, tmpValue); + EXPECT_TRUE(tmpValue == key2Vec[i].second); + g_kvDelegatePtr->Get(key1Vec[i].first, tmpValue); + EXPECT_TRUE(tmpValue != key1Vec[i].second); + } +} + +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, QueryRequestPacketTest001, TestSize.Level1) +{ + /** + * @tc.steps: step1. prepare a QuerySyncRequestPacket. + */ + auto packet = new (std::nothrow) DataRequestPacket; + ASSERT_TRUE(packet != nullptr); + auto kvEntry = new (std::nothrow) GenericSingleVerKvEntry; + ASSERT_TRUE(kvEntry != nullptr); + kvEntry->SetTimestamp(1); + SyncEntry syncData {.entries = {kvEntry}}; +#ifndef OMIT_ZLIB + ASSERT_TRUE(GenericSingleVerKvEntry::Compress(syncData.entries, syncData.compressedEntries, + {CompressAlgorithm::ZLIB, SOFTWARE_VERSION_CURRENT}) == E_OK); + packet->SetCompressAlgo(CompressAlgorithm::ZLIB); + packet->SetFlag(4); // set IS_COMPRESS_DATA flag true +#endif + packet->SetBasicInfo(-E_NOT_SUPPORT, SOFTWARE_VERSION_CURRENT, SyncModeType::QUERY_PUSH_PULL); + packet->SetData(syncData.entries); + packet->SetCompressData(syncData.compressedEntries); + packet->SetEndWaterMark(INT8_MAX); + packet->SetWaterMark(INT16_MAX, INT32_MAX, INT64_MAX); + QuerySyncObject syncQuery(Query::Select().PrefixKey({'2'})); + packet->SetQuery(syncQuery); + packet->SetQueryId(syncQuery.GetIdentify()); + packet->SetReserved(std::vector {INT8_MAX}); + + /** + * @tc.steps: step2. put the QuerySyncRequestPacket into a message. + */ + Message msg; + msg.SetExternalObject(packet); + msg.SetMessageId(QUERY_SYNC_MESSAGE); + msg.SetMessageType(TYPE_REQUEST); + + /** + * @tc.steps: step3. Serialization the message to a buffer. + */ + int len = SingleVerSerializeManager::CalculateLen(&msg); + vector buffer(len); + ASSERT_EQ(SingleVerSerializeManager::Serialization(buffer.data(), buffer.size(), &msg), E_OK); + + /** + * @tc.steps: step4. DeSerialization the buffer to a message. + */ + Message outMsg(QUERY_SYNC_MESSAGE); + outMsg.SetMessageType(TYPE_REQUEST); + ASSERT_EQ(SingleVerSerializeManager::DeSerialization(buffer.data(), buffer.size(), &outMsg), E_OK); + + /** + * @tc.steps: step5. checkout the outMsg. + * @tc.expected: step5. outMsg equal the the in msg + */ + auto outPacket = outMsg.GetObject(); + EXPECT_EQ(outPacket->GetVersion(), SOFTWARE_VERSION_CURRENT); + EXPECT_EQ(outPacket->GetMode(), SyncModeType::QUERY_PUSH_PULL); + EXPECT_EQ(outPacket->GetEndWaterMark(), static_cast(INT8_MAX)); + EXPECT_EQ(outPacket->GetLocalWaterMark(), static_cast(INT16_MAX)); + EXPECT_EQ(outPacket->GetPeerWaterMark(), static_cast(INT32_MAX)); + EXPECT_EQ(outPacket->GetDeletedWaterMark(), static_cast(INT64_MAX)); +#ifndef OMIT_ZLIB + EXPECT_EQ(outPacket->GetFlag(), static_cast(4)); // check IS_COMPRESS_DATA flag true +#endif + EXPECT_EQ(outPacket->GetQueryId(), syncQuery.GetIdentify()); + EXPECT_EQ(outPacket->GetReserved(), std::vector {INT8_MAX}); + EXPECT_EQ(outPacket->GetSendCode(), -E_NOT_SUPPORT); + EXPECT_EQ(outPacket->GetData()[0]->GetTimestamp(), 1u); +} + +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, QueryAckPacketTest001, TestSize.Level1) +{ + /** + * @tc.steps: step1. prepare a QuerySyncAckPacket. + */ + DataAckPacket packet; + packet.SetVersion(SOFTWARE_VERSION_CURRENT); + packet.SetData(INT64_MAX); + packet.SetRecvCode(-E_NOT_SUPPORT); + std::vector reserved = {INT8_MAX}; + packet.SetReserved(reserved); + + /** + * @tc.steps: step2. put the QuerySyncAckPacket into a message. + */ + Message msg; + msg.SetCopiedObject(packet); + msg.SetMessageId(QUERY_SYNC_MESSAGE); + msg.SetMessageType(TYPE_RESPONSE); + + /** + * @tc.steps: step3. Serialization the message to a buffer. + */ + int len = SingleVerSerializeManager::CalculateLen(&msg); + LOGE("test leng = %d", len); + uint8_t *buffer = new (nothrow) uint8_t[len]; + ASSERT_TRUE(buffer != nullptr); + int errCode = SingleVerSerializeManager::Serialization(buffer, len, &msg); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step4. DeSerialization the buffer to a message. + */ + Message outMsg; + outMsg.SetMessageId(QUERY_SYNC_MESSAGE); + outMsg.SetMessageType(TYPE_RESPONSE); + errCode = SingleVerSerializeManager::DeSerialization(buffer, len, &outMsg); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step5. checkout the outMsg. + * @tc.expected: step5. outMsg equal the the in msg + */ + auto outPacket = outMsg.GetObject(); + EXPECT_EQ(outPacket->GetVersion(), SOFTWARE_VERSION_CURRENT); + EXPECT_EQ(outPacket->GetData(), static_cast(INT64_MAX)); + std::vector reserved2 = {INT8_MAX}; + EXPECT_EQ(outPacket->GetReserved(), reserved2); + EXPECT_EQ(outPacket->GetRecvCode(), -E_NOT_SUPPORT); + delete[] buffer; +} + +/** + * @tc.name: GetQueryWaterMark 001 + * @tc.desc: Test metaData save and get queryWaterMark. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, GetQueryWaterMark001, TestSize.Level1) +{ + VirtualSingleVerSyncDBInterface storage; + Metadata meta; + + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + int errCode = meta.Initialize(&storage); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step2. save receive and send watermark + * @tc.expected: step2. E_OK + */ + WaterMark w1 = 1; + EXPECT_EQ(meta.SetRecvQueryWaterMark("Q1", "D1", w1), E_OK); + EXPECT_EQ(meta.SetSendQueryWaterMark("Q1", "D1", w1), E_OK); + + /** + * @tc.steps: step3. get receive and send watermark + * @tc.expected: step3. E_OK and get the latest value + */ + WaterMark w = 0; + EXPECT_EQ(meta.GetRecvQueryWaterMark("Q1", "D1", w), E_OK); + EXPECT_EQ(w1, w); + EXPECT_EQ(meta.GetSendQueryWaterMark("Q1", "D1", w), E_OK); + EXPECT_EQ(w1, w); + + /** + * @tc.steps: step4. set peer and local watermark + * @tc.expected: step4. E_OK + */ + WaterMark w2 = 2; + EXPECT_EQ(meta.SaveLocalWaterMark("D1", w2), E_OK); + EXPECT_EQ(meta.SavePeerWaterMark("D1", w2, true), E_OK); + + /** + * @tc.steps: step5. get receive and send watermark + * @tc.expected: step5. E_OK and get the w1 + */ + EXPECT_EQ(meta.GetRecvQueryWaterMark("Q1", "D1", w), E_OK); + EXPECT_EQ(w2, w); + EXPECT_EQ(meta.GetSendQueryWaterMark("Q1", "D1", w), E_OK); + EXPECT_EQ(w2, w); + + /** + * @tc.steps: step6. set peer and local watermark + * @tc.expected: step6. E_OK + */ + WaterMark w3 = 3; + EXPECT_EQ(meta.SaveLocalWaterMark("D2", w3), E_OK); + EXPECT_EQ(meta.SavePeerWaterMark("D2", w3, true), E_OK); + + /** + * @tc.steps: step7. get receive and send watermark + * @tc.expected: step7. E_OK and get the w3 + */ + EXPECT_EQ(meta.GetRecvQueryWaterMark("Q2", "D2", w), E_OK); + EXPECT_EQ(w3, w); + EXPECT_EQ(meta.GetSendQueryWaterMark("Q2", "D2", w), E_OK); + EXPECT_EQ(w3, w); + + /** + * @tc.steps: step8. get not exit receive and send watermark + * @tc.expected: step8. E_OK and get the 0 + */ + EXPECT_EQ(meta.GetRecvQueryWaterMark("Q3", "D3", w), E_OK); + EXPECT_EQ(w, 0u); + EXPECT_EQ(meta.GetSendQueryWaterMark("Q3", "D3", w), E_OK); + EXPECT_EQ(w, 0u); +} + +/** + * @tc.name: GetQueryWaterMark 002 + * @tc.desc: Test metaData save and get queryWaterMark after push or pull mode. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, GetQueryWaterMark002, TestSize.Level1) +{ + VirtualSingleVerSyncDBInterface storage; + Metadata meta; + + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + int errCode = meta.Initialize(&storage); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step2. set peer and local watermark + * @tc.expected: step2. E_OK + */ + WaterMark w1 = 2; + EXPECT_EQ(meta.SaveLocalWaterMark("D1", w1), E_OK); + EXPECT_EQ(meta.SavePeerWaterMark("D1", w1, true), E_OK); + + /** + * @tc.steps: step2. save receive and send watermark + * @tc.expected: step2. E_OK + */ + WaterMark w2 = 1; + EXPECT_EQ(meta.SetRecvQueryWaterMark("Q1", "D1", w2), E_OK); + EXPECT_EQ(meta.SetSendQueryWaterMark("Q1", "D1", w2), E_OK); + + /** + * @tc.steps: step3. get receive and send watermark + * @tc.expected: step3. E_OK and get the bigger value + */ + WaterMark w = 0; + EXPECT_EQ(meta.GetRecvQueryWaterMark("Q1", "D1", w), E_OK); + EXPECT_EQ(w1, w); + EXPECT_EQ(meta.GetSendQueryWaterMark("Q1", "D1", w), E_OK); + EXPECT_EQ(w1, w); +} + +/** + * @tc.name: ClearQueryWaterMark 001 + * @tc.desc: Test metaData clear watermark function. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, ClearQueryWaterMark001, TestSize.Level1) +{ + VirtualSingleVerSyncDBInterface storage; + Metadata meta; + + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + int errCode = meta.Initialize(&storage); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step2. save receive watermark + * @tc.expected: step2. E_OK + */ + WaterMark w1 = 1; + EXPECT_EQ(meta.SetRecvQueryWaterMark("Q1", "D1", w1), E_OK); + + /** + * @tc.steps: step3. erase peer watermark + * @tc.expected: step3. E_OK + */ + EXPECT_EQ(meta.EraseDeviceWaterMark("D1", true), E_OK); + + /** + * @tc.steps: step4. get receive watermark + * @tc.expected: step4. E_OK receive watermark is zero + */ + WaterMark w2 = -1; + EXPECT_EQ(meta.GetRecvQueryWaterMark("Q1", "D1", w2), E_OK); + EXPECT_EQ(w2, 0u); + + /** + * @tc.steps: step5. set peer watermark + * @tc.expected: step5. E_OK + */ + WaterMark w3 = 2; + EXPECT_EQ(meta.SavePeerWaterMark("D1", w3, true), E_OK); + + /** + * @tc.steps: step6. get receive watermark + * @tc.expected: step6. E_OK receive watermark is peer watermark + */ + WaterMark w4 = -1; + EXPECT_EQ(meta.GetRecvQueryWaterMark("Q1", "D1", w4), E_OK); + EXPECT_EQ(w4, w3); +} + +/** + * @tc.name: ClearQueryWaterMark 002 + * @tc.desc: Test metaData clear watermark function. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, ClearQueryWaterMark002, TestSize.Level1) +{ + VirtualSingleVerSyncDBInterface storage; + Metadata meta; + + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + int errCode = meta.Initialize(&storage); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step2. save receive watermark + * @tc.expected: step2. E_OK + */ + WaterMark w1 = 1; + EXPECT_EQ(meta.SetRecvQueryWaterMark("Q1", "D1", w1), E_OK); + EXPECT_EQ(meta.SetRecvQueryWaterMark("Q2", "D1", w1), E_OK); + EXPECT_EQ(meta.SetRecvQueryWaterMark("Q1", "D2", w1), E_OK); + + /** + * @tc.steps: step3. erase peer watermark, make sure data remove in db + * @tc.expected: step3. E_OK + */ + Metadata anotherMeta; + ASSERT_EQ(anotherMeta.Initialize(&storage), E_OK); + EXPECT_EQ(anotherMeta.EraseDeviceWaterMark("D1", true), E_OK); + + /** + * @tc.steps: step4. get receive watermark + * @tc.expected: step4. E_OK receive watermark is zero + */ + WaterMark w2 = -1; + EXPECT_EQ(anotherMeta.GetRecvQueryWaterMark("Q1", "D1", w2), E_OK); + EXPECT_EQ(w2, 0u); + w2 = -1; + EXPECT_EQ(anotherMeta.GetRecvQueryWaterMark("Q2", "D1", w2), E_OK); + EXPECT_EQ(w2, 0u); + w2 = -1; + EXPECT_EQ(anotherMeta.GetRecvQueryWaterMark("Q1", "D2", w2), E_OK); + EXPECT_EQ(w2, w1); +} + +/** + * @tc.name: GetDeleteKeyWaterMark 001 + * @tc.desc: Test metaData save and get deleteWaterMark. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, GetDeleteKeyWaterMark001, TestSize.Level1) +{ + VirtualSingleVerSyncDBInterface storage; + Metadata meta; + + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + int errCode = meta.Initialize(&storage); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step2. save receive and send watermark + * @tc.expected: step2. E_OK + */ + WaterMark w1 = 1; + EXPECT_EQ(meta.SetRecvDeleteSyncWaterMark("D1", w1), E_OK); + EXPECT_EQ(meta.SetSendDeleteSyncWaterMark("D1", w1), E_OK); + + /** + * @tc.steps: step3. get receive and send watermark + * @tc.expected: step3. E_OK and get the latest value + */ + WaterMark w = 0; + EXPECT_EQ(meta.GetRecvDeleteSyncWaterMark("D1", w), E_OK); + EXPECT_EQ(w1, w); + EXPECT_EQ(meta.GetSendDeleteSyncWaterMark("D1", w), E_OK); + EXPECT_EQ(w1, w); + + /** + * @tc.steps: step4. set peer and local watermark + * @tc.expected: step4. E_OK + */ + WaterMark w2 = 2; + EXPECT_EQ(meta.SaveLocalWaterMark("D1", w2), E_OK); + EXPECT_EQ(meta.SavePeerWaterMark("D1", w2, true), E_OK); + + /** + * @tc.steps: step5. get receive and send watermark + * @tc.expected: step5. E_OK and get the w1 + */ + EXPECT_EQ(meta.GetRecvDeleteSyncWaterMark("D1", w), E_OK); + EXPECT_EQ(w2, w); + EXPECT_EQ(meta.GetSendDeleteSyncWaterMark("D1", w), E_OK); + EXPECT_EQ(w2, w); + + /** + * @tc.steps: step6. set peer and local watermark + * @tc.expected: step6. E_OK + */ + WaterMark w3 = 3; + EXPECT_EQ(meta.SaveLocalWaterMark("D2", w3), E_OK); + EXPECT_EQ(meta.SavePeerWaterMark("D2", w3, true), E_OK); + + /** + * @tc.steps: step7. get receive and send watermark + * @tc.expected: step7. E_OK and get the w3 + */ + EXPECT_EQ(meta.GetRecvDeleteSyncWaterMark("D2", w), E_OK); + EXPECT_EQ(w3, w); + EXPECT_EQ(meta.GetSendDeleteSyncWaterMark("D2", w), E_OK); + EXPECT_EQ(w3, w); + + /** + * @tc.steps: step8. get not exit receive and send watermark + * @tc.expected: step8. E_OK and get the 0 + */ + EXPECT_EQ(meta.GetRecvDeleteSyncWaterMark("D3", w), E_OK); + EXPECT_EQ(w, 0u); + EXPECT_EQ(meta.GetSendDeleteSyncWaterMark("D3", w), E_OK); + EXPECT_EQ(w, 0u); +} + +/** + * @tc.name: GetDeleteKeyWaterMark 002 + * @tc.desc: Test metaData save and get deleteWaterMark after push or pull mode. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, GetDeleteKeyWaterMark002, TestSize.Level1) +{ + VirtualSingleVerSyncDBInterface storage; + Metadata meta; + + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + int errCode = meta.Initialize(&storage); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step2. set peer and local watermark + * @tc.expected: step2. E_OK + */ + WaterMark w1 = 3; + EXPECT_EQ(meta.SaveLocalWaterMark("D1", w1), E_OK); + EXPECT_EQ(meta.SavePeerWaterMark("D1", w1, true), E_OK); + + /** + * @tc.steps: step2. save receive and send watermark + * @tc.expected: step2. E_OK + */ + WaterMark w2 = 1; + EXPECT_EQ(meta.SetRecvDeleteSyncWaterMark("D1", w2), E_OK); + EXPECT_EQ(meta.SetSendDeleteSyncWaterMark("D1", w2), E_OK); + + /** + * @tc.steps: step3. get receive and send watermark + * @tc.expected: step3. E_OK and get the bigger value + */ + WaterMark w = 0; + EXPECT_EQ(meta.GetRecvDeleteSyncWaterMark("D1", w), E_OK); + EXPECT_EQ(w1, w); + EXPECT_EQ(meta.GetSendDeleteSyncWaterMark("D1", w), E_OK); + EXPECT_EQ(w1, w); +} + +/** + * @tc.name: ClearDeleteKeyWaterMark 001 + * @tc.desc: Test metaData clear watermark function. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, ClearDeleteKeyWaterMark001, TestSize.Level1) +{ + VirtualSingleVerSyncDBInterface storage; + Metadata meta; + + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + int errCode = meta.Initialize(&storage); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step2. save receive watermark + * @tc.expected: step2. E_OK + */ + WaterMark w1 = 1; + EXPECT_EQ(meta.SetRecvDeleteSyncWaterMark("D1", w1), E_OK); + + /** + * @tc.steps: step3. erase peer watermark + * @tc.expected: step3. E_OK + */ + EXPECT_EQ(meta.EraseDeviceWaterMark("D1", true), E_OK); + + /** + * @tc.steps: step4. get receive watermark + * @tc.expected: step4. E_OK receive watermark is zero + */ + WaterMark w2 = -1; + EXPECT_EQ(meta.GetRecvDeleteSyncWaterMark("D1", w2), E_OK); + EXPECT_EQ(w2, 0u); + + /** + * @tc.steps: step5. set peer watermark + * @tc.expected: step5. E_OK + */ + WaterMark w3 = 2; + EXPECT_EQ(meta.SavePeerWaterMark("D1", w3, true), E_OK); + + /** + * @tc.steps: step6. get receive watermark + * @tc.expected: step6. E_OK receive watermark is peer watermark + */ + WaterMark w4 = -1; + EXPECT_EQ(meta.GetRecvDeleteSyncWaterMark("D1", w4), E_OK); + EXPECT_EQ(w4, w3); +} + +/** + * @tc.name: VerifyCacheAndDb 001 + * @tc.desc: Test metaData watermark cache and db are consistent and correct. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, VerifyMetaDataQuerySync001, TestSize.Level1) +{ + Metadata meta; + VirtualSingleVerSyncDBInterface storage; + + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + int errCode = meta.Initialize(&storage); + ASSERT_EQ(errCode, E_OK); + + const std::string deviceId = "D1"; + const std::string queryId = "Q1"; + + /** + * @tc.steps: step2. save deleteSync watermark + * @tc.expected: step2. E_OK + */ + WaterMark deleteWaterMark = 1; + EXPECT_EQ(meta.SetRecvDeleteSyncWaterMark(deviceId, deleteWaterMark), E_OK); + EXPECT_EQ(meta.SetSendDeleteSyncWaterMark(deviceId, deleteWaterMark), E_OK); + + /** + * @tc.steps: step3. save querySync watermark + * @tc.expected: step2. E_OK + */ + WaterMark queryWaterMark = 2; + EXPECT_EQ(meta.SetRecvQueryWaterMark(queryId, deviceId, queryWaterMark), E_OK); + EXPECT_EQ(meta.SetSendQueryWaterMark(queryId, deviceId, queryWaterMark), E_OK); + + /** + * @tc.steps: step4. initialize meta with storage + * @tc.expected: step4. E_OK + */ + Metadata anotherMeta; + ASSERT_EQ(anotherMeta.Initialize(&storage), E_OK); + + /** + * @tc.steps: step5. verify delete sync data + * @tc.expected: step5. E_OK and waterMark equal to deleteWaterMark + */ + WaterMark waterMark; + EXPECT_EQ(anotherMeta.GetRecvDeleteSyncWaterMark(deviceId, waterMark), E_OK); + EXPECT_EQ(waterMark, deleteWaterMark); + EXPECT_EQ(anotherMeta.GetSendDeleteSyncWaterMark(deviceId, waterMark), E_OK); + EXPECT_EQ(waterMark, deleteWaterMark); + + /** + * @tc.steps: step6. verify query sync data + * @tc.expected: step6. E_OK and waterMark equal to queryWaterMark + */ + EXPECT_EQ(anotherMeta.GetRecvQueryWaterMark(queryId, deviceId, waterMark), E_OK); + EXPECT_EQ(waterMark, queryWaterMark); + EXPECT_EQ(anotherMeta.GetSendQueryWaterMark(queryId, deviceId, waterMark), E_OK); + EXPECT_EQ(waterMark, queryWaterMark); +} + +/** + * @tc.name: VerifyLruMap 001 + * @tc.desc: Test metaData watermark cache lru ability. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, VerifyLruMap001, TestSize.Level1) +{ + LruMap lruMap; + const int maxCacheItems = 200; + + /** + * @tc.steps: step1. fill items to LruMap + * @tc.expected: step1. E_OK + */ + const int startCount = 0; + for (int i = startCount; i < maxCacheItems; i++) { + std::string key = std::to_string(i); + QueryWaterMark value; + value.recvWaterMark = i + 1; + EXPECT_EQ(lruMap.Put(key, value), E_OK); + } + + /** + * @tc.steps: step2. get the first item + * @tc.expected: step2. E_OK first item will move to last + */ + std::string firstItemKey = std::to_string(startCount); + QueryWaterMark firstItemValue; + EXPECT_EQ(lruMap.Get(firstItemKey, firstItemValue), E_OK); + EXPECT_EQ(firstItemValue.recvWaterMark, 1u); + + /** + * @tc.steps: step3. insert new items to LruMap + * @tc.expected: step3. the second items was removed + */ + std::string key = std::to_string(maxCacheItems); + QueryWaterMark value; + value.recvWaterMark = maxCacheItems; + EXPECT_EQ(lruMap.Put(key, value), E_OK); + + /** + * @tc.steps: step4. get the second item + * @tc.expected: step4. E_NOT_FOUND it was removed by algorithm + */ + std::string secondItemKey = std::to_string(startCount + 1); + QueryWaterMark secondItemValue; + EXPECT_EQ(lruMap.Get(secondItemKey, secondItemValue), -E_NOT_FOUND); + EXPECT_EQ(secondItemValue.recvWaterMark, 0u); +} + +/** + * @tc.name: VerifyMetaDataInit 001 + * @tc.desc: Test metaData init correctly + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, VerifyMetaDataInit001, TestSize.Level1) +{ + Metadata meta; + VirtualSingleVerSyncDBInterface storage; + + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + ASSERT_EQ(meta.Initialize(&storage), E_OK); + + DeviceID deviceA = "DeviceA"; + DeviceID deviceB = "DeviceA"; + WaterMark setWaterMark = 1; + + /** + * @tc.steps: step2. meta save and get waterMark + * @tc.expected: step2. expect get the same waterMark + */ + EXPECT_EQ(meta.SaveLocalWaterMark(deviceA, setWaterMark), E_OK); + EXPECT_EQ(meta.SaveLocalWaterMark(deviceB, setWaterMark), E_OK); + WaterMark getWaterMark = 0; + meta.GetLocalWaterMark(deviceA, getWaterMark); + EXPECT_EQ(getWaterMark, setWaterMark); + meta.GetLocalWaterMark(deviceB, getWaterMark); + EXPECT_EQ(getWaterMark, setWaterMark); + + + /** + * @tc.steps: step3. init again + * @tc.expected: step3. E_OK + */ + Metadata anotherMeta; + ASSERT_EQ(anotherMeta.Initialize(&storage), E_OK); + + /** + * @tc.steps: step4. get waterMark again + * @tc.expected: step4. expect get the same waterMark + */ + anotherMeta.GetLocalWaterMark(deviceA, getWaterMark); + EXPECT_EQ(getWaterMark, setWaterMark); + anotherMeta.GetLocalWaterMark(deviceB, getWaterMark); + EXPECT_EQ(getWaterMark, setWaterMark); +} + +namespace { +void InitVerifyStorageEnvironment(Metadata &meta, VirtualSingleVerSyncDBInterface &storage, + const std::string &deviceId, const int &startCount, const uint32_t &maxStoreItems) +{ + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + ASSERT_EQ(meta.Initialize(&storage), E_OK); + + /** + * @tc.steps: step2. fill items to metadata + * @tc.expected: step2. E_OK + */ + for (uint32_t i = startCount; i < maxStoreItems; i++) { + std::string queryId = std::to_string(i); + WaterMark recvWaterMark = i + 1; + EXPECT_EQ(meta.SetRecvQueryWaterMark(queryId, deviceId, recvWaterMark), E_OK); + } +} +} + +/** + * @tc.name: VerifyManagerQuerySyncStorage 001 + * @tc.desc: Test metaData remove least used querySync storage items. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, VerifyManagerQuerySyncStorage001, TestSize.Level3) +{ + Metadata meta; + VirtualSingleVerSyncDBInterface storage; + const uint32_t maxStoreItems = 100000; + const int startCount = 0; + const std::string deviceId = "Device"; + + InitVerifyStorageEnvironment(meta, storage, deviceId, startCount, maxStoreItems); + + /** + * @tc.steps: step3. insert new items to metadata + * @tc.expected: step3. E_OK + */ + std::string newQueryId = std::to_string(maxStoreItems); + WaterMark newWaterMark = maxStoreItems + 1; + EXPECT_EQ(meta.SetRecvQueryWaterMark(newQueryId, deviceId, newWaterMark), E_OK); + + /** + * @tc.steps: step4. touch the first item + * @tc.expected: step4. E_OK update first item used time + */ + std::string firstItemKey = std::to_string(startCount); + WaterMark firstWaterMark = 11u; + EXPECT_EQ(meta.SetRecvQueryWaterMark(firstItemKey, deviceId, firstWaterMark), E_OK); + + /** + * @tc.steps: step5. initialize new meta with storage + * @tc.expected: step5. the second item will be removed + */ + Metadata newMeta; + ASSERT_EQ(newMeta.Initialize(&storage), E_OK); + + /** + * @tc.steps: step6. touch the first item + * @tc.expected: step6. E_OK it still exist + */ + WaterMark exceptWaterMark; + EXPECT_EQ(newMeta.GetRecvQueryWaterMark(firstItemKey, deviceId, exceptWaterMark), E_OK); + EXPECT_EQ(exceptWaterMark, firstWaterMark); + + /** + * @tc.steps: step7. get the second item + * @tc.expected: step7. NOT_FOUND secondWaterMark is zero + */ + WaterMark secondWaterMark; + std::string secondQueryId = std::to_string(startCount + 1); + EXPECT_EQ(newMeta.GetRecvQueryWaterMark(secondQueryId, deviceId, secondWaterMark), E_OK); + EXPECT_EQ(secondWaterMark, 0u); +} + +/** + * @tc.name: VerifyMetaDbCreateTime 001 + * @tc.desc: Test metaData get and set cbCreateTime. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, VerifyMetaDbCreateTime001, TestSize.Level1) +{ + Metadata meta; + VirtualSingleVerSyncDBInterface storage; + /** + * @tc.steps: step1. initialize meta with storage + * @tc.expected: step1. E_OK + */ + int errCode = meta.Initialize(&storage); + ASSERT_EQ(errCode, E_OK); + /** + * @tc.steps: step2. set local and peer watermark and dbCreateTime + * @tc.expected: step4. E_OK + */ + WaterMark value = 2; + EXPECT_EQ(meta.SaveLocalWaterMark("D1", value), E_OK); + EXPECT_EQ(meta.SavePeerWaterMark("D1", value, true), E_OK); + EXPECT_EQ(meta.SetDbCreateTime("D1", 10u, true), E_OK); + /** + * @tc.steps: step3. check peer and local watermark and dbCreateTime + * @tc.expected: step4. E_OK + */ + WaterMark curValue = 0; + meta.GetLocalWaterMark("D1", curValue); + EXPECT_EQ(value, curValue); + meta.GetPeerWaterMark("D1", curValue); + EXPECT_EQ(value, curValue); + uint64_t curDbCreatTime = 0; + meta.GetDbCreateTime("D1", curDbCreatTime); + EXPECT_EQ(curDbCreatTime, 10u); + /** + * @tc.steps: step3. change dbCreateTime and check + * @tc.expected: step4. E_OK + */ + EXPECT_EQ(meta.SetDbCreateTime("D1", 20u, true), E_OK); + uint64_t clearDeviceDataMark = INT_MAX; + meta.GetRemoveDataMark("D1", clearDeviceDataMark); + EXPECT_EQ(clearDeviceDataMark, 1u); + EXPECT_EQ(meta.ResetMetaDataAfterRemoveData("D1"), E_OK); + meta.GetRemoveDataMark("D1", clearDeviceDataMark); + EXPECT_EQ(clearDeviceDataMark, 0u); + meta.GetDbCreateTime("D1", curDbCreatTime); + EXPECT_EQ(curDbCreatTime, 20u); +} + +/** + * @tc.name: VerifyManagerQuerySyncStorage 002 + * @tc.desc: Test metaData remove least used querySync storage items when exit wrong data. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, VerifyManagerQuerySyncStorage002, TestSize.Level3) +{ + Metadata meta; + VirtualSingleVerSyncDBInterface storage; + const uint32_t maxStoreItems = 100000; + const int startCount = 0; + const std::string deviceId = "Device"; + + InitVerifyStorageEnvironment(meta, storage, deviceId, startCount, maxStoreItems); + + /** + * @tc.steps: step3. insert a wrong Value + * @tc.expected: step3. E_OK + */ + std::string newQueryId = std::to_string(maxStoreItems); + Key dbKey; + DBCommon::StringToVector(QuerySyncWaterMarkHelper::GetQuerySyncPrefixKey() + + DBCommon::TransferHashString(deviceId) + newQueryId, dbKey); + Value wrongValue; + EXPECT_EQ(storage.PutMetaData(dbKey, wrongValue), E_OK); + + /** + * @tc.steps: step4. initialize new meta with storage + * @tc.expected: step4. E_OK + */ + Metadata newMeta; + ASSERT_EQ(newMeta.Initialize(&storage), E_OK); + + /** + * @tc.steps: step5. touch the first item + * @tc.expected: step5. E_OK still exit + */ + std::string firstItemKey = std::to_string(startCount); + WaterMark exceptWaterMark; + EXPECT_EQ(newMeta.GetRecvQueryWaterMark(firstItemKey, deviceId, exceptWaterMark), E_OK); + EXPECT_EQ(exceptWaterMark, 1u); +} + +/** + * @tc.name: AllPredicateQuerySync001 + * @tc.desc: Test normal push sync for AllPredicate data. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, AllPredicateQuerySync001, TestSize.Level1) +{ + /** + * @tc.steps: step1. InitSchemaDb + */ + InitSchemaDb(); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step2. deviceA put {key11, SCHEMA_VALUE1} - {key19, SCHEMA_VALUE1} + {key21, SCHEMA_VALUE2} - {key29, SCHEMA_VALUE2} + */ + Value value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end()); + Value value2(SCHEMA_VALUE2.begin(), SCHEMA_VALUE2.end()); + Key key = {'1'}; + Key key2 = {'2'}; + const int dataSize = 4000; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + key2.push_back(i); + status = g_schemaKvDelegatePtr->Put(key, value); + status = g_schemaKvDelegatePtr->Put(key2, value2); + ASSERT_TRUE(status == OK); + key.pop_back(); + key2.pop_back(); + } + ASSERT_TRUE(status == OK); + + /** + * @tc.steps: step3. deviceA call query sync and wait + * @tc.expected: step3. sync should return OK. + */ + Query query = Query::Select().EqualTo("$.field_name1", 1); + std::map result; + status = g_tool.SyncTest(g_schemaKvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result, query); + ASSERT_TRUE(status == OK); + + /** + * @tc.expected: step4. onComplete should be called, DeviceB have {key11, SCHEMA_VALUE1} - {key19, SCHEMA_VALUE1} + */ + ASSERT_TRUE(result.size() == devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + VirtualDataItem item2; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + key2.push_back(i); + g_deviceB->GetData(key, item); + EXPECT_TRUE(g_deviceB->GetData(key2, item2) != E_OK); + EXPECT_TRUE(item.value == value); + key.pop_back(); + key2.pop_back(); + } +} + +/** + * @tc.name: AllPredicateQuerySync002 + * @tc.desc: Test wrong query param push sync for AllPredicate data. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, AllPredicateQuerySync002, TestSize.Level1) +{ + /** + * @tc.steps: step1. InitSchemaDb + */ + InitSchemaDb(); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step2. deviceA call query sync and wait + * @tc.expected: step2. sync should return INVALID_QUERY_FIELD + */ + Query query = Query::Select().GreaterThan("field_name11", 10); + std::map result; + status = g_tool.SyncTest(g_schemaKvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result, query); + ASSERT_TRUE(status == INVALID_QUERY_FIELD); + status = g_tool.SyncTest(g_schemaKvDelegatePtr, devices, SYNC_MODE_PUSH_PULL, result, query); + ASSERT_TRUE(status == INVALID_QUERY_FIELD); + status = g_tool.SyncTest(g_schemaKvDelegatePtr, devices, SYNC_MODE_PULL_ONLY, result, query); + ASSERT_TRUE(status == INVALID_QUERY_FIELD); +} + +/** + * @tc.name: AllPredicateQuerySync003 + * @tc.desc: Test normal push sync for AllPredicate data with limit + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, AllPredicateQuerySync003, TestSize.Level1) +{ + /** + * @tc.steps: step1. InitSchemaDb + */ + InitSchemaDb(); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step2. deviceA put {key1, SCHEMA_VALUE1} - {key9, SCHEMA_VALUE1} + */ + Value value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end()); + Value value2(SCHEMA_VALUE2.begin(), SCHEMA_VALUE2.end()); + Key key = {'1'}; + Key key2 = {'2'}; + const int dataSize = 10; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + key2.push_back(i); + status = g_schemaKvDelegatePtr->Put(key, value); + status = g_schemaKvDelegatePtr->Put(key2, value2); + ASSERT_TRUE(status == OK); + key.pop_back(); + key2.pop_back(); + } + ASSERT_TRUE(status == OK); + + /** + * @tc.steps: step3. deviceA call query sync with limit and wait + * @tc.expected: step3. sync should return OK. + */ + Query query = Query::Select().EqualTo("$.field_name1", 1).Limit(20, 0); + std::map result; + status = g_tool.SyncTest(g_schemaKvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result, query); + ASSERT_TRUE(status == OK); + + /** + * @tc.expected: step4. onComplete should be called, DeviceB have {key1, SCHEMA_VALUE1} - {key9, SCHEMA_VALUE1} + */ + ASSERT_TRUE(result.size() == devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + VirtualDataItem item2; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + key2.push_back(i); + g_deviceB->GetData(key, item); + EXPECT_TRUE(g_deviceB->GetData(key2, item2) != E_OK); + EXPECT_TRUE(item.value == value); + key.pop_back(); + key2.pop_back(); + } +} + +/** + * @tc.name: AllPredicateQuerySync004 + * @tc.desc: Test normal pull sync for AllPredicate data. + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PQuerySyncTest, AllPredicateQuerySync004, TestSize.Level1) +{ + /** + * @tc.steps: step1. InitSchemaDb + */ + InitSchemaDb(); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step2. deviceB put {key11, SCHEMA_VALUE1} - {key19, SCHEMA_VALUE1} + */ + Value value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end()); + Key key = {'1'}; + const int dataSize = 10; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + g_deviceB->PutData(key, value, 10 + i, 0); + ASSERT_TRUE(status == OK); + key.pop_back(); + } + ASSERT_TRUE(status == OK); + + /** + * @tc.steps: step3. deviceA call query sync and wait + * @tc.expected: step3. sync should return OK. + */ + Query query = Query::Select().EqualTo("$.field_name1", 1); + std::map result; + status = g_tool.SyncTest(g_schemaKvDelegatePtr, devices, SYNC_MODE_PULL_ONLY, result, query); + ASSERT_TRUE(status == OK); + + /** + * @tc.expected: step4. onComplete should be called, DeviceA have {key11, SCHEMA_VALUE1} - {key19, SCHEMA_VALUE1} + */ + ASSERT_TRUE(result.size() == devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + Value item; + Value item2; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + g_schemaKvDelegatePtr->Get(key, item); + EXPECT_TRUE(item == value); + key.pop_back(); + } +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cd127f1ebe43c98f1a065286d4294cf25729dff6 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_subsribe_sync_test.cpp @@ -0,0 +1,728 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "db_constant.h" +#include "distributeddb_data_generate_unit_test.h" +#include "distributeddb_tools_unit_test.h" +#include "kv_store_nb_delegate.h" +#include "kv_virtual_device.h" +#include "platform_specific.h" +#include "query.h" +#include "query_sync_object.h" +#include "single_ver_data_sync.h" +#include "single_ver_serialize_manager.h" +#include "subscribe_manager.h" +#include "sync_types.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; +using namespace std; + +namespace { + string g_testDir; + const string SCHEMA_STORE_ID = "kv_store_sync_schema_test"; + const std::string DEVICE_A = "deviceA"; + const std::string DEVICE_B = "deviceB"; + + KvStoreDelegateManager g_schemaMgr(SCHEMA_APP_ID, USER_ID); + KvStoreConfig g_config; + DistributedDBToolsUnitTest g_tool; + DBStatus g_schemaKvDelegateStatus = INVALID_ARGS; + KvStoreNbDelegate* g_schemaKvDelegatePtr = nullptr; + VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; + KvVirtualDevice* g_deviceB = nullptr; + + // the type of g_kvDelegateCallback is function + auto g_schemaKvDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, + placeholders::_1, placeholders::_2, std::ref(g_schemaKvDelegateStatus), std::ref(g_schemaKvDelegatePtr)); + const string SCHEMA_STRING = + "{\"SCHEMA_VERSION\":\"1.0\"," + "\"SCHEMA_MODE\":\"STRICT\"," + "\"SCHEMA_DEFINE\":{" + "\"field_name1\":\"BOOL\"," + "\"field_name2\":\"BOOL\"," + "\"field_name3\":\"INTEGER, NOT NULL\"," + "\"field_name4\":\"LONG, DEFAULT 100\"," + "\"field_name5\":\"DOUBLE, NOT NULL, DEFAULT 3.14\"," + "\"field_name6\":\"STRING, NOT NULL, DEFAULT '3.1415'\"," + "\"field_name7\":\"LONG, DEFAULT 100\"," + "\"field_name8\":\"LONG, DEFAULT 100\"," + "\"field_name9\":\"LONG, DEFAULT 100\"," + "\"field_name10\":\"LONG, DEFAULT 100\"" + "}," + "\"SCHEMA_INDEXES\":[\"$.field_name1\", \"$.field_name2\"]}"; + + const std::string SCHEMA_VALUE1 = + "{\"field_name1\":true," + "\"field_name2\":false," + "\"field_name3\":10," + "\"field_name4\":20," + "\"field_name5\":3.14," + "\"field_name6\":\"3.1415\"," + "\"field_name7\":100," + "\"field_name8\":100," + "\"field_name9\":100," + "\"field_name10\":100}"; +} + +class DistributedDBSingleVerP2PSubscribeSyncTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBSingleVerP2PSubscribeSyncTest::SetUpTestCase(void) +{ + /** + * @tc.setup: Init datadir and Virtual Communicator. + */ + DistributedDBToolsUnitTest::TestDirInit(g_testDir); + g_config.dataDir = g_testDir; + g_schemaMgr.SetKvStoreConfig(g_config); + + string dir = g_testDir + "/single_ver"; + DIR* dirTmp = opendir(dir.c_str()); + if (dirTmp == nullptr) { + OS::MakeDBDirectory(dir); + } else { + closedir(dirTmp); + } + + g_communicatorAggregator = new (std::nothrow) VirtualCommunicatorAggregator(); + ASSERT_TRUE(g_communicatorAggregator != nullptr); + RuntimeContext::GetInstance()->SetCommunicatorAggregator(g_communicatorAggregator); +} + +void DistributedDBSingleVerP2PSubscribeSyncTest::TearDownTestCase(void) +{ + /** + * @tc.teardown: Release virtual Communicator and clear data dir. + */ + if (DistributedDBToolsUnitTest::RemoveTestDbFiles(g_testDir) != 0) { + LOGE("rm test db files error!"); + } + RuntimeContext::GetInstance()->SetCommunicatorAggregator(nullptr); +} + +void DistributedDBSingleVerP2PSubscribeSyncTest::SetUp(void) +{ + DistributedDBToolsUnitTest::PrintTestCaseInfo(); + /** + * @tc.setup: create virtual device B and get a KvStoreNbDelegate as deviceA + */ + g_deviceB = new (std::nothrow) KvVirtualDevice(DEVICE_B); + ASSERT_TRUE(g_deviceB != nullptr); + VirtualSingleVerSyncDBInterface *syncInterfaceB = new (std::nothrow) VirtualSingleVerSyncDBInterface(); + ASSERT_TRUE(syncInterfaceB != nullptr); + ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, syncInterfaceB), E_OK); +} + +void DistributedDBSingleVerP2PSubscribeSyncTest::TearDown(void) +{ + /** + * @tc.teardown: Release device A, B + */ + if (g_schemaKvDelegatePtr != nullptr) { + ASSERT_EQ(g_schemaMgr.CloseKvStore(g_schemaKvDelegatePtr), OK); + g_schemaKvDelegatePtr = nullptr; + DBStatus status = g_schemaMgr.DeleteKvStore(SCHEMA_STORE_ID); + LOGD("delete kv store status %d", status); + ASSERT_TRUE(status == OK); + } + if (g_deviceB != nullptr) { + delete g_deviceB; + g_deviceB = nullptr; + } + PermissionCheckCallbackV2 nullCallback; + EXPECT_EQ(g_schemaMgr.SetPermissionCheckCallback(nullCallback), OK); +} + +void InitSubSchemaDb() +{ + g_config.dataDir = g_testDir; + g_schemaMgr.SetKvStoreConfig(g_config); + KvStoreNbDelegate::Option option; + option.schema = SCHEMA_STRING; + g_schemaMgr.GetKvStore(SCHEMA_STORE_ID, option, g_schemaKvDelegateCallback); + ASSERT_TRUE(g_schemaKvDelegateStatus == OK); + ASSERT_TRUE(g_schemaKvDelegatePtr != nullptr); +} + + +void CheckUnFinishedMap(uint32_t sizeA, uint32_t sizeB, std::vector &deviceAQueies, + std::vector &deviceBQueies, SubscribeManager &subManager) +{ + std::map> allSyncQueries; + subManager.GetAllUnFinishSubQueries(allSyncQueries); + ASSERT_TRUE(allSyncQueries[DEVICE_A].size() == sizeA); + ASSERT_TRUE(allSyncQueries[DEVICE_B].size() == sizeB); + for (auto &item : allSyncQueries[DEVICE_A]) { + std::string queryId = item.GetIdentify(); + ASSERT_TRUE(std::find(deviceAQueies.begin(), deviceAQueies.end(), queryId) != deviceAQueies.end()); + } + for (auto &item : allSyncQueries[DEVICE_B]) { + std::string queryId = item.GetIdentify(); + ASSERT_TRUE(std::find(deviceBQueies.begin(), deviceBQueies.end(), queryId) != deviceBQueies.end()); + } +} + +void InitLocalSubscribeMap(QuerySyncObject &queryCommonObj, std::map &queryMap, + std::vector &deviceAQueies, std::vector &deviceBQueies, SubscribeManager &subManager) +{ + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(DEVICE_A, queryCommonObj) == E_OK); + ASSERT_TRUE(subManager.ActiveLocalSubscribeQuery(DEVICE_A, queryCommonObj) == E_OK); + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(DEVICE_B, queryCommonObj) == E_OK); + ASSERT_TRUE(subManager.ActiveLocalSubscribeQuery(DEVICE_B, queryCommonObj) == E_OK); + queryMap[queryCommonObj.GetIdentify()] = queryCommonObj; + deviceAQueies.push_back(queryCommonObj.GetIdentify()); + deviceBQueies.push_back(queryCommonObj.GetIdentify()); + for (int i = 0; i < 3; i++) { // 3 subscribe + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + i)})); + deviceAQueies.push_back(querySyncObj.GetIdentify()); + queryMap[querySyncObj.GetIdentify()] = querySyncObj; + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(DEVICE_A, querySyncObj) == E_OK); + ASSERT_TRUE(subManager.ActiveLocalSubscribeQuery(DEVICE_A, querySyncObj) == E_OK); + } + for (int i = 0; i < 1; i++) { + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('b' + i)})); + deviceBQueies.push_back(querySyncObj.GetIdentify()); + queryMap[querySyncObj.GetIdentify()] = querySyncObj; + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(DEVICE_B, querySyncObj) == E_OK); + ASSERT_TRUE(subManager.ActiveLocalSubscribeQuery(DEVICE_B, querySyncObj) == E_OK); + } +} +/** + * @tc.name: SubscribeRequestTest001 + * @tc.desc: test Serialize/DoSerialize SubscribeRequest + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, SubscribeRequestTest001, TestSize.Level1) +{ + /** + * @tc.steps: step1. prepare a SubscribeRequest. + */ + auto packet = new (std::nothrow) SubscribeRequest; + ASSERT_TRUE(packet != nullptr); + packet->SetPacketHead(100, SOFTWARE_VERSION_CURRENT, SUBSCRIBE_QUERY_CMD, 1); + Query query = Query::Select().EqualTo("$.field_name1", 1); + QuerySyncObject syncQuery(query); + packet->SetQuery(syncQuery); + + /** + * @tc.steps: step2. put the SubscribeRequest Packet into a message. + */ + Message msg; + msg.SetExternalObject(packet); + msg.SetMessageId(CONTROL_SYNC_MESSAGE); + msg.SetMessageType(TYPE_REQUEST); + + /** + * @tc.steps: step3. Serialization the message to a buffer. + */ + int len = SingleVerSerializeManager::CalculateLen(&msg); + LOGE("test leng = %d", len); + uint8_t *buffer = new (nothrow) uint8_t[len]; + ASSERT_TRUE(buffer != nullptr); + ASSERT_EQ(SingleVerSerializeManager::Serialization(buffer, len, &msg), E_OK); + + /** + * @tc.steps: step4. DeSerialization the buffer to a message. + */ + Message outMsg; + outMsg.SetMessageId(CONTROL_SYNC_MESSAGE); + outMsg.SetMessageType(TYPE_REQUEST); + ASSERT_EQ(SingleVerSerializeManager::DeSerialization(buffer, len, &outMsg), E_OK); + + /** + * @tc.steps: step5. checkout the outMsg. + * @tc.expected: step5. outMsg equal the the in msg + */ + auto outPacket = outMsg.GetObject(); + EXPECT_EQ(outPacket->GetVersion(), SOFTWARE_VERSION_CURRENT); + EXPECT_EQ(outPacket->GetSendCode(), 100); + EXPECT_EQ(outPacket->GetcontrolCmdType(), SUBSCRIBE_QUERY_CMD); + EXPECT_EQ(outPacket->GetFlag(), 1u); + EXPECT_EQ(outPacket->GetQuery().GetIdentify(), syncQuery.GetIdentify()); + delete[] buffer; +} + +/** + * @tc.name: ControlAckTest001 + * @tc.desc: test Serialize/DoSerialize ControlAckPacket + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, ControlAckTest001, TestSize.Level1) +{ + /** + * @tc.steps: step1. prepare a ControlAckPacket. + */ + ControlAckPacket packet; + packet.SetPacketHead(-E_NOT_SUPPORT, SOFTWARE_VERSION_CURRENT, SUBSCRIBE_QUERY_CMD, 1); + + /** + * @tc.steps: step2. put the QuerySyncAckPacket into a message. + */ + Message msg; + msg.SetCopiedObject(packet); + msg.SetMessageId(CONTROL_SYNC_MESSAGE); + msg.SetMessageType(TYPE_RESPONSE); + + /** + * @tc.steps: step3. Serialization the message to a buffer. + */ + int len = SingleVerSerializeManager::CalculateLen(&msg); + LOGE("test leng = %d", len); + uint8_t *buffer = new (nothrow) uint8_t[len]; + ASSERT_TRUE(buffer != nullptr); + int errCode = SingleVerSerializeManager::Serialization(buffer, len, &msg); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step4. DeSerialization the buffer to a message. + */ + Message outMsg; + outMsg.SetMessageId(CONTROL_SYNC_MESSAGE); + outMsg.SetMessageType(TYPE_RESPONSE); + errCode = SingleVerSerializeManager::DeSerialization(buffer, len, &outMsg); + ASSERT_EQ(errCode, E_OK); + + /** + * @tc.steps: step5. checkout the outMsg. + * @tc.expected: step5. outMsg equal the the in msg + */ + auto outPacket = outMsg.GetObject(); + EXPECT_EQ(outPacket->GetVersion(), SOFTWARE_VERSION_CURRENT); + EXPECT_EQ(outPacket->GetRecvCode(), -E_NOT_SUPPORT); + EXPECT_EQ(outPacket->GetcontrolCmdType(), SUBSCRIBE_QUERY_CMD); + EXPECT_EQ(outPacket->GetFlag(), 1u); + delete[] buffer; +} + +/** + * @tc.name: subscribeManager001 + * @tc.desc: test subscribe class subscribe local function with one device + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeManager001, TestSize.Level1) +{ + SubscribeManager subManager; + std::string device = "device_A"; + /** + * @tc.steps: step1. test one device limit four subscribe queries in local map + */ + LOGI("============step 1============"); + for (int i = 0; i < 4; i++) { + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + i)})); + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(device, querySyncObj) == E_OK); + ASSERT_TRUE(subManager.ActiveLocalSubscribeQuery(device, querySyncObj) == E_OK); + } + std::vector subscribeQueries; + subManager.GetLocalSubscribeQueries(device, subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 4); + subscribeQueries.clear(); + QuerySyncObject querySyncObj1(Query::Select().PrefixKey({'a', static_cast('a' + 4)})); + int errCode = subManager.ReserveLocalSubscribeQuery(device, querySyncObj1); + ASSERT_TRUE(errCode != E_OK); + /** + * @tc.steps: step2. allow to subscribe existed query + */ + LOGI("============step 2============"); + QuerySyncObject querySyncObj2(Query::Select().PrefixKey({'a', static_cast('a' + 3)})); + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(device, querySyncObj2) == E_OK); + ASSERT_TRUE(subManager.ActiveLocalSubscribeQuery(device, querySyncObj2) == E_OK); + subManager.GetLocalSubscribeQueries(device, subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 4); + subscribeQueries.clear(); + /** + * @tc.steps: step3. unsubscribe no existed queries + */ + LOGI("============step 3============"); + subManager.RemoveLocalSubscribeQuery(device, querySyncObj1); + subManager.GetLocalSubscribeQueries(device, subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 4); + subscribeQueries.clear(); + /** + * @tc.steps: step4. unsubscribe queries + */ + LOGI("============step 4============"); + for (int i = 0; i < 4; i++) { + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + i)})); + subManager.RemoveLocalSubscribeQuery(device, querySyncObj); + } + subManager.GetLocalSubscribeQueries(device, subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 0); + + /** + * @tc.steps: step5. reserve twice while subscribe queries + */ + LOGI("============step 5============"); + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(device, querySyncObj2) == E_OK); + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(device, querySyncObj2) == E_OK); + ASSERT_TRUE(subManager.ActiveLocalSubscribeQuery(device, querySyncObj2) == E_OK); + subManager.GetLocalSubscribeQueries(device, subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 1); + subscribeQueries.clear(); + subManager.RemoveLocalSubscribeQuery(device, querySyncObj2); + subManager.GetLocalSubscribeQueries(device, subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 0); +} + +/** + * @tc.name: subscribeManager002 + * @tc.desc: test subscribe class subscribe remote function with one device + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeManager002, TestSize.Level1) +{ + SubscribeManager subManager; + std::string device = "device_A"; + /** + * @tc.steps: step1. test one device limit four subscribe queries in remote map + */ + LOGI("============step 1============"); + for (int i = 0; i < 4; i++) { + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + i)})); + ASSERT_TRUE(subManager.ReserveRemoteSubscribeQuery(device, querySyncObj) == E_OK); + ASSERT_TRUE(subManager.ActiveRemoteSubscribeQuery(device, querySyncObj) == E_OK); + } + QuerySyncObject querySyncObj1(Query::Select().PrefixKey({'a', static_cast('a' + 4)})); + ASSERT_TRUE(subManager.ReserveRemoteSubscribeQuery(device, querySyncObj1) != E_OK); + std::vector subscribeQueryId; + subManager.GetRemoteSubscribeQueryIds(device, subscribeQueryId); +ASSERT_TRUE(subscribeQueryId.size() == 4); + subscribeQueryId.clear(); + /** + * @tc.steps: step2. allow to subscribe existed query + */ + LOGI("============step 2============"); + QuerySyncObject querySyncObj2(Query::Select().PrefixKey({'a', static_cast('a' + 3)})); + ASSERT_TRUE(subManager.ReserveRemoteSubscribeQuery(device, querySyncObj2) == E_OK); + ASSERT_TRUE(subManager.ActiveRemoteSubscribeQuery(device, querySyncObj2) == E_OK); + subManager.GetRemoteSubscribeQueryIds(device, subscribeQueryId); + ASSERT_TRUE(subscribeQueryId.size() == 4); + subscribeQueryId.clear(); + /** + * @tc.steps: step3. unsubscribe no existed queries + */ + LOGI("============step 3============"); + subManager.RemoveRemoteSubscribeQuery(device, querySyncObj1); + subManager.GetRemoteSubscribeQueryIds(device, subscribeQueryId); + ASSERT_TRUE(subscribeQueryId.size() == 4); + subscribeQueryId.clear(); + /** + * @tc.steps: step4. unsubscribe queries + */ + LOGI("============step 4============"); + for (int i = 0; i < 4; i++) { + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + i)})); + subManager.RemoveRemoteSubscribeQuery(device, querySyncObj); + } + subManager.GetRemoteSubscribeQueryIds(device, subscribeQueryId); + ASSERT_TRUE(subscribeQueryId.size() == 0); +} + +/** + * @tc.name: subscribeManager003 + * @tc.desc: test subscribe class subscribe remote function with multi device + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeManager003, TestSize.Level1) +{ + SubscribeManager subManager; + std::string device = "device_"; + std::vector subscribeQueries; + /** + * @tc.steps: step1. test mutil device limit 32 devices in remote map and check each device has one subscribe + */ + LOGI("============step 1============"); + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + 1)})); + for (int i = 0; i < 32; i++) { + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(device + std::to_string(i), querySyncObj) == E_OK); + ASSERT_TRUE(subManager.ActiveLocalSubscribeQuery(device + std::to_string(i), querySyncObj) == E_OK); + } + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(device + std::to_string(33), querySyncObj) != E_OK); + for (int i = 0; i < 32; i++) { + subManager.GetLocalSubscribeQueries(device + std::to_string(i), subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 1); + subscribeQueries.clear(); + } + /** + * @tc.steps: step2. clear remote subscribe query map and check each device has no subscribe + */ + LOGI("============step 2============"); + for (int i = 0; i < 32; i++) { + subManager.ClearLocalSubscribeQuery(device + std::to_string(i)); + subManager.GetLocalSubscribeQueries(device + std::to_string(i), subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 0); + subscribeQueries.clear(); + } + /** + * @tc.steps: step3. test mutil device limit 8 queries in db and check each device has one subscribe + */ + LOGI("============step 3============"); + for (int i = 0; i < 8; i++) { + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + i)})); + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(device + std::to_string(i), querySyncObj) == E_OK); + ASSERT_TRUE(subManager.ActiveLocalSubscribeQuery(device + std::to_string(i), querySyncObj) == E_OK); + } + QuerySyncObject querySyncObj1(Query::Select().PrefixKey({'a', static_cast('a' + 8)})); + ASSERT_TRUE(subManager.ReserveLocalSubscribeQuery(device + std::to_string(8), querySyncObj1) != E_OK); +} + +/** + * @tc.name: subscribeManager004 + * @tc.desc: test subscribe class subscribe remote function with multi device + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeManager004, TestSize.Level1) +{ + SubscribeManager subManager; + std::string device = "device_"; + std::vector subscribeQueryId; + /** + * @tc.steps: step1. test mutil device limit 32 devices in remote map and check each device has one subscribe + */ + LOGI("============step 1============"); + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + 1)})); + for (int i = 0; i < 32; i++) { + ASSERT_TRUE(subManager.ReserveRemoteSubscribeQuery(device + std::to_string(i), querySyncObj) == E_OK); + ASSERT_TRUE(subManager.ActiveRemoteSubscribeQuery(device + std::to_string(i), querySyncObj) == E_OK); + } + ASSERT_TRUE(subManager.ReserveRemoteSubscribeQuery(device + std::to_string(33), querySyncObj) != E_OK); + for (int i = 0; i < 32; i++) { + subManager.GetRemoteSubscribeQueryIds(device + std::to_string(i), subscribeQueryId); + ASSERT_TRUE(subscribeQueryId.size() == 1); + subscribeQueryId.clear(); + } + /** + * @tc.steps: step2. clear remote subscribe query map and check each device has no subscribe + */ + LOGI("============step 2============"); + for (int i = 0; i < 32; i++) { + subManager.ClearRemoteSubscribeQuery(device + std::to_string(i)); + subManager.GetRemoteSubscribeQueryIds(device + std::to_string(i), subscribeQueryId); + ASSERT_TRUE(subscribeQueryId.size() == 0); + subscribeQueryId.clear(); + } + subManager.ClearRemoteSubscribeQuery(device); + /** + * @tc.steps: step3. test mutil device limit 8 queries in db and check each device has one subscribe + */ + LOGI("============step 3============"); + for (int i = 0; i < 8; i++) { + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + i)})); + ASSERT_TRUE(subManager.ReserveRemoteSubscribeQuery(device + std::to_string(i), querySyncObj) == E_OK); + ASSERT_TRUE(subManager.ActiveRemoteSubscribeQuery(device + std::to_string(i), querySyncObj) == E_OK); + } + QuerySyncObject querySyncObj1(Query::Select().PrefixKey({'a', static_cast('a' + 8)})); + ASSERT_TRUE(subManager.ReserveRemoteSubscribeQuery(device + std::to_string(8), querySyncObj1) != E_OK); +} + +/** + * @tc.name: subscribeManager005 + * @tc.desc: test subscribe class subscribe remote function with put into unfinished map + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeManager005, TestSize.Level1) +{ + SubscribeManager subManager; + std::vector subscribeQueries; + std::map queryMap; + std::vector deviceAQueies; + std::vector deviceBQueies; + QuerySyncObject queryCommonObj(Query::Select().PrefixKey({'a'})); + /** + * @tc.steps: step1. test one devices has 4 subscribes and another has 2 in local map, put into unfinished map + */ + LOGI("============step 1============"); + InitLocalSubscribeMap(queryCommonObj, queryMap, deviceAQueies, deviceBQueies, subManager); + /** + * @tc.steps: step2. check all device unFinished subscribe queries and put into unfinished map + */ + LOGI("============step 2============"); + subManager.GetLocalSubscribeQueries(DEVICE_A, subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 4); + subManager.PutLocalUnFiniedSubQueries(DEVICE_A, subscribeQueries); + subscribeQueries.clear(); + subManager.GetLocalSubscribeQueries(DEVICE_B, subscribeQueries); + ASSERT_TRUE(subscribeQueries.size() == 2); + subManager.PutLocalUnFiniedSubQueries(DEVICE_B, subscribeQueries); + subscribeQueries.clear(); + /** + * @tc.steps: step3. get all device unFinished subscribe queries and check + */ + LOGI("============step 3============"); + CheckUnFinishedMap(4, 2, deviceAQueies, deviceBQueies, subManager); + /** + * @tc.steps: step4. active some subscribe queries + */ + LOGI("============step 4============"); + subManager.ActiveLocalSubscribeQuery(DEVICE_A, queryCommonObj); + subManager.ActiveLocalSubscribeQuery(DEVICE_A, queryMap[deviceAQueies[3]]); + subManager.ActiveLocalSubscribeQuery(DEVICE_B, queryMap[deviceBQueies[1]]); + deviceAQueies.erase(deviceAQueies.begin() + 3); + deviceAQueies.erase(deviceAQueies.begin()); + queryMap.erase(queryMap[deviceBQueies[1]].GetIdentify()); + deviceBQueies.erase(deviceBQueies.begin() + 1); + /** + * @tc.steps: step5. get all device unFinished subscribe queries and check + */ + LOGI("============step 5============"); + CheckUnFinishedMap(2, 1, deviceAQueies, deviceBQueies, subManager); + /** + * @tc.steps: step6. remove left subscribe queries + */ + LOGI("============step 6============"); + for (int i = 0; i < 2; i++) { + QuerySyncObject querySyncObj(Query::Select().PrefixKey({'a', static_cast('a' + i)})); + subManager.RemoveLocalSubscribeQuery(DEVICE_A, querySyncObj); + } + subManager.RemoveLocalSubscribeQuery(DEVICE_A, queryCommonObj); + subManager.RemoveLocalSubscribeQuery(DEVICE_B, queryCommonObj); + /** + * @tc.steps: step7. get all device unFinished subscribe queries and check + */ + LOGI("============step 7============"); + CheckUnFinishedMap(0, 0, deviceAQueies, deviceBQueies, subManager); +} + +/** + * @tc.name: subscribeSync001 + * @tc.desc: test subscribe normal sync + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeSync001, TestSize.Level1) +{ + /** + * @tc.steps: step1. InitSchemaDb + */ + LOGI("============step 1============"); + InitSubSchemaDb(); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + Query query = Query::Select().EqualTo("$.field_name1", 1); + QuerySyncObject querySyncObj(query); + + /** + * @tc.steps: step2. deviceB subscribe query to deviceA + */ + LOGI("============step 2============"); + g_deviceB->Subscribe(querySyncObj, true, 1); + + /** + * @tc.steps: step3. deviceA put {key1, SCHEMA_VALUE1} and wait 1s + */ + LOGI("============step 3============"); + Value value(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end()); + Key key = {'1'}; + status = g_schemaKvDelegatePtr->Put(key, value); + EXPECT_EQ(status, OK); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + /** + * @tc.steps: step4. deviceB has {key11, SCHEMA_VALUE1} + */ + LOGI("============step 4============"); + VirtualDataItem item; + g_deviceB->GetData(key, item); + EXPECT_TRUE(item.value == value); + + /** + * @tc.steps: step5. deviceB unsubscribe query to deviceA + */ + g_deviceB->UnSubscribe(querySyncObj, true, 2); + + /** + * @tc.steps: step5. deviceA put {key2, SCHEMA_VALUE1} and wait 1s + */ + LOGI("============step 5============"); + Value value2(SCHEMA_VALUE1.begin(), SCHEMA_VALUE1.end()); + Key key2 = {'2'}; + status = g_schemaKvDelegatePtr->Put(key2, value2); + EXPECT_EQ(status, OK); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + /** + * @tc.steps: step6. deviceB don't has {key2, SCHEMA_VALUE1} + */ + LOGI("============step 6============"); + VirtualDataItem item2; + EXPECT_TRUE(g_deviceB->GetData(key2, item2) != E_OK); +} + +/** + * @tc.name: subscribeSync002 + * @tc.desc: test subscribe sync over 32 devices,limit,orderBy + * @tc.type: FUNC + * @tc.require: AR000FN6G9 + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSubscribeSyncTest, subscribeSync002, TestSize.Level1) +{ + /** + * @tc.steps: step1. InitSchemaDb + */ + LOGI("============step 1============"); + InitSubSchemaDb(); + std::vector devices; + std::string device = "device_"; + Query query = Query::Select().EqualTo("$.field_name1", 1); + + /** + * @tc.steps: step2. deviceA subscribe query to 33 devices, and return overlimit + */ + LOGI("============step 2============"); + for (int i = 0; i < 33; i++) { + devices.push_back(device + std::to_string(i)); + } + EXPECT_TRUE(g_schemaKvDelegatePtr->SubscribeRemoteQuery(devices, nullptr, query, true) == OVER_MAX_LIMITS); + + /** + * @tc.steps: step3. deviceA subscribe query with limit + */ + LOGI("============step 3============"); + devices.clear(); + devices.push_back("device_B"); + Query query2 = Query::Select().EqualTo("$.field_name1", 1).Limit(20, 0); + EXPECT_TRUE(g_schemaKvDelegatePtr->SubscribeRemoteQuery(devices, nullptr, query2, true) == NOT_SUPPORT); + + /** + * @tc.steps: step4. deviceA subscribe query with orderBy + */ + LOGI("============step 4============"); + Query query3 = Query::Select().EqualTo("$.field_name1", 1).OrderBy("$.field_name7"); + EXPECT_TRUE(g_schemaKvDelegatePtr->SubscribeRemoteQuery(devices, nullptr, query3, true) == NOT_SUPPORT); +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_check_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_check_test.cpp index 9b160250e052586f4192df8dc5e46da19b7ac7bf..b869131ec418d2dee6669214b67707a032742d72 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_check_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_check_test.cpp @@ -15,13 +15,13 @@ #include -#include "distributeddb_tools_unit_test.h" #include "distributeddb_data_generate_unit_test.h" -#include "vitural_communicator_aggregator.h" -#include "vitural_device.h" -#include "virtual_single_ver_sync_db_Interface.h" +#include "distributeddb_tools_unit_test.h" +#include "kv_virtual_device.h" #include "platform_specific.h" #include "process_system_api_adapter_impl.h" +#include "single_ver_data_packet.h" +#include "virtual_communicator_aggregator.h" using namespace testing::ext; using namespace DistributedDB; @@ -33,6 +33,13 @@ namespace { const string STORE_ID = "kv_stroe_sync_check_test"; const std::string DEVICE_B = "deviceB"; const std::string DEVICE_C = "deviceC"; + const int LOCAL_WATER_MARK_NOT_INIT = 0xaa; + const int EIGHT_HUNDRED = 800; + const int NORMAL_SYNC_SEND_REQUEST_CNT = 3; + const int TWO_CNT = 2; + const int SLEEP_MILLISECONDS = 500; + const int TEN_SECONDS = 10; + const int THREE_HUNDRED = 300; KvStoreDelegateManager g_mgr(APP_ID, USER_ID); KvStoreConfig g_config; @@ -40,8 +47,8 @@ namespace { DBStatus g_kvDelegateStatus = INVALID_ARGS; KvStoreNbDelegate* g_kvDelegatePtr = nullptr; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; - VituralDevice* g_deviceB = nullptr; - VituralDevice* g_deviceC = nullptr; + KvVirtualDevice* g_deviceB = nullptr; + KvVirtualDevice* g_deviceC = nullptr; VirtualSingleVerSyncDBInterface *g_syncInterfaceB = nullptr; VirtualSingleVerSyncDBInterface *g_syncInterfaceC = nullptr; @@ -51,7 +58,7 @@ namespace { #ifndef LOW_LEVEL_MEM_DEV const int KEY_LEN = 20; // 20 Bytes const int VALUE_LEN = 4 * 1024 * 1024; // 4MB - const int ENTRY_NUM = 6; // 6 entries + const int ENTRY_NUM = 2; // 16 entries #endif } @@ -103,6 +110,7 @@ void DistributedDBSingleVerP2PSyncCheckTest::TearDownTestCase(void) void DistributedDBSingleVerP2PSyncCheckTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create virtual device B and C, and get a KvStoreNbDelegate as deviceA */ @@ -112,13 +120,13 @@ void DistributedDBSingleVerP2PSyncCheckTest::SetUp(void) g_mgr.GetKvStore(STORE_ID, option, g_kvDelegateCallback); ASSERT_TRUE(g_kvDelegateStatus == OK); ASSERT_TRUE(g_kvDelegatePtr != nullptr); - g_deviceB = new (std::nothrow) VituralDevice(DEVICE_B); + g_deviceB = new (std::nothrow) KvVirtualDevice(DEVICE_B); ASSERT_TRUE(g_deviceB != nullptr); g_syncInterfaceB = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(g_syncInterfaceB != nullptr); ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, g_syncInterfaceB), E_OK); - g_deviceC = new (std::nothrow) VituralDevice(DEVICE_C); + g_deviceC = new (std::nothrow) KvVirtualDevice(DEVICE_C); ASSERT_TRUE(g_deviceC != nullptr); g_syncInterfaceC = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(g_syncInterfaceC != nullptr); @@ -145,6 +153,9 @@ void DistributedDBSingleVerP2PSyncCheckTest::TearDown(void) delete g_deviceC; g_deviceC = nullptr; } + if (g_communicatorAggregator != nullptr) { + g_communicatorAggregator->RegOnDispatch(nullptr); + } } /** @@ -264,7 +275,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, BigDataSync001, TestSize.Level1 devices.push_back(g_deviceC->GetDeviceId()); /** - * @tc.steps: step1. deviceA put 6 bigData + * @tc.steps: step1. deviceA put 16 bigData */ std::vector entries; std::vector keys; @@ -375,11 +386,11 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, BigDataSync003, TestSize.Level1 DistributedDBUnitTest::GenerateRecords(ENTRY_NUM, entries, keys, KEY_LEN, VALUE_LEN); for (uint32_t i = 0; i < entries.size(); i++) { - if (i % 3 == 0) { // 0 3 for deivec B + if (i % 3 == 0) { // 0 3 6 9 12 15 for deivec B g_deviceB->PutData(entries[i].key, entries[i].value, 0, 0); - } else if (i % 3 == 1) { // 1 4 for device C + } else if (i % 3 == 1) { // 1 4 7 10 13 16 for device C g_deviceC->PutData(entries[i].key, entries[i].value, 0, 0); - } else { // 2 5 for device A + } else { // 2 5 8 11 14 for device A status = g_kvDelegatePtr->Put(entries[i].key, entries[i].value); ASSERT_TRUE(status == OK); } @@ -447,7 +458,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, PushFinishedNotify001, TestSize /** * @tc.steps: step2. deviceB put k2, v2, and deviceA pull from deviceB - * @tc.expected: step2. deviceA can not revice push finished notify + * @tc.expected: step2. deviceA can not receive push finished notify */ EXPECT_EQ(g_kvDelegatePtr->Put(KEY_2, VALUE_2), OK); std::map result; @@ -458,7 +469,7 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, PushFinishedNotify001, TestSize /** * @tc.steps: step3. deviceB put k3, v3, and deviceA push and pull to deviceB - * @tc.expected: step3. deviceA can not revice push finished notify + * @tc.expected: step3. deviceA can not receive push finished notify */ EXPECT_EQ(g_kvDelegatePtr->Put(KEY_3, VALUE_3), OK); status = g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_PULL, result); @@ -482,4 +493,782 @@ HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, PushFinishedNotify001, TestSize */ status = g_kvDelegatePtr->SetRemotePushFinishedNotify(nullptr); ASSERT_EQ(status, OK); +} + +namespace { +void RegOnDispatchWithDelayAck(bool &errCodeAck, bool &afterErrAck) +{ + // just delay the busy ack + g_communicatorAggregator->RegOnDispatch([&errCodeAck, &afterErrAck](const std::string &dev, Message *inMsg) { + if (dev != g_deviceB->GetDeviceId()) { + return; + } + auto *packet = inMsg->GetObject(); + if (packet->GetRecvCode() == -E_BUSY) { + errCodeAck = true; + while (!afterErrAck) { + } + LOGW("NOW SEND BUSY ACK"); + } else if (errCodeAck) { + afterErrAck = true; + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + }); +} + +void RegOnDispatchWithOffline(bool &offlineFlag, bool &invalid, condition_variable &conditionOffline) +{ + g_communicatorAggregator->RegOnDispatch([&offlineFlag, &invalid, &conditionOffline]( + const std::string &dev, Message *inMsg) { + auto *packet = inMsg->GetObject(); + if (dev != DEVICE_B) { + if (packet->GetRecvCode() == LOCAL_WATER_MARK_NOT_INIT) { + offlineFlag = true; + conditionOffline.notify_all(); + LOGW("[Dispatch] NOTIFY OFFLINE"); + std::this_thread::sleep_for(std::chrono::microseconds (EIGHT_HUNDRED)); + } + } else if (!invalid && inMsg->GetMessageType() == TYPE_REQUEST) { + LOGW("[Dispatch] NOW INVALID THIS MSG"); + inMsg->SetMessageType(TYPE_INVALID); + inMsg->SetMessageId(INVALID_MESSAGE_ID); + invalid = true; + } + }); +} + +void RegOnDispatchWithInvalidMsg(bool &invalid) +{ + g_communicatorAggregator->RegOnDispatch([&invalid]( + const std::string &dev, Message *inMsg) { + if (dev == DEVICE_B && !invalid && inMsg->GetMessageType() == TYPE_REQUEST) { + LOGW("[Dispatch] NOW INVALID THIS MSG"); + inMsg->SetMessageType(TYPE_INVALID); + inMsg->SetMessageId(INVALID_MESSAGE_ID); + invalid = true; + } + }); +} + +void PrepareEnv(vector &devices, Key &key, Query &query) +{ + /** + * @tc.steps: step1. ensure the watermark is no zero and finish timeSync and abilitySync + * @tc.expected: step1. should return OK. + */ + Value value = {'1'}; + std::map result; + ASSERT_TRUE(g_kvDelegatePtr->Put(key, value) == OK); + + DBStatus status = g_tool.SyncTest(g_kvDelegatePtr, devices, DistributedDB::SYNC_MODE_PUSH_ONLY, result, query); + EXPECT_TRUE(status == OK); + ASSERT_TRUE(result[g_deviceB->GetDeviceId()] == OK); +} + +void Sync(vector &devices, const DBStatus &targetStatus) +{ + std::map result; + DBStatus status = g_tool.SyncTest(g_kvDelegatePtr, devices, DistributedDB::SYNC_MODE_PUSH_ONLY, result); + EXPECT_TRUE(status == OK); + for (const auto &deviceId : devices) { + ASSERT_TRUE(result[deviceId] == targetStatus); + } +} + +void SyncWithQuery(vector &devices, const Query &query, const DBStatus &targetStatus) +{ + std::map result; + DBStatus status = g_tool.SyncTest(g_kvDelegatePtr, devices, DistributedDB::SYNC_MODE_PUSH_ONLY, result, query); + EXPECT_TRUE(status == OK); + for (const auto &deviceId : devices) { + ASSERT_TRUE(result[deviceId] == targetStatus); + } +} + +void SyncWithDeviceOffline(vector &devices, Key &key, Query &query) +{ + Value value = {'2'}; + ASSERT_TRUE(g_kvDelegatePtr->Put(key, value) == OK); + + /** + * @tc.steps: step2. invalid the sync msg + * @tc.expected: step2. should return TIME_OUT. + */ + SyncWithQuery(devices, query, TIME_OUT); + + /** + * @tc.steps: step3. device offline when sync + * @tc.expected: step3. should return COMM_FAILURE. + */ + SyncWithQuery(devices, query, COMM_FAILURE); +} +} + +/** + * @tc.name: AckSessionCheck 001 + * @tc.desc: Test ack session check function. + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, AckSessionCheck001, TestSize.Level3) +{ + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step1. deviceB sync to deviceA just for timeSync and abilitySync + * @tc.expected: step1. should return OK. + */ + std::map result; + ASSERT_TRUE(g_deviceB->Sync(SYNC_MODE_PUSH_ONLY, true) == OK); + + /** + * @tc.steps: step2. deviceA StartTransaction for prevent other sync action deviceB sync will fail + * @tc.expected: step2. should return OK. + */ + ASSERT_TRUE(g_kvDelegatePtr->StartTransaction() == OK); + + bool errCodeAck = false; + bool afterErrAck = false; + RegOnDispatchWithDelayAck(errCodeAck, afterErrAck); + + Key key = {'1'}; + Value value = {'1'}; + TimeStamp currentTime; + (void)OS::GetCurrentSysTimeInMicrosecond(currentTime); + EXPECT_TRUE(g_deviceB->PutData(key, value, currentTime, 0) == OK); + EXPECT_TRUE(g_deviceB->Sync(SYNC_MODE_PUSH_ONLY, true) == OK); + + Value outValue; + EXPECT_TRUE(g_kvDelegatePtr->Get(key, outValue) == NOT_FOUND); + + /** + * @tc.steps: step3. release the writeHandle and try again, sync success + * @tc.expected: step3. should return OK. + */ + EXPECT_TRUE(g_kvDelegatePtr->Commit() == OK); + EXPECT_TRUE(g_deviceB->Sync(SYNC_MODE_PUSH_ONLY, true) == OK); + + EXPECT_TRUE(g_kvDelegatePtr->Get(key, outValue) == E_OK); + EXPECT_EQ(outValue, value); +} + +/** + * @tc.name: AckSafeCheck001 + * @tc.desc: Test ack session check filter all bad ack in device offline scene. + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, AckSafeCheck001, TestSize.Level3) +{ + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + g_deviceB->Online(); + + Key key = {'1'}; + Query query = Query::Select().PrefixKey(key); + PrepareEnv(devices, key, query); + + std::condition_variable conditionOnline; + std::condition_variable conditionOffline; + bool onlineFlag = false; + bool invalid = false; + bool offlineFlag = false; + thread subThread([&onlineFlag, &conditionOnline, &offlineFlag, &conditionOffline]() { + LOGW("[Dispatch] NOW DEVICES IS OFFLINE"); + std::mutex offlineMtx; + std::unique_lock lck(offlineMtx); + conditionOffline.wait(lck, [&offlineFlag]{ return offlineFlag; }); + g_deviceB->Offline(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + g_deviceB->Online(); + onlineFlag = true; + conditionOnline.notify_all(); + LOGW("[Dispatch] NOW DEVICES IS ONLINE"); + }); + subThread.detach(); + + RegOnDispatchWithOffline(offlineFlag, invalid, conditionOffline); + + SyncWithDeviceOffline(devices, key, query); + + std::mutex onlineMtx; + std::unique_lock lck(onlineMtx); + conditionOnline.wait(lck, [&onlineFlag]{ return onlineFlag; }); + + /** + * @tc.steps: step4. sync again if has problem it will sync never end + * @tc.expected: step4. should return OK. + */ + SyncWithQuery(devices, query, OK); +} + + +/** + * @tc.name: WaterMarkCheck001 + * @tc.desc: Test waterMark work correct in lost package scene. + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, WaterMarkCheck001, TestSize.Level1) +{ + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + g_deviceB->Online(); + + Key key = {'1'}; + Query query = Query::Select().PrefixKey(key); + PrepareEnv(devices, key, query); + + /** + * @tc.steps: step2. query sync and set queryWaterMark + * @tc.expected: step2. should return OK. + */ + Value value = {'2'}; + ASSERT_TRUE(g_kvDelegatePtr->Put(key, value) == OK); + SyncWithQuery(devices, query, OK); + + /** + * @tc.steps: step3. sync and invalid msg for set local device waterMark + * @tc.expected: step3. should return TIME_OUT. + */ + bool invalidMsg = false; + RegOnDispatchWithInvalidMsg(invalidMsg); + value = {'3'}; + ASSERT_TRUE(g_kvDelegatePtr->Put(key, value) == OK); + Sync(devices, TIME_OUT); + + /** + * @tc.steps: step4. sync again see it work correct + * @tc.expected: step4. should return OK. + */ + SyncWithQuery(devices, query, OK); +} + +void RegOnDispatchToGetSyncCount(int &sendRequestCount, int sleepMs = 0) +{ + g_communicatorAggregator->RegOnDispatch([sleepMs, &sendRequestCount]( + const std::string &dev, Message *inMsg) { + if (dev == DEVICE_B && inMsg->GetMessageType() == TYPE_REQUEST) { + std::this_thread::sleep_for(std::chrono::milliseconds(sleepMs)); + sendRequestCount++; + LOGD("sendRequestCount++..."); + } + }); +} + +void TestDifferentSyncMode(SyncMode mode) +{ + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step1. deviceA put {k1, v1} + */ + Key key = {'1'}; + Value value = {'1'}; + DBStatus status = g_kvDelegatePtr->Put(key, value); + ASSERT_TRUE(status == OK); + + int sendRequestCount = 0; + RegOnDispatchToGetSyncCount(sendRequestCount); + + /** + * @tc.steps: step2. deviceA call sync and wait + * @tc.expected: step2. sync should return OK. + */ + std::map result; + status = g_tool.SyncTest(g_kvDelegatePtr, devices, mode, result); + ASSERT_TRUE(status == OK); + + /** + * @tc.expected: step2. onComplete should be called, DeviceB have {k1,v1}, send request message 3 times + */ + ASSERT_TRUE(result.size() == devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + g_deviceB->GetData(key, item); + EXPECT_TRUE(item.value == value); + + EXPECT_EQ(sendRequestCount, NORMAL_SYNC_SEND_REQUEST_CNT); + + /** + * @tc.steps: step3. reset sendRequestCount to 0, deviceA call sync and wait again without any change in db + * @tc.expected: step3. sync should return OK, and sendRequestCount should be 1, because this merge can not + * be skipped + */ + sendRequestCount = 0; + status = g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PUSH_ONLY, result); + ASSERT_TRUE(status == OK); + EXPECT_EQ(sendRequestCount, 1); +} + +/** + * @tc.name: PushSyncMergeCheck001 + * @tc.desc: Test push sync task merge, task can not be merged when the two sync task is not in the queue + * at the same time. + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangshijie + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck001, TestSize.Level1) +{ + TestDifferentSyncMode(SYNC_MODE_PUSH_ONLY); +} + +/** + * @tc.name: PushSyncMergeCheck002 + * @tc.desc: Test push_pull sync task merge, task can not be merged when the two sync task is not in the queue + * at the same time. + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangshijie + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck002, TestSize.Level1) +{ + TestDifferentSyncMode(SYNC_MODE_PUSH_PULL); +} + +void PrepareForSyncMergeTest(std::vector &devices, int &sendRequestCount) +{ + /** + * @tc.steps: step1. deviceA put {k1, v1} + */ + Key key = {'1'}; + Value value = {'1'}; + DBStatus status = g_kvDelegatePtr->Put(key, value); + ASSERT_TRUE(status == OK); + + RegOnDispatchToGetSyncCount(sendRequestCount, SLEEP_MILLISECONDS); + + /** + * @tc.steps: step2. deviceA call sync and don't wait + * @tc.expected: step2. sync should return OK. + */ + status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, + [&sendRequestCount, devices, key, value](const std::map& statusMap) { + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &pair : statusMap) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + g_deviceB->GetData(key, item); + EXPECT_EQ(item.value, value); + EXPECT_EQ(sendRequestCount, NORMAL_SYNC_SEND_REQUEST_CNT); + + // reset sendRequestCount to 0 + sendRequestCount = 0; + }); + ASSERT_TRUE(status == OK); +} + +/** + * @tc.name: PushSyncMergeCheck003 + * @tc.desc: Test push sync task merge, task can not be merged when there is change in db since last push sync + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangshijie + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck003, TestSize.Level3) +{ + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + int sendRequestCount = 0; + PrepareForSyncMergeTest(devices, sendRequestCount); + + /** + * @tc.steps: step3. deviceA call sync and don't wait + * @tc.expected: step3. sync should return OK. + */ + Key key = {'1'}; + Value value = {'2'}; + status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, + [&sendRequestCount, devices, key, value, this](const std::map& statusMap) { + /** + * @tc.expected: when the second sync task return, sendRequestCount should be 1, because this merge can not be + * skipped, but it is no need to do time sync and ability sync, only need to do data sync + */ + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &pair : statusMap) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + g_deviceB->GetData(key, item); + EXPECT_EQ(item.value, value); + }); + ASSERT_TRUE(status == OK); + + /** + * @tc.steps: step4. deviceA put {k1, v2} + */ + while (sendRequestCount < TWO_CNT) { + std::this_thread::sleep_for(std::chrono::milliseconds(THREE_HUNDRED)); + } + status = g_kvDelegatePtr->Put(key, value); + ASSERT_TRUE(status == OK); + // wait for the second sync task finish + std::this_thread::sleep_for(std::chrono::seconds(TEN_SECONDS)); + EXPECT_EQ(sendRequestCount, 1); +} + +/** + * @tc.name: PushSyncMergeCheck004 + * @tc.desc: Test push sync task merge, task can be merged when there is no change in db since last push sync + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangshijie + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck004, TestSize.Level3) +{ + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + int sendRequestCount = 0; + PrepareForSyncMergeTest(devices, sendRequestCount); + + /** + * @tc.steps: step3. deviceA call sync and don't wait + * @tc.expected: step3. sync should return OK. + */ + status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, + [devices, this](const std::map& statusMap) { + /** + * @tc.expected: when the second sync task return, sendRequestCount should be 0, because this merge can be + * skipped + */ + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &pair : statusMap) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + }); + ASSERT_TRUE(status == OK); + std::this_thread::sleep_for(std::chrono::seconds(TEN_SECONDS)); + EXPECT_EQ(sendRequestCount, 0); +} + +void RegOnDispatchWithInvalidMsgAndCnt(int &sendRequestCount, int sleepMs, bool &invalid) +{ + g_communicatorAggregator->RegOnDispatch([&sendRequestCount, sleepMs, &invalid]( + const std::string &dev, Message *inMsg) { + if (dev == DEVICE_B && !invalid && inMsg->GetMessageType() == TYPE_REQUEST) { + inMsg->SetMessageType(TYPE_INVALID); + inMsg->SetMessageId(INVALID_MESSAGE_ID); + sendRequestCount++; + invalid = true; + LOGW("[Dispatch]invalid THIS MSG, sendRequestCount = %d", sendRequestCount); + std::this_thread::sleep_for(std::chrono::milliseconds(sleepMs)); + } + }); +} + +/** + * @tc.name: PushSyncMergeCheck005 + * @tc.desc: Test push sync task merge, task cannot be merged when the last push sync is failed + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangshijie + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, SyncMergeCheck005, TestSize.Level3) +{ + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + /** + * @tc.steps: step1. deviceA put {k1, v1} + */ + Key key = {'1'}; + Value value = {'1'}; + status = g_kvDelegatePtr->Put(key, value); + ASSERT_TRUE(status == OK); + + int sendRequestCount = 0; + bool invalid = false; + RegOnDispatchWithInvalidMsgAndCnt(sendRequestCount, SLEEP_MILLISECONDS, invalid); + + /** + * @tc.steps: step2. deviceA call sync and don't wait + * @tc.expected: step2. sync should return TIME_OUT. + */ + status =g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, + [&sendRequestCount, devices, this](const std::map& statusMap) { + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &deviceId : devices) { + ASSERT_EQ(statusMap.at(deviceId), TIME_OUT); + } + }); + EXPECT_TRUE(status == OK); + + /** + * @tc.steps: step3. deviceA call sync and don't wait + * @tc.expected: step3. sync should return OK. + */ + status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, + [key, value, &sendRequestCount, devices, this](const std::map& statusMap) { + /** + * @tc.expected: when the second sync task return, sendRequestCount should be 3, because this merge can not be + * skipped, deviceB should have {k1, v1}. + */ + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &pair : statusMap) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_EQ(pair.second, OK); + } + VirtualDataItem item; + g_deviceB->GetData(key, item); + EXPECT_EQ(item.value, value); + }); + ASSERT_TRUE(status == OK); + while (sendRequestCount < 1) { + std::this_thread::sleep_for(std::chrono::milliseconds(THREE_HUNDRED)); + } + sendRequestCount = 0; + RegOnDispatchToGetSyncCount(sendRequestCount, SLEEP_MILLISECONDS); + + // wait for the second sync task finish + std::this_thread::sleep_for(std::chrono::seconds(TEN_SECONDS)); + EXPECT_EQ(sendRequestCount, NORMAL_SYNC_SEND_REQUEST_CNT); +} + +void PrePareForQuerySyncMergeTest(bool isQuerySync, std::vector &devices, + Key &key, Value &value, int &sendRequestCount) +{ + DBStatus status = OK; + /** + * @tc.steps: step1. deviceA put {k1, v1}...{k10, v10} + */ + Query query = Query::Select().PrefixKey(key); + const int dataSize = 10; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + value.push_back(i); + status = g_kvDelegatePtr->Put(key, value); + ASSERT_TRUE(status == OK); + key.pop_back(); + value.pop_back(); + } + + RegOnDispatchToGetSyncCount(sendRequestCount, SLEEP_MILLISECONDS); + /** + * @tc.steps: step2. deviceA call query sync and don't wait + * @tc.expected: step2. sync should return OK. + */ + auto completeCallBack = [&sendRequestCount, &key, &value, dataSize, devices] + (const std::map& statusMap) { + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &pair : statusMap) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_EQ(pair.second, OK); + } + // when first sync finish, DeviceB have {k1,v1}, {k3,v3}, {k5,v5} .. send request message 3 times + VirtualDataItem item; + for (int i = 0; i < dataSize; i++) { + key.push_back(i); + value.push_back(i); + g_deviceB->GetData(key, item); + EXPECT_EQ(item.value, value); + key.pop_back(); + value.pop_back(); + } + EXPECT_EQ(sendRequestCount, NORMAL_SYNC_SEND_REQUEST_CNT); + // reset sendRequestCount to 0 + sendRequestCount = 0; + }; + if (isQuerySync) { + status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, completeCallBack, query, false); + } else { + status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, completeCallBack); + } + ASSERT_TRUE(status == OK); +} + +/** + * @tc.name: QuerySyncMergeCheck001 + * @tc.desc: Test query push sync task merge, task can be merged when there is no change in db since last query sync + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangshijie + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck001, TestSize.Level3) +{ + std::vector devices; + int sendRequestCount = 0; + devices.push_back(g_deviceB->GetDeviceId()); + + Key key{'1'}; + Value value{'1'}; + Query query = Query::Select().PrefixKey(key); + PrePareForQuerySyncMergeTest(true, devices, key, value, sendRequestCount); + + /** + * @tc.steps: step3. deviceA call query sync and don't wait + * @tc.expected: step3. sync should return OK. + */ + DBStatus status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, + [devices, this](const std::map& statusMap) { + /** + * @tc.expected: when the second sync task return, sendRequestCount should be 0, because this merge can be + * skipped because there is no change in db since last query sync + */ + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &pair : statusMap) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + }, query, false); + ASSERT_TRUE(status == OK); + std::this_thread::sleep_for(std::chrono::seconds(TEN_SECONDS)); + EXPECT_EQ(sendRequestCount, 0); +} + +/** + * @tc.name: QuerySyncMergeCheck002 + * @tc.desc: Test query push sync task merge, task can not be merged when there is change in db since last sync + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangshijie + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck002, TestSize.Level3) +{ + std::vector devices; + int sendRequestCount = 0; + devices.push_back(g_deviceB->GetDeviceId()); + + Key key{'1'}; + Value value{'1'}; + Query query = Query::Select().PrefixKey(key); + PrePareForQuerySyncMergeTest(true, devices, key, value, sendRequestCount); + + /** + * @tc.steps: step3. deviceA call query sync and don't wait + * @tc.expected: step3. sync should return OK. + */ + Value value3{'3'}; + DBStatus status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, + [&sendRequestCount, devices, key, value3, this](const std::map& statusMap) { + /** + * @tc.expected: when the second sync task return, sendRequestCount should be 1, because this merge can not be + * skipped when there is change in db since last query sync, deviceB have {k1, v1'} + */ + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &pair : statusMap) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + g_deviceB->GetData(key, item); + EXPECT_TRUE(item.value == value3); + EXPECT_EQ(sendRequestCount, 1); + }, query, false); + ASSERT_TRUE(status == OK); + + /** + * @tc.steps: step4. deviceA put {k1, v1'} + * @tc.steps: step4. reset sendRequestCount to 0, deviceA call sync and wait + * @tc.expected: step4. sync should return OK, and sendRequestCount should be 1, because this merge can not + * be skipped + */ + while (sendRequestCount < TWO_CNT) { + std::this_thread::sleep_for(std::chrono::milliseconds(THREE_HUNDRED)); + } + g_kvDelegatePtr->Put(key, value3); + std::this_thread::sleep_for(std::chrono::seconds(TEN_SECONDS)); +} + +/** + * @tc.name: QuerySyncMergeCheck003 + * @tc.desc: Test query push sync task merge, task can not be merged when then query id is different + * @tc.type: FUNC + * @tc.require: AR000F3OOV + * @tc.author: zhangshijie + */ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck003, TestSize.Level3) +{ + std::vector devices; + int sendRequestCount = 0; + devices.push_back(g_deviceB->GetDeviceId()); + + Key key{'1'}; + Value value{'1'}; + PrePareForQuerySyncMergeTest(true, devices, key, value, sendRequestCount); + + /** + * @tc.steps: step3. deviceA call another query sync + * @tc.expected: step3. sync should return OK. + */ + Key key2 = {'2'}; + Value value2 = {'2'}; + DBStatus status = g_kvDelegatePtr->Put(key2, value2); + ASSERT_TRUE(status == OK); + Query query2 = Query::Select().PrefixKey(key2); + status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, + [&sendRequestCount, key2, value2, devices, this](const std::map& statusMap) { + /** + * @tc.expected: when the second sync task return, sendRequestCount should be 1, because this merge can not be + * skipped, deviceB have {k2,v2} + */ + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &pair : statusMap) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + VirtualDataItem item; + g_deviceB->GetData(key2, item); + EXPECT_TRUE(item.value == value2); + EXPECT_EQ(sendRequestCount, 1); + }, query2, false); + ASSERT_TRUE(status == OK); + std::this_thread::sleep_for(std::chrono::seconds(TEN_SECONDS)); +} + +/** +* @tc.name: QuerySyncMergeCheck004 +* @tc.desc: Test query push sync task merge, task can be merged when there is no change in db since last push sync +* @tc.type: FUNC +* @tc.require: AR000F3OOV +* @tc.author: zhangshijie +*/ +HWTEST_F(DistributedDBSingleVerP2PSyncCheckTest, QuerySyncMergeCheck004, TestSize.Level3) +{ + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + Key key{'1'}; + Value value{'1'}; + int sendRequestCount = 0; + PrePareForQuerySyncMergeTest(false, devices, key, value, sendRequestCount); + + /** + * @tc.steps: step3. deviceA call query sync without any change in db + * @tc.expected: step3. sync should return OK, and sendRequestCount should be 0, because this merge can be skipped + */ + Query query = Query::Select().PrefixKey(key); + status = g_kvDelegatePtr->Sync(devices, SYNC_MODE_PUSH_ONLY, + [devices, this](const std::map& statusMap) { + /** + * @tc.expected step3: when the second sync task return, sendRequestCount should be 0, because this merge + * can be skipped because there is no change in db since last push sync + */ + ASSERT_TRUE(statusMap.size() == devices.size()); + for (const auto &pair : statusMap) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + EXPECT_TRUE(pair.second == OK); + } + }, query, false); + ASSERT_TRUE(status == OK); + std::this_thread::sleep_for(std::chrono::seconds(TEN_SECONDS)); + EXPECT_EQ(sendRequestCount, 0); } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp index 23706825d6a0a35b9471d2aa4276f978739e368c..5ae7a97af6c1e9041c6b662ec7620ccf28288c1c 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_single_ver_p2p_sync_test.cpp @@ -13,22 +13,16 @@ * limitations under the License. */ +#include #include #include -#include -#include "distributeddb_tools_unit_test.h" +#include "db_constant.h" #include "distributeddb_data_generate_unit_test.h" -#include "kv_store_observer.h" +#include "distributeddb_tools_unit_test.h" #include "kv_store_nb_delegate.h" -#include "vitural_communicator_aggregator.h" -#include "vitural_communicator.h" -#include "vitural_device.h" -#include "isyncer.h" -#include "virtual_single_ver_sync_db_Interface.h" -#include "time_sync.h" +#include "kv_virtual_device.h" #include "platform_specific.h" -#include "db_constant.h" using namespace testing::ext; using namespace DistributedDB; @@ -52,8 +46,8 @@ namespace { DBStatus g_kvDelegateStatus = INVALID_ARGS; KvStoreNbDelegate* g_kvDelegatePtr = nullptr; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; - VituralDevice* g_deviceB = nullptr; - VituralDevice* g_deviceC = nullptr; + KvVirtualDevice *g_deviceB = nullptr; + KvVirtualDevice *g_deviceC = nullptr; // the type of g_kvDelegateCallback is function auto g_kvDelegateCallback = bind(&DistributedDBToolsUnitTest::KvStoreNbDelegateCallback, @@ -103,6 +97,7 @@ void DistributedDBSingleVerP2PSyncTest::TearDownTestCase(void) void DistributedDBSingleVerP2PSyncTest::SetUp(void) { + DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create virtual device B and C, and get a KvStoreNbDelegate as deviceA */ @@ -110,13 +105,13 @@ void DistributedDBSingleVerP2PSyncTest::SetUp(void) g_mgr.GetKvStore(STORE_ID, option, g_kvDelegateCallback); ASSERT_TRUE(g_kvDelegateStatus == OK); ASSERT_TRUE(g_kvDelegatePtr != nullptr); - g_deviceB = new (std::nothrow) VituralDevice(DEVICE_B); + g_deviceB = new (std::nothrow) KvVirtualDevice(DEVICE_B); ASSERT_TRUE(g_deviceB != nullptr); VirtualSingleVerSyncDBInterface *syncInterfaceB = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(syncInterfaceB != nullptr); ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, syncInterfaceB), E_OK); - g_deviceC = new (std::nothrow) VituralDevice(DEVICE_C); + g_deviceC = new (std::nothrow) KvVirtualDevice(DEVICE_C); ASSERT_TRUE(g_deviceC != nullptr); VirtualSingleVerSyncDBInterface *syncInterfaceC = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(syncInterfaceC != nullptr); @@ -1966,6 +1961,141 @@ HWTEST_F(DistributedDBSingleVerP2PSyncTest, PermissionCheck006, TestSize.Level3) EXPECT_EQ(g_mgr.SetPermissionCheckCallback(nullCallback), OK); } +/** + * @tc.name: PermissionCheck007 + * @tc.desc: deviceA PermissionCheck, deviceB not pass, deviceC pass in SYNC_MODE_AUTO_PUSH + * @tc.type: FUNC + * @tc.require: AR000G3RLS + * @tc.author: zhuwentao + */ +HWTEST_F(DistributedDBSingleVerP2PSyncTest, PermissionCheck007, TestSize.Level3) +{ + /** + * @tc.steps: step1. SetPermissionCheckCallback + * @tc.expected: step1. return OK. + */ + auto permissionCheckCallback = [] (const std::string &userId, const std::string &appId, const std::string &storeId, + const std::string &deviceId, uint8_t flag) -> bool { + if (deviceId == g_deviceC->GetDeviceId() && + (flag & (CHECK_FLAG_RECEIVE | CHECK_FLAG_AUTOSYNC))) { + LOGD("in RunPermissionCheck callback func, check not pass, flag:%d", flag); + return false; + } else { + LOGD("in RunPermissionCheck callback func, check pass, flag:%d", flag); + return true; + } + }; + EXPECT_EQ(g_mgr.SetPermissionCheckCallback(permissionCheckCallback), OK); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + devices.push_back(g_deviceC->GetDeviceId()); + /** + * @tc.steps: step2. deviceA set auto sync + */ + bool autoSync = true; + PragmaData data = static_cast(&autoSync); + status = g_kvDelegatePtr->Pragma(AUTO_SYNC, data); + ASSERT_EQ(status, OK); + + /** + * @tc.steps: step3. deviceA put {k1, v1}, and sleep 1s + */ + Key key = {'1'}; + Value value = {'1'}; + status = g_kvDelegatePtr->Put(key, value); + ASSERT_TRUE(status == OK); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_TIME)); + + /** + * @tc.steps: step3. check value in device B and not in device C. + */ + VirtualDataItem item; + g_deviceC->GetData(key, item); + EXPECT_TRUE(item.value.empty()); + g_deviceB->GetData(key, item); + EXPECT_TRUE(item.value == value); + PermissionCheckCallbackV2 nullCallback; + EXPECT_EQ(g_mgr.SetPermissionCheckCallback(nullCallback), OK); +} + +/** ++ * @tc.name: PermissionCheck008 ++ * @tc.desc: deviceA PermissionCheck, deviceB not pass, deviceC pass in SYNC_MODE_AUTO_PULL ++ * @tc.type: FUNC ++ * @tc.require: AR000G3RLS ++ * @tc.author: zhangqiquan ++ */ +HWTEST_F(DistributedDBSingleVerP2PSyncTest, PermissionCheck008, TestSize.Level3) +{ + /** + * @tc.steps: step1. SetPermissionCheckCallback + * @tc.expected: step1. return OK. + */ + auto permissionCheckCallback = [] (const std::string &userId, const std::string &appId, const std::string &storeId, + const std::string &deviceId, uint8_t flag) -> bool { + if (deviceId == g_deviceC->GetDeviceId() && + (flag & CHECK_FLAG_SPONSOR)) { + LOGD("in RunPermissionCheck callback func, check not pass, flag:%d", flag); + return false; + } else { + LOGD("in RunPermissionCheck callback func, check pass, flag:%d", flag); + return true; + } + }; + EXPECT_EQ(g_mgr.SetPermissionCheckCallback(permissionCheckCallback), OK); + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + devices.push_back(g_deviceC->GetDeviceId()); + + /** + * @tc.steps: step2. deviceB put {k1, v1} + */ + Key key = {'1'}; + Value value = {'1'}; + g_deviceB->PutData(key, value, 0, 0); + + /** + * @tc.steps: step2. device put {k2, v2} + */ + Key key2 = {'2'}; + Value value2 = {'2'}; + g_deviceC->PutData(key2, value2, 0, 0); + ASSERT_TRUE(status == OK); + + /** + * @tc.steps: step3. deviceA call push sync + * @tc.expected: step3. sync should return OK. + */ + std::map result; + status = g_tool.SyncTest(g_kvDelegatePtr, devices, SYNC_MODE_PULL_ONLY, result); + ASSERT_TRUE(status == OK); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_TIME)); + + /** + * @tc.expected: step4. onComplete should be called, + * status == PERMISSION_CHECK_FORBID_SYNC, deviceB and deviceC do not have {k1, v1} + */ + ASSERT_TRUE(result.size() == devices.size()); + for (const auto &pair : result) { + LOGD("dev %s, status %d", pair.first.c_str(), pair.second); + if (g_deviceC->GetDeviceId() == pair.first) { + EXPECT_TRUE(pair.second == PERMISSION_CHECK_FORBID_SYNC); + } else { + EXPECT_TRUE(pair.second == OK); + } + } + /** + * @tc.steps: step5. check value in device A + */ + Value value4; + EXPECT_TRUE(g_kvDelegatePtr->Get(key, value4) == OK); + EXPECT_TRUE(g_kvDelegatePtr->Get(key2, value4) == NOT_FOUND); + PermissionCheckCallbackV2 nullCallback; + EXPECT_EQ(g_mgr.SetPermissionCheckCallback(nullCallback), OK); +} + /** * @tc.name: SaveDataNotify001 * @tc.desc: Test SaveDataNotify function, delay < 30s should sync ok, > 36 should timeout @@ -2032,3 +2162,63 @@ HWTEST_F(DistributedDBSingleVerP2PSyncTest, SaveDataNotify001, TestSize.Level3) ASSERT_TRUE(result.size() == devices.size()); ASSERT_TRUE(result[DEVICE_B] == TIME_OUT); } + +/** + * @tc.name: SametimeSync001 + * @tc.desc: Test 2 device sync with each other + * @tc.type: FUNC + * @tc.require: AR000CCPOM + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSingleVerP2PSyncTest, SametimeSync001, TestSize.Level3) +{ + DBStatus status = OK; + std::vector devices; + devices.push_back(g_deviceB->GetDeviceId()); + + int responseCount = 0; + int requestCount = 0; + Key key = {'1'}; + Value value = {'1'}; + /** + * @tc.steps: step1. make sure deviceB send pull firstly and response_pull secondly + * @tc.expected: step1. deviceA put data when finish push task. put data should return OK. + */ + g_communicatorAggregator->RegOnDispatch([&responseCount, &requestCount, &key, &value]( + const std::string &target, DistributedDB::Message *msg) { + if (target == "real_device" && msg->GetMessageId() == DATA_SYNC_MESSAGE) { + if (msg->GetMessageType() == TYPE_RESPONSE) { + responseCount++; + if (responseCount == 1) { // 1 is the ack which B response A's push task + EXPECT_EQ(g_kvDelegatePtr->Put(key, value), DBStatus::OK); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } else if (responseCount == 2) { // 2 is the ack which B response A's response_pull task + msg->SetErrorNo(E_FEEDBACK_COMMUNICATOR_NOT_FOUND); + } + } if (msg->GetMessageType() == TYPE_REQUEST) { + requestCount++; + if (requestCount == 1) { // 1 is A push task + std::this_thread::sleep_for(std::chrono::seconds(2)); // sleep 2 sec + } + } + } + }); + /** + * @tc.steps: step2. deviceA,deviceB sync to each other at same time + * @tc.expected: step2. sync should return OK. + */ + std::map result; + std::thread subThread([]{ + g_deviceB->Sync(DistributedDB::SYNC_MODE_PULL_ONLY, true); + }); + status = g_tool.SyncTest(g_kvDelegatePtr, devices, DistributedDB::SYNC_MODE_PUSH_PULL, result); + subThread.join(); + g_communicatorAggregator->RegOnDispatch(nullptr); + + EXPECT_TRUE(status == OK); + ASSERT_TRUE(result.size() == devices.size()); + EXPECT_TRUE(result[DEVICE_B] == OK); + Value actualValue; + g_kvDelegatePtr->Get(key, actualValue); + EXPECT_EQ(actualValue, value); +} diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2fc18aa7e61d52d5a6ac3eeb731a7678ad4bbf8d --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_sync_module_test.cpp @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "db_constant.h" +#include "distributeddb_tools_unit_test.h" +#include "mock_communicator.h" +#include "mock_single_ver_data_sync.h" +#include "mock_single_ver_sync_task_context.h" +#include "sliding_window_receiver.h" + +using namespace testing::ext; +using namespace DistributedDB; +using namespace DistributedDBUnitTest; + +class DistributedDBSyncModuleTest : public testing::Test { +public: + static void SetUpTestCase(void); + static void TearDownTestCase(void); + void SetUp(); + void TearDown(); +}; + +void DistributedDBSyncModuleTest::SetUpTestCase(void) +{ +} + +void DistributedDBSyncModuleTest::TearDownTestCase(void) +{ +} + +void DistributedDBSyncModuleTest::SetUp(void) +{ +} + +void DistributedDBSyncModuleTest::TearDown(void) +{ +} + + +namespace { + void InitMessage(DataSyncMessageInfo &info) + { + info.messageId_ = DistributedDB::DATA_SYNC_MESSAGE; + info.messageType_ = DistributedDB::TYPE_REQUEST; + info.sendCode_ = -DistributedDB::E_UNFINISHED; + info.mode_ = DistributedDB::PUSH; + } + + int BuildPushMessage(DataSyncMessageInfo &info, DistributedDB::Message *&message) + { + return DistributedDBToolsUnitTest::BuildMessage(info, message); + } +} + +/** + * @tc.name: SlidingWindow001 + * @tc.desc: Test sliding window receive data when water error happen + * @tc.type: FUNC + * @tc.require: AR000EV1G6 + * @tc.author: zhangqiquan + */ +HWTEST_F(DistributedDBSyncModuleTest, SlidingWindow001, TestSize.Level1) +{ + std::shared_ptr mockSingleVerDataSync = std::make_shared(); + auto *context = new MockSingleVerSyncTaskContext(); + + SlidingWindowReceiver receiver; + receiver.Initialize(context, mockSingleVerDataSync); + uint32_t sequenceId = 1; + uint32_t sessionId = 1; + uint64_t packedId = 1; + DataSyncMessageInfo info{}; + InitMessage(info); + info.sequenceId_ = sequenceId; + info.sessionId_ = sessionId; + info.packetId_ = packedId; + + DistributedDB::Message *message = nullptr; + int status = BuildPushMessage(info, message); + EXPECT_EQ(status, E_OK); + + /** + * @tc.steps: step1. sliding window receive data first and now water error happen + * @tc.expected: step1. receive msg should return -E_NOT_NEED_DELETE_MSG. + */ + EXPECT_CALL(*context, HandleDataRequestRecv(testing::_)).WillRepeatedly(testing::Return(E_OK)); + EXPECT_CALL(*static_cast(mockSingleVerDataSync.get()), + SendDataAck(testing::_, testing::_, testing::_, testing::_)).WillRepeatedly(testing::Return(E_OK)); + context->SetReceiveWaterMarkErr(true); + EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); + + /** + * @tc.steps: step2. sliding window receive data which is has diff sessionId + * @tc.expected: step2. receive msg should return -E_NOT_NEED_DELETE_MSG. + */ + info.sessionId_++; + status = BuildPushMessage(info, message); + EXPECT_EQ(status, E_OK); + EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); + + /** + * @tc.steps: step3. sliding window receive data which is has diff sequenceId + * @tc.expected: step3. receive msg should return -E_NOT_NEED_DELETE_MSG. + */ + info.sequenceId_++; + info.packetId_++; + status = BuildPushMessage(info, message); + EXPECT_EQ(status, E_OK); + EXPECT_EQ(receiver.Receive(message), -E_NOT_NEED_DELETE_MSG); + + receiver.Clear(); + // ensure to call DecRefObj in another thread + std::this_thread::sleep_for(std::chrono::seconds(1)); + RefObject::KillAndDecObjRef(context); + context = nullptr; +} \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_syncer_device_manager_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_syncer_device_manager_test.cpp index 12cadb2912b7aebf50b387ca7d1c3bd825a06b04..c4593b97fb791fcad6747071df311bbcaca147c1 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_syncer_device_manager_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_syncer_device_manager_test.cpp @@ -16,14 +16,15 @@ #include #include -#include "vitural_communicator_aggregator.h" -#include "vitural_communicator.h" -#include "vitural_device.h" #include "device_manager.h" -#include "virtual_single_ver_sync_db_Interface.h" +#include "distributeddb_tools_unit_test.h" +#include "kv_virtual_device.h" #include "log_print.h" #include "parcel.h" #include "sync_types.h" +#include "virtual_communicator.h" +#include "virtual_communicator_aggregator.h" +#include "virtual_single_ver_sync_db_Interface.h" using namespace testing::ext; using namespace DistributedDB; @@ -34,8 +35,8 @@ namespace { const std::string DEVICE_C = "deviceC"; VirtualCommunicatorAggregator* g_communicatorAggregator = nullptr; VirtualCommunicator* g_virtualCommunicator = nullptr; - VituralDevice* g_deviceB = nullptr; - VituralDevice* g_deviceC = nullptr; + KvVirtualDevice *g_deviceB = nullptr; + KvVirtualDevice *g_deviceC = nullptr; DeviceManager *g_deviceManager = nullptr; const int WAIT_TIME = 1000; } @@ -57,7 +58,7 @@ void DistributedDBSyncerDeviceManagerTest::SetUpTestCase(void) ASSERT_TRUE(g_communicatorAggregator != nullptr); RuntimeContext::GetInstance()->SetCommunicatorAggregator(g_communicatorAggregator); int errCode; - g_virtualCommunicator = static_cast(g_communicatorAggregator->AllocCommunicator(0, errCode)); + g_virtualCommunicator = static_cast(g_communicatorAggregator->AllocCommunicator(0, errCode)); ASSERT_TRUE(g_virtualCommunicator != nullptr); } @@ -72,23 +73,24 @@ void DistributedDBSyncerDeviceManagerTest::TearDownTestCase(void) void DistributedDBSyncerDeviceManagerTest::SetUp(void) { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: Init a DeviceManager and DeviceB, C */ g_deviceManager = new (std::nothrow) DeviceManager; ASSERT_TRUE(g_deviceManager != nullptr); - g_deviceManager->Initialize(g_virtualCommunicator, nullptr); + g_deviceManager->Initialize(g_virtualCommunicator, nullptr, nullptr); g_virtualCommunicator->RegOnConnectCallback( std::bind(&DeviceManager::OnDeviceConnectCallback, g_deviceManager, std::placeholders::_1, std::placeholders::_2), nullptr); - g_deviceB = new (std::nothrow) VituralDevice(DEVICE_B); + g_deviceB = new (std::nothrow) KvVirtualDevice(DEVICE_B); ASSERT_TRUE(g_deviceB != nullptr); VirtualSingleVerSyncDBInterface *syncInterfaceB = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(syncInterfaceB != nullptr); ASSERT_EQ(g_deviceB->Initialize(g_communicatorAggregator, syncInterfaceB), E_OK); - g_deviceC = new (std::nothrow) VituralDevice(DEVICE_C); + g_deviceC = new (std::nothrow) KvVirtualDevice(DEVICE_C); ASSERT_TRUE(g_deviceC != nullptr); VirtualSingleVerSyncDBInterface *syncInterfaceC = new (std::nothrow) VirtualSingleVerSyncDBInterface(); ASSERT_TRUE(syncInterfaceC != nullptr); @@ -216,7 +218,7 @@ HWTEST_F(DistributedDBSyncerDeviceManagerTest, GetDevices001, TestSize.Level0) HWTEST_F(DistributedDBSyncerDeviceManagerTest, SendBroadCast001, TestSize.Level1) { bool deviceBReviced = false; -bool deviceCReviced = false; + bool deviceCReviced = false; /** * @tc.steps: step1. deviceB, C set OnRemoteDataChanged callback diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp index 434f89c50faa7e6fb00267f31d95382ccd4f198e..f002c34f572a333c189ebe7f10cfaf5282250d9d 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/distributeddb_time_sync_test.cpp @@ -15,13 +15,13 @@ #include -#include "distributeddb_tools_unit_test.h" #include "distributeddb_data_generate_unit_test.h" -#include "virtual_time_sync_communicator.h" +#include "distributeddb_tools_unit_test.h" #include "isyncer.h" -#include "virtual_single_ver_sync_db_Interface.h" -#include "sync_types.h" #include "single_ver_sync_state_machine.h" +#include "sync_types.h" +#include "virtual_single_ver_sync_db_Interface.h" +#include "virtual_time_sync_communicator.h" using namespace std; using namespace testing::ext; @@ -65,6 +65,7 @@ void DistributedDBTimeSyncTest::TearDownTestCase(void) void DistributedDBTimeSyncTest::SetUp(void) { + DistributedDBUnitTest::DistributedDBToolsUnitTest::PrintTestCaseInfo(); /** * @tc.setup: create the instance for virtual communicator, virtual storage component and time syncer */ @@ -170,7 +171,7 @@ HWTEST_F(DistributedDBTimeSyncTest, NormalSync001, TestSize.Level0) * @tc.expected: step4. (offsetB - offsetA ) - timeOffset < 100ms. */ TimeOffset timeOffset = 0; - g_timeSyncA->GetTimeOffset(timeOffset); + g_timeSyncA->GetTimeOffset(timeOffset, TIME_SYNC_WAIT_TIME); offsetB = g_metadataB->GetLocalTimeOffset(); offsetA = g_metadataA->GetLocalTimeOffset(); EXPECT_TRUE(abs(offsetB - offsetA - timeOffset) < NETWORK_DELAY); @@ -211,7 +212,7 @@ HWTEST_F(DistributedDBTimeSyncTest, NormalSync002, TestSize.Level0) * @tc.expected: step3. (offsetB - offsetA ) - timeOffset < 100ms. */ TimeOffset timeOffset; - g_timeSyncA->GetTimeOffset(timeOffset); + g_timeSyncA->GetTimeOffset(timeOffset, TIME_SYNC_WAIT_TIME); TimeOffset offsetB = g_metadataB->GetLocalTimeOffset(); TimeOffset offsetA = g_metadataA->GetLocalTimeOffset(); EXPECT_TRUE(abs(offsetB - offsetA - timeOffset) < NETWORK_DELAY); @@ -263,7 +264,7 @@ HWTEST_F(DistributedDBTimeSyncTest, NormalSync003, TestSize.Level0) * @tc.expected: step4. (offsetB - offsetA ) - timeOffset < 100ms. */ TimeOffset timeOffset = 0; - g_timeSyncA->GetTimeOffset(timeOffset); + g_timeSyncA->GetTimeOffset(timeOffset, TIME_SYNC_WAIT_TIME); TimeOffset absTimeOffset = abs(timeOffset); EXPECT_TRUE(abs(offsetB - offsetA - absTimeOffset) < NETWORK_DELAY); @@ -475,6 +476,6 @@ HWTEST_F(DistributedDBTimeSyncTest, SyncTimeout001, TestSize.Level2) * @tc.expected: step2. Start the time sync task return E_TIMEOUT */ TimeOffset offset; - errCode = g_timeSyncA->GetTimeOffset(offset); + errCode = g_timeSyncA->GetTimeOffset(offset, TIME_SYNC_WAIT_TIME); EXPECT_TRUE(errCode == -E_TIMEOUT); } \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4771e45222f1a23a479cc998c879997abfe2342c --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.cpp @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "generic_virtual_device.h" + +#include "multi_ver_sync_task_context.h" +#include "single_ver_sync_task_context.h" + +namespace DistributedDB { +GenericVirtualDevice::GenericVirtualDevice(std::string deviceId) + : communicateHandle_(nullptr), + communicatorAggregator_(nullptr), + storage_(nullptr), + metadata_(nullptr), + deviceId_(std::move(deviceId)), + remoteDeviceId_("real_device"), + context_(nullptr) +{ +} + +GenericVirtualDevice::~GenericVirtualDevice() +{ + std::mutex cvMutex; + std::condition_variable cv; + bool finished = false; + Offline(); + + if (communicateHandle_ != nullptr) { + communicateHandle_->RegOnMessageCallback(nullptr, nullptr); + communicatorAggregator_->ReleaseCommunicator(communicateHandle_); + communicateHandle_ = nullptr; + } + communicatorAggregator_ = nullptr; + + if (context_ != nullptr) { + ISyncInterface *storage = storage_; + context_->OnLastRef([storage, &cv, &cvMutex, &finished]() { + delete storage; + { + std::lock_guard lock(cvMutex); + finished = true; + } + cv.notify_one(); + }); + RefObject::KillAndDecObjRef(context_); + std::unique_lock lock(cvMutex); + cv.wait(lock, [&finished] { return finished; }); + } else { + delete storage_; + } + context_ = nullptr; + metadata_ = nullptr; + storage_ = nullptr; +} + +int GenericVirtualDevice::Initialize(VirtualCommunicatorAggregator *comAggregator, ISyncInterface *syncInterface) +{ + if ((comAggregator == nullptr) || (syncInterface == nullptr)) { + return -E_INVALID_ARGS; + } + + communicatorAggregator_ = comAggregator; + int errCode = E_OK; + communicateHandle_ = communicatorAggregator_->AllocCommunicator(deviceId_, errCode); + if (communicateHandle_ == nullptr) { + return errCode; + } + + storage_ = syncInterface; + metadata_ = std::make_shared(); + if (metadata_->Initialize(storage_) != E_OK) { + LOGE("metadata_ init failed"); + return -E_NOT_SUPPORT; + } + if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_SVD) { + context_ = new (std::nothrow) SingleVerSyncTaskContext; + subManager_ = std::make_shared(); + static_cast(context_)->SetSubscribeManager(subManager_); + } else if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_RELATION) { + context_ = new (std::nothrow) SingleVerSyncTaskContext; + } else { + context_ = new (std::nothrow) MultiVerSyncTaskContext; + } + if (context_ == nullptr) { + return -E_OUT_OF_MEMORY; + } + communicateHandle_->RegOnMessageCallback( + std::bind(&GenericVirtualDevice::MessageCallback, this, std::placeholders::_1, std::placeholders::_2), []() {}); + context_->Initialize(remoteDeviceId_, storage_, metadata_, communicateHandle_); + context_->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); + context_->RegOnSyncTask(std::bind(&GenericVirtualDevice::StartResponseTask, this)); + return E_OK; +} + +void GenericVirtualDevice::SetDeviceId(const std::string &deviceId) +{ + deviceId_ = deviceId; +} + +std::string GenericVirtualDevice::GetDeviceId() const +{ + return deviceId_; +} + +int GenericVirtualDevice::MessageCallback(const std::string &deviceId, Message *inMsg) +{ + if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { + if (onRemoteDataChanged_) { + onRemoteDataChanged_(deviceId); + delete inMsg; + inMsg = nullptr; + return E_OK; + } + delete inMsg; + inMsg = nullptr; + return -E_INVALID_ARGS; + } + + LOGD("[GenericVirtualDevice] onMessage, src %s", deviceId.c_str()); + RefObject::IncObjRef(context_); + RefObject::IncObjRef(communicateHandle_); + SyncTaskContext *context = context_; + ICommunicator *communicateHandle = communicateHandle_; + std::thread thread([context, communicateHandle, inMsg]() { + int errCode = context->ReceiveMessageCallback(inMsg); + if (errCode != -E_NOT_NEED_DELETE_MSG) { + delete inMsg; + } + RefObject::DecObjRef(context); + RefObject::DecObjRef(communicateHandle); + }); + thread.detach(); + return E_OK; +} + +void GenericVirtualDevice::OnRemoteDataChanged(const std::function &callback) +{ + onRemoteDataChanged_ = callback; +} + +void GenericVirtualDevice::Online() +{ + static_cast(communicateHandle_)->Enable(); + communicatorAggregator_->OnlineDevice(deviceId_); +} + +void GenericVirtualDevice::Offline() +{ + static_cast(communicateHandle_)->Disable(); + communicatorAggregator_->OfflineDevice(deviceId_); +} + +int GenericVirtualDevice::StartResponseTask() +{ + LOGD("[KvVirtualDevice] StartResponseTask"); + RefObject::AutoLock lockGuard(context_); + int status = context_->GetTaskExecStatus(); + if ((status == SyncTaskContext::RUNNING) || context_->IsKilled()) { + LOGD("[KvVirtualDevice] StartResponseTask status:%d", status); + return -E_NOT_SUPPORT; + } + if (context_->IsTargetQueueEmpty()) { + LOGD("[KvVirtualDevice] StartResponseTask IsTargetQueueEmpty is empty"); + return E_OK; + } + context_->SetTaskExecStatus(ISyncTaskContext::RUNNING); + context_->MoveToNextTarget(); + LOGI("[KvVirtualDevice] machine StartSync"); + context_->UnlockObj(); + int errCode = context_->StartStateMachine(); + context_->LockObj(); + if (errCode != E_OK) { + LOGE("[KvVirtualDevice] machine StartSync failed"); + context_->SetOperationStatus(SyncOperation::OP_FAILED); + } + return errCode; +} + +TimeOffset GenericVirtualDevice::GetLocalTimeOffset() const +{ + return metadata_->GetLocalTimeOffset(); +} + +int GenericVirtualDevice::Sync(SyncMode mode, bool wait) +{ + auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, nullptr, wait); + if (operation == nullptr) { + return -E_OUT_OF_MEMORY; + } + operation->Initialize(); + operation->SetOnSyncFinished([operation](int id) { + operation->NotifyIfNeed(); + }); + context_->AddSyncOperation(operation); + operation->WaitIfNeed(); + RefObject::KillAndDecObjRef(operation); + return E_OK; +} + +int GenericVirtualDevice::Sync(SyncMode mode, const Query &query, bool wait) +{ + auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, nullptr, wait); + if (operation == nullptr) { + return -E_OUT_OF_MEMORY; + } + operation->Initialize(); + operation->SetOnSyncFinished([operation](int id) { + operation->NotifyIfNeed(); + }); + QuerySyncObject querySyncObject(query); + int errCode = querySyncObject.Init(); + if (errCode != E_OK) { + return errCode; + } + operation->SetQuery(querySyncObject); + context_->AddSyncOperation(operation); + operation->WaitIfNeed(); + RefObject::KillAndDecObjRef(operation); + return errCode; +} +} // DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h new file mode 100644 index 0000000000000000000000000000000000000000..3605d2cfbf9418c606fe1da7a5bd20365e081bae --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/generic_virtual_device.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GENERIC_VIRTUAL_DEVICE_H +#define GENERIC_VIRTUAL_DEVICE_H + +#include + +#include "ikvdb_sync_interface.h" +#include "meta_data.h" +#include "subscribe_manager.h" +#include "sync_task_context.h" +#include "types.h" +#include "virtual_communicator_aggregator.h" + +namespace DistributedDB { +class VirtualCommunicatorAggregator; + +class GenericVirtualDevice { +public: + explicit GenericVirtualDevice(std::string deviceId); + virtual ~GenericVirtualDevice(); + + int Initialize(VirtualCommunicatorAggregator *comAggregator, ISyncInterface *syncInterface); + void SetDeviceId(const std::string &deviceId); + std::string GetDeviceId() const; + int MessageCallback(const std::string &deviceId, Message *inMsg); + void OnRemoteDataChanged(const std::function &callback); + void Online(); + void Offline(); + int StartResponseTask(); + TimeOffset GetLocalTimeOffset() const; + virtual int Sync(SyncMode mode, bool wait); + virtual int Sync(SyncMode mode, const Query &query, bool wait); + +protected: + ICommunicator *communicateHandle_; + VirtualCommunicatorAggregator *communicatorAggregator_; + ISyncInterface *storage_; + std::shared_ptr metadata_; + std::string deviceId_; + std::string remoteDeviceId_; + SyncTaskContext *context_; + std::function onRemoteDataChanged_; + + std::shared_ptr subManager_; +}; +} +#endif // GENERIC_VIRTUAL_DEVICE_H diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/kv_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/kv_virtual_device.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1c32515947a2a5c0d98c0093a0e88b966ba6907e --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/kv_virtual_device.cpp @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "kv_virtual_device.h" + +#include "log_print.h" +#include "virtual_multi_ver_sync_db_interface.h" + +namespace DistributedDB { +KvVirtualDevice::KvVirtualDevice(const std::string &deviceId) : GenericVirtualDevice(deviceId) +{ +} + +KvVirtualDevice::~KvVirtualDevice() +{ +} + +int KvVirtualDevice::GetData(const Key &key, VirtualDataItem &item) +{ + VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); + return syncAble->GetSyncData(key, item); +} + +int KvVirtualDevice::GetData(const Key &key, Value &value) +{ + VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); + return syncInterface->GetData(key, value); +} + +int KvVirtualDevice::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) +{ + VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); + LOGI("dev %s put data time %llu", deviceId_.c_str(), time); + return syncAble->PutData(key, value, time, flag); +} + +int KvVirtualDevice::PutData(const Key &key, const Value &value) +{ + VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); + return syncInterface->PutData(key, value); +} + +int KvVirtualDevice::DeleteData(const Key &key) +{ + VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); + return syncInterface->DeleteData(key); +} + +int KvVirtualDevice::StartTransaction() +{ + VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); + return syncInterface->StartTransaction(); +} + +int KvVirtualDevice::Commit() +{ + VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); + return syncInterface->Commit(); +} + + +void KvVirtualDevice::SetSaveDataDelayTime(uint64_t milliDelayTime) +{ + VirtualSingleVerSyncDBInterface *syncInterface = static_cast(storage_); + syncInterface->SetSaveDataDelayTime(milliDelayTime); +} + +int KvVirtualDevice::Subscribe(QuerySyncObject query, bool wait, int id) +{ + auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, SUBSCRIBE_QUERY, nullptr, wait); + if (operation == nullptr) { + return -E_OUT_OF_MEMORY; + } + operation->Initialize(); + operation->SetOnSyncFinished([operation](int id) { + operation->NotifyIfNeed(); + }); + operation->SetQuery(query); + context_->AddSyncOperation(operation); + operation->WaitIfNeed(); + RefObject::KillAndDecObjRef(operation); + return E_OK; +} + +int KvVirtualDevice::UnSubscribe(QuerySyncObject query, bool wait, int id) +{ + auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, UNSUBSCRIBE_QUERY, nullptr, wait); + if (operation == nullptr) { + return -E_OUT_OF_MEMORY; + } + operation->Initialize(); + operation->SetOnSyncFinished([operation](int id) { + operation->NotifyIfNeed(); + }); + operation->SetQuery(query); + context_->AddSyncOperation(operation); + operation->WaitIfNeed(); + RefObject::KillAndDecObjRef(operation); + return E_OK; +} +} // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/kv_virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/kv_virtual_device.h new file mode 100644 index 0000000000000000000000000000000000000000..c9172d74811b5c4b7f838326665451a7dc752d86 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/kv_virtual_device.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef KV_VIRTUAL_DEVICE_H +#define KV_VIRTUAL_DEVICE_H + +#include "generic_virtual_device.h" + +#include "virtual_single_ver_sync_db_Interface.h" +namespace DistributedDB { +class KvVirtualDevice final : public GenericVirtualDevice { +public: + explicit KvVirtualDevice(const std::string &deviceId); + ~KvVirtualDevice() override; + + int GetData(const Key &key, VirtualDataItem &item); + int GetData(const Key &key, Value &value); + int PutData(const Key &key, const Value &value, const TimeStamp &time, int flag); + int PutData(const Key &key, const Value &value); + int DeleteData(const Key &key); + int StartTransaction(); + int Commit(); + void SetSaveDataDelayTime(uint64_t milliDelayTime); + + int Subscribe(QuerySyncObject query, bool wait, int id); + int UnSubscribe(QuerySyncObject query, bool wait, int id); +}; +} // namespace DistributedDB + +#endif // KV_VIRTUAL_DEVICE_H diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_communicator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_communicator.h new file mode 100644 index 0000000000000000000000000000000000000000..52a103ec07191c6c949604d8936b90bd4bf5a5e8 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_communicator.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MOCK_COMMUNICATOR_H +#define MOCK_COMMUNICATOR_H + +#include +#include "icommunicator.h" + +namespace DistributedDB { +class MockCommunicator : public ICommunicator { +public: + MOCK_CONST_METHOD1(GetLocalIdentity, int(std::string &)); + MOCK_CONST_METHOD2(GetRemoteCommunicatorVersion, int(const std::string &, uint16_t &)); + MOCK_METHOD4(SendMessage, int(const std::string &, const Message *, bool, uint32_t)); + MOCK_METHOD5(SendMessage, int(const std::string &, const Message *, bool, uint32_t, const OnSendEnd &)); + MOCK_CONST_METHOD0(GetCommunicatorMtuSize, uint32_t(void)); + MOCK_CONST_METHOD1(GetCommunicatorMtuSize, uint32_t(const std::string &)); + MOCK_CONST_METHOD0(GetTimeout, uint32_t(void)); + MOCK_CONST_METHOD1(GetTimeout, uint32_t(const std::string &)); + MOCK_METHOD2(RegOnConnectCallback, int(const OnConnectCallback &, const Finalizer &)); + MOCK_METHOD2(RegOnSendableCallback, int(const std::function &, const Finalizer &)); + MOCK_METHOD2(RegOnMessageCallback, int(const OnMessageCallback &, const Finalizer &)); + MOCK_METHOD0(Activate, void(void)); + MOCK_CONST_METHOD1(IsDeviceOnline, bool(const std::string &)); +}; +} // namespace DistributedDB +#endif // #define MOCK_COMMUNICATOR_H + diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h new file mode 100644 index 0000000000000000000000000000000000000000..5bd453f601979ee076bf2b2f9c2135c3ba088225 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_data_sync.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MOCK_SINGLE_VER_DATA_SYNC_H +#define MOCK_SINGLE_VER_DATA_SYNC_H + +#include +#include "single_ver_data_sync.h" + +namespace DistributedDB { +class MockSingleVerDataSync : public SingleVerDataSync { +public: + MOCK_METHOD4(SendDataAck, int(SingleVerSyncTaskContext *, const Message *, int32_t, WaterMark)); +}; +} // namespace DistributedDB + +#endif // MOCK_SINGLE_VER_DATA_SYNC_H diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h new file mode 100644 index 0000000000000000000000000000000000000000..23ad7d58a97e917e0510cf5b26ebda1bb965e2d0 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/mock_single_ver_sync_task_context.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H +#define MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H + +#include +#include "single_ver_sync_task_context.h" + +namespace DistributedDB { +class MockSingleVerSyncTaskContext : public SingleVerSyncTaskContext { +public: + MOCK_METHOD1(HandleDataRequestRecv, int(const Message *)); + MOCK_CONST_METHOD0(IsReceiveWaterMarkErr, bool(void)); +}; +} // namespace DistributedDB +#endif // MOCK_SINGLE_VER_SYNC_TASK_CONTEXT_H diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp new file mode 100644 index 0000000000000000000000000000000000000000..16272ff4ef8a1105a68b11505b2d59e083da9447 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "relational_virtual_device.h" +#include "virtual_relational_ver_sync_db_interface.h" +namespace DistributedDB { +RelationalVirtualDevice::RelationalVirtualDevice(const std::string &deviceId) : GenericVirtualDevice(deviceId) +{ +} + +RelationalVirtualDevice::~RelationalVirtualDevice() +{ +} + +int RelationalVirtualDevice::PutData(const std::string &tableName, const std::vector &dataList) +{ + return static_cast(storage_)->PutLocalData(dataList, tableName); +} + +int RelationalVirtualDevice::GetAllSyncData(const std::string &tableName, std::vector &data) +{ + return static_cast(storage_)->GetAllSyncData(tableName, data); +} + +void RelationalVirtualDevice::SetLocalFieldInfo(const std::vector &localFieldInfo) +{ + static_cast(storage_)->SetLocalFieldInfo(localFieldInfo); +} + +int RelationalVirtualDevice::Sync(SyncMode mode, bool wait) +{ + return -E_NOT_SUPPORT; +} +} // DistributedDB +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h new file mode 100644 index 0000000000000000000000000000000000000000..32f974bcd8f8f028ef1da6909a8502c1435c73fb --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/relational_virtual_device.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RELATIONAL_VIRTUAL_DEVICE_H +#define RELATIONAL_VIRTUAL_DEVICE_H +#ifdef RELATIONAL_STORE + +#include "generic_virtual_device.h" +#include "relational_schema_object.h" +#include "data_transformer.h" + +namespace DistributedDB { +class RelationalVirtualDevice final : public GenericVirtualDevice { +public: + explicit RelationalVirtualDevice(const std::string &deviceId); + ~RelationalVirtualDevice() override; + + int PutData(const std::string &tableName, const std::vector &dataList); + int GetAllSyncData(const std::string &tableName, std::vector &data); + void SetLocalFieldInfo(const std::vector &localFieldInfo); + int Sync(SyncMode mode, bool wait) override; +}; +} +#endif +#endif // RELATIONAL_VIRTUAL_DEVICE_H diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp old mode 100755 new mode 100644 similarity index 82% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.cpp rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp index 476816ea0fe19c24fca23386dc0c0c5c854516fb..1f12f5b3c17060c937af5b185f6d08bbb67b7671 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.cpp @@ -13,11 +13,11 @@ * limitations under the License. */ -#include "vitural_communicator.h" +#include "virtual_communicator.h" -#include "vitural_communicator_aggregator.h" -#include "sync_engine.h" #include "log_print.h" +#include "sync_engine.h" +#include "virtual_communicator_aggregator.h" namespace DistributedDB { int VirtualCommunicator::RegOnMessageCallback(const OnMessageCallback &onMessage, const Finalizer &inOper) @@ -88,15 +88,21 @@ void VirtualCommunicator::CallbackOnMessage(const std::string &srcTarget, Messag void VirtualCommunicator::CallbackOnConnect(const std::string &target, bool isConnect) const { + { + std::lock_guard lock(devicesMapLock_); + if (target != deviceId_) { + onlineDevicesMap_[target] = isConnect; + } + } std::lock_guard lock(onConnectLock_); - if (isEnable_ && onConnect_ && (target != deviceId_)) { + if (isEnable_ && onConnect_) { onConnect_(target, isConnect); } } uint32_t VirtualCommunicator::GetCommunicatorMtuSize() const { - return 5 * 1024 * 1024; // 5M + return 5 * 1024 * 1024; // 5 * 1024 * 1024B } uint32_t VirtualCommunicator::GetCommunicatorMtuSize(const std::string &target) const @@ -104,6 +110,16 @@ uint32_t VirtualCommunicator::GetCommunicatorMtuSize(const std::string &target) return GetCommunicatorMtuSize(); } +uint32_t VirtualCommunicator::GetTimeout() const +{ + return 5 * 1000; // 5 * 1000ms +} + +uint32_t VirtualCommunicator::GetTimeout(const std::string &target) const +{ + return GetTimeout(); +} + int VirtualCommunicator::GetLocalIdentity(std::string &outTarget) const { outTarget = deviceId_; @@ -142,6 +158,18 @@ bool VirtualCommunicator::IsEnabled() const return isEnable_; } +bool VirtualCommunicator::IsDeviceOnline(const std::string &device) const +{ + bool res = true; + { + std::lock_guard lock(devicesMapLock_); + if (onlineDevicesMap_.find(device) != onlineDevicesMap_.end()) { + res = onlineDevicesMap_[device]; + } + } + return res; +} + VirtualCommunicator::~VirtualCommunicator() { } diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h similarity index 90% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.h rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h index e192548705cef06d42595b395b2c5dc956789b42..89b6cecbbf37499f7c9d1f1056c2cc15728c4002 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator.h @@ -16,23 +16,21 @@ #ifndef VIRTUAL_COMMUNICATOR_H #define VIRTUAL_COMMUNICATOR_H -#include +#include +#include #include #include +#include #include -#include -#include +#include +#include "icommunicator.h" #include "ref_object.h" #include "serial_buffer.h" -#include "icommunicator.h" -#include "vitural_device.h" namespace DistributedDB { class VirtualCommunicatorAggregator; -class VituralDevice; - class VirtualCommunicator : public ICommunicator { public: VirtualCommunicator(const std::string &deviceId, VirtualCommunicatorAggregator *communicatorAggregator); @@ -48,6 +46,9 @@ public: uint32_t GetCommunicatorMtuSize() const override; uint32_t GetCommunicatorMtuSize(const std::string &target) const override; + + uint32_t GetTimeout() const override; + uint32_t GetTimeout(const std::string &target) const override; int GetLocalIdentity(std::string &outTarget) const override; int SendMessage(const std::string &dstTarget, const Message *inMsg, bool nonBlock, uint32_t timeout) override; @@ -58,7 +59,7 @@ public: void CallbackOnMessage(const std::string &srcTarget, Message *inMsg) const; - void CallbackOnConnect(const std::string &target, bool isConnec) const; + void CallbackOnConnect(const std::string &target, bool isConnect) const; int GeneralVirtualSyncId(); @@ -72,6 +73,8 @@ public: bool IsEnabled() const; + bool IsDeviceOnline(const std::string &device) const override; + private: int TimeSync(); int DataSync(); @@ -82,6 +85,8 @@ private: mutable std::mutex onConnectLock_; OnConnectCallback onConnect_; + mutable std::mutex devicesMapLock_; + mutable std::map onlineDevicesMap_; std::string remoteDeviceId_ = "real_device"; std::mutex syncIdLock_; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp old mode 100755 new mode 100644 similarity index 93% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.cpp rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp index 5884b95b75c3932c2fe703eaa5af5e083815b1ff..3bea5ac2fb44b4b45f486604e48ad426978c104c --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.cpp @@ -12,14 +12,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "vitural_communicator.h" +#include "virtual_communicator_aggregator.h" #include #include +#include #include "db_errno.h" -#include "vitural_communicator_aggregator.h" #include "log_print.h" +#include "runtime_context.h" namespace DistributedDB { int VirtualCommunicatorAggregator::Initialize(IAdapter *inAdapter) @@ -173,12 +174,16 @@ void VirtualCommunicatorAggregator::DispatchMessage(const std::string &srcTarget Message *msg = const_cast(inMsg); msg->SetTarget(srcTarget); RefObject::IncObjRef(communicator); - std::thread thread([communicator, srcTarget, msg]() { + auto onDispatch = onDispatch_; + std::thread thread([communicator, srcTarget, dstTarget, msg, onDispatch]() { + if (onDispatch) { + onDispatch(dstTarget, msg); + } communicator->CallbackOnMessage(srcTarget, msg); RefObject::DecObjRef(communicator); }); thread.detach(); - CallSendEnd(OK, onEnd); + CallSendEnd(E_OK, onEnd); } else { LOGE("[VirtualCommunicatorAggregator] DispatchMessage, can't find dstTarget %s", dstTarget.c_str()); delete inMsg; @@ -220,5 +225,11 @@ void VirtualCommunicatorAggregator::CallSendEnd(int errCode, const OnSendEnd &on }); } } + +void VirtualCommunicatorAggregator::RegOnDispatch( + const std::function &onDispatch) +{ + onDispatch_ = onDispatch; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h similarity index 93% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.h rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h index 3d87d174784ce07c2b187fcbe73668b5f0ff1189..28b92184ded03420a99f98199edd47e6348b3798 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_communicator_aggregator.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_communicator_aggregator.h @@ -19,14 +19,11 @@ #include #include "icommunicator_aggregator.h" -#include "vitural_device.h" -#include "vitural_communicator.h" +#include "virtual_communicator.h" namespace DistributedDB { class ICommunicator; // Forward Declaration -class VituralDevice; - class VirtualCommunicatorAggregator : public ICommunicatorAggregator { public: // Return 0 as success. Return negative as error @@ -67,6 +64,8 @@ public: bool GetBlockValue() const; + void RegOnDispatch(const std::function &onDispatch); + ~VirtualCommunicatorAggregator() {}; VirtualCommunicatorAggregator() {}; @@ -82,6 +81,7 @@ private: bool isBlock_ = false; CommunicatorLackCallback onCommLack_; OnConnectCallback onConnect_; + std::function onDispatch_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp old mode 100755 new mode 100644 similarity index 68% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.cpp rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp index 3029a69a3647d96ac5a2c047df666371b41da5a6..eaa1f7764c710163d2eeaf4967bdfb1a492d3c95 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.cpp @@ -12,30 +12,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "vitural_device.h" +#include "virtual_device.h" #include #include -#include "virtual_single_ver_sync_db_Interface.h" -#include "virtual_multi_ver_sync_db_interface.h" -#include "vitural_communicator.h" -#include "vitural_communicator_aggregator.h" -#include "single_ver_sync_state_machine.h" -#include "multi_ver_sync_state_machine.h" #include "device_manager.h" #include "log_print.h" +#include "multi_ver_sync_state_machine.h" +#include "single_ver_subscribe_manager.h" +#include "single_ver_sync_state_machine.h" +#include "sync_types.h" +#include "virtual_communicator.h" +#include "virtual_communicator_aggregator.h" +#include "virtual_multi_ver_sync_db_interface.h" +#include "virtual_single_ver_sync_db_Interface.h" namespace DistributedDB { -VituralDevice::VituralDevice(const std::string &deviceId) +VirtualDevice::VirtualDevice(const std::string &deviceId) : communicateHandle_(nullptr), + communicatorAggregator_(nullptr), storage_(nullptr), metadata_(nullptr), - deviceId_(deviceId) + deviceId_(deviceId), + remoteDeviceId_("real_device"), + context_(nullptr) { } -VituralDevice::~VituralDevice() +VirtualDevice::~VirtualDevice() { std::mutex cvMutex; std::condition_variable cv; @@ -70,9 +75,10 @@ VituralDevice::~VituralDevice() context_ = nullptr; metadata_ = nullptr; storage_ = nullptr; + subManager_ = nullptr; } -int VituralDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface) +int VirtualDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface) { if ((communicatorAggregator == nullptr) || (syncInterface == nullptr)) { return -E_INVALID_ARGS; @@ -91,77 +97,78 @@ int VituralDevice::Initialize(VirtualCommunicatorAggregator *communicatorAggrega LOGE("metadata_ init failed"); return -E_NOT_SUPPORT; } - if (storage_->GetInterfaceType() == IKvDBSyncInterface::SYNC_SVD) { context_ = new (std::nothrow) SingleVerSyncTaskContext; + subManager_ = std::make_shared(); + static_cast(context_)->SetSubscribeManager(subManager_); } else { context_ = new (std::nothrow) MultiVerSyncTaskContext; } if (context_ == nullptr) { return -E_OUT_OF_MEMORY; } - communicateHandle_->RegOnMessageCallback(std::bind(&VituralDevice::MessageCallback, this, + communicateHandle_->RegOnMessageCallback(std::bind(&VirtualDevice::MessageCallback, this, std::placeholders::_1, std::placeholders::_2), []() {}); context_->Initialize(remoteDeviceId_, storage_, metadata_, communicateHandle_); context_->SetRetryStatus(SyncTaskContext::NO_NEED_RETRY); - context_->RegOnSyncTask(std::bind(&VituralDevice::StartResponseTask, this)); + context_->RegOnSyncTask(std::bind(&VirtualDevice::StartResponseTask, this)); return E_OK; } -void VituralDevice::SetDeviceId(const std::string &deviceId) +void VirtualDevice::SetDeviceId(const std::string &deviceId) { deviceId_ = deviceId; } -std::string VituralDevice::GetDeviceId() const +std::string VirtualDevice::GetDeviceId() const { return deviceId_; } -int VituralDevice::GetData(const Key &key, VirtualDataItem &item) +int VirtualDevice::GetData(const Key &key, VirtualDataItem &item) { VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); return syncAble->GetSyncData(key, item); } -int VituralDevice::GetData(const Key &key, Value &value) +int VirtualDevice::GetData(const Key &key, Value &value) { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->GetData(key, value); } -int VituralDevice::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) +int VirtualDevice::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) { VirtualSingleVerSyncDBInterface *syncAble = static_cast(storage_); LOGI("dev %s put data time %llu", deviceId_.c_str(), time); return syncAble->PutData(key, value, time, flag); } -int VituralDevice::PutData(const Key &key, const Value &value) +int VirtualDevice::PutData(const Key &key, const Value &value) { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->PutData(key, value); } -int VituralDevice::DeleteData(const Key &key) +int VirtualDevice::DeleteData(const Key &key) { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->DeleteData(key); } -int VituralDevice::StartTransaction() +int VirtualDevice::StartTransaction() { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->StartTransaction(); } -int VituralDevice::Commit() +int VirtualDevice::Commit() { VirtualMultiVerSyncDBInterface *syncInterface = static_cast(storage_); return syncInterface->Commit(); } -int VituralDevice::MessageCallback(const std::string &deviceId, Message *inMsg) +int VirtualDevice::MessageCallback(const std::string &deviceId, Message *inMsg) { if (inMsg->GetMessageId() == LOCAL_DATA_CHANGED) { if (onRemoteDataChanged_) { @@ -175,7 +182,7 @@ int VituralDevice::MessageCallback(const std::string &deviceId, Message *inMsg) return -E_INVALID_ARGS; } - LOGD("[VituralDevice] onMessag, src %s", deviceId.c_str()); + LOGD("[VirtualDevice] onMessag, src %s", deviceId.c_str()); RefObject::IncObjRef(context_); RefObject::IncObjRef(communicateHandle_); SyncTaskContext *context = context_; @@ -192,61 +199,61 @@ int VituralDevice::MessageCallback(const std::string &deviceId, Message *inMsg) return E_OK; } -void VituralDevice::OnRemoteDataChanged(const std::function &callback) +void VirtualDevice::OnRemoteDataChanged(const std::function &callback) { onRemoteDataChanged_ = callback; } -void VituralDevice::Online() +void VirtualDevice::Online() { static_cast(communicateHandle_)->Enable(); communicatorAggregator_->OnlineDevice(deviceId_); } -void VituralDevice::Offline() +void VirtualDevice::Offline() { static_cast(communicateHandle_)->Disable(); communicatorAggregator_->OfflineDevice(deviceId_); } -int VituralDevice::StartResponseTask() +int VirtualDevice::StartResponseTask() { - LOGD("[VituralDevice] StartResponseTask"); + LOGD("[VirtualDevice] StartResponseTask"); RefObject::AutoLock lockGuard(context_); int status = context_->GetTaskExecStatus(); if ((status == SyncTaskContext::RUNNING) || context_->IsKilled()) { - LOGD("[VituralDevice] StartResponseTask status:%d", status); + LOGD("[VirtualDevice] StartResponseTask status:%d", status); return -E_NOT_SUPPORT; } if (context_->IsTargetQueueEmpty()) { - LOGD("[VituralDevice] StartResponseTask IsTargetQueueEmpty is empty"); + LOGD("[VirtualDevice] StartResponseTask IsTargetQueueEmpty is empty"); return E_OK; } context_->SetTaskExecStatus(ISyncTaskContext::RUNNING); context_->MoveToNextTarget(); - LOGI("[VituralDevice] machine StartSync"); + LOGI("[VirtualDevice] machine StartSync"); context_->UnlockObj(); int errCode = context_->StartStateMachine(); context_->LockObj(); if (errCode != E_OK) { - LOGE("[VituralDevice] machine StartSync failed"); - context_->SetOperationStatus(SyncOperation::FAILED); + LOGE("[VirtualDevice] machine StartSync failed"); + context_->SetOperationStatus(SyncOperation::OP_FAILED); } return errCode; } -TimeOffset VituralDevice::GetLocalTimeOffset() const +TimeOffset VirtualDevice::GetLocalTimeOffset() const { return metadata_->GetLocalTimeOffset(); } -void VituralDevice::SetSaveDataDelayTime(uint64_t milliDelayTime) +void VirtualDevice::SetSaveDataDelayTime(uint64_t milliDelayTime) { VirtualSingleVerSyncDBInterface *syncInterface = static_cast(storage_); syncInterface->SetSaveDataDelayTime(milliDelayTime); } -int VituralDevice::Sync(SyncMode mode, bool wait) +int VirtualDevice::Sync(SyncMode mode, bool wait) { auto operation = new (std::nothrow) SyncOperation(1, {remoteDeviceId_}, mode, nullptr, wait); if (operation == nullptr) { @@ -255,10 +262,44 @@ int VituralDevice::Sync(SyncMode mode, bool wait) operation->Initialize(); operation->SetOnSyncFinished([operation](int id) { operation->NotifyIfNeed(); - RefObject::KillAndDecObjRef(operation); }); context_->AddSyncOperation(operation); operation->WaitIfNeed(); + RefObject::KillAndDecObjRef(operation); + return E_OK; +} + +int VirtualDevice::Subscribe(QuerySyncObject query, bool wait, int id) +{ + auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, SUBSCRIBE_QUERY, nullptr, wait); + if (operation == nullptr) { + return -E_OUT_OF_MEMORY; + } + operation->Initialize(); + operation->SetOnSyncFinished([operation](int id) { + operation->NotifyIfNeed(); + }); + operation->SetQuery(query); + context_->AddSyncOperation(operation); + operation->WaitIfNeed(); + RefObject::KillAndDecObjRef(operation); + return E_OK; +} + +int VirtualDevice::UnSubscribe(QuerySyncObject query, bool wait, int id) +{ + auto operation = new (std::nothrow) SyncOperation(id, {remoteDeviceId_}, UNSUBSCRIBE_QUERY, nullptr, wait); + if (operation == nullptr) { + return -E_OUT_OF_MEMORY; + } + operation->Initialize(); + operation->SetOnSyncFinished([operation](int id) { + operation->NotifyIfNeed(); + }); + operation->SetQuery(query); + context_->AddSyncOperation(operation); + operation->WaitIfNeed(); + RefObject::KillAndDecObjRef(operation); return E_OK; } } // namespace DistributedDB \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h old mode 100755 new mode 100644 similarity index 83% rename from services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.h rename to services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h index e520a0e74ea554501bb9d34c8393349b550936e2..f6d93ff9916609c1876b90af6a5d22ab95fe441d --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/vitural_device.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_device.h @@ -25,10 +25,10 @@ namespace DistributedDB { class VirtualCommunicatorAggregator; -class VituralDevice { +class VirtualDevice { public: - explicit VituralDevice(const std::string &deviceId); - ~VituralDevice(); + explicit VirtualDevice(const std::string &deviceId); + ~VirtualDevice(); int Initialize(VirtualCommunicatorAggregator *communicatorAggregator, IKvDBSyncInterface *syncInterface); void SetDeviceId(const std::string &deviceId); @@ -48,16 +48,19 @@ public: TimeOffset GetLocalTimeOffset() const; void SetSaveDataDelayTime(uint64_t milliDelayTime); int Sync(SyncMode mode, bool wait); + int Subscribe(QuerySyncObject query, bool wait, int id); + int UnSubscribe(QuerySyncObject query, bool wait, int id); private: ICommunicator *communicateHandle_; - VirtualCommunicatorAggregator *communicatorAggregator_ = nullptr; + VirtualCommunicatorAggregator *communicatorAggregator_; IKvDBSyncInterface *storage_; std::shared_ptr metadata_; std::string deviceId_; - std::string remoteDeviceId_ = "real_device"; - SyncTaskContext *context_ = nullptr; + std::string remoteDeviceId_; + SyncTaskContext *context_; std::function onRemoteDataChanged_; + std::shared_ptr subManager_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp index d9ee109f9e68e528d64ccb06dfad43cead8311f7..fff9546960aaf41b3d1764d9d48dfa26f83d9ba2 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.cpp @@ -64,6 +64,11 @@ int VirtualMultiVerSyncDBInterface::PutMetaData(const Key &key, const Value &val return kvStore_->PutMetaData(key, value); } +int VirtualMultiVerSyncDBInterface::DeleteMetaData(const std::vector &keys) +{ + return kvStore_->DeleteMetaData(keys); +} + int VirtualMultiVerSyncDBInterface::GetAllMetaKeys(std::vector &keys) const { return kvStore_->GetAllMetaKeys(keys); @@ -224,5 +229,10 @@ const KvDBProperties &VirtualMultiVerSyncDBInterface::GetDbProperties() const { return properties_; } + +int VirtualMultiVerSyncDBInterface::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + return -E_NOT_SUPPORT; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.h index 17695c480347ade77c1a32db0ed078477c01add6..113684c2f9a9e6660f679965b75f5ece551caf74 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_multi_ver_sync_db_interface.h @@ -16,10 +16,9 @@ #ifndef VIRTUAL_MULTI_VER_SYNC_INTERFACE_H #define VIRTUAL_MULTI_VER_SYNC_INTERFACE_H +#include "distributeddb_tools_unit_test.h" #include "multi_ver_natural_store.h" - #include "multi_ver_natural_store_connection.h" -#include "distributeddb_tools_unit_test.h" namespace DistributedDB { class VirtualMultiVerSyncDBInterface final : public MultiVerKvDBSyncInterface { @@ -41,6 +40,12 @@ public: int PutMetaData(const Key &key, const Value &value) override; + // Delete multiple meta data records in a transaction. + int DeleteMetaData(const std::vector &keys) override; + + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + int GetAllMetaKeys(std::vector &keys) const override; bool IsCommitExisted(const MultiVerCommitNode &) const override; diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp new file mode 100644 index 0000000000000000000000000000000000000000..abad85f7abfc2cd5b2998650ac3c723b7cf9ecd9 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.cpp @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef RELATIONAL_STORE +#include "virtual_relational_ver_sync_db_interface.h" +#include "generic_single_ver_kv_entry.h" +#include "virtual_single_ver_sync_db_Interface.h" + +namespace DistributedDB { +namespace { + int GetEntriesFromItems(std::vector &entries, const std::vector &dataItems) + { + int errCode = E_OK; + for (auto &item : dataItems) { + auto entry = new (std::nothrow) GenericSingleVerKvEntry(); + if (entry == nullptr) { + LOGE("Create entry failed."); + errCode = -E_OUT_OF_MEMORY; + break; + } + DataItem storageItem; + storageItem.key = item.key; + storageItem.value = item.value; + storageItem.flag = item.flag; + storageItem.timeStamp = item.timeStamp; + storageItem.writeTimeStamp = item.writeTimeStamp; + storageItem.hashKey = item.hashKey; + entry->SetEntryData(std::move(storageItem)); + entries.push_back(entry); + } + if (errCode != E_OK) { + LOGD("[GetEntriesFromItems] failed:%d", errCode); + for (auto &kvEntry : entries) { + delete kvEntry; + kvEntry = nullptr; + } + entries.clear(); + } + LOGD("[GetEntriesFromItems] size:%d", dataItems.size()); + return errCode; + } +} + +int VirtualRelationalVerSyncDBInterface::PutSyncDataWithQuery(const QueryObject &object, + const std::vector &entries, const std::string &deviceName) +{ + LOGD("[PutSyncData] size %d", entries.size()); + std::vector dataItems; + for (auto itemEntry : entries) { + auto *entry = static_cast(itemEntry); + if (entry != nullptr) { + DataItem item; + item.origDev = entry->GetOrigDevice(); + item.flag = entry->GetFlag(); + item.timeStamp = entry->GetTimestamp(); + item.writeTimeStamp = entry->GetWriteTimestamp(); + entry->GetKey(item.key); + entry->GetValue(item.value); + entry->GetHashKey(item.hashKey); + dataItems.push_back(item); + } + } + OptTableDataWithLog optTableDataWithLog; + optTableDataWithLog.tableName = object.GetTableName(); + int errCode = DataTransformer::TransformDataItem(dataItems, localFieldInfo_, + localFieldInfo_, optTableDataWithLog); + if (errCode != E_OK) { + return errCode; + } + std::vector dataList; + std::set hashKeySet; + for (const auto &optRowDataWithLog : optTableDataWithLog.dataList) { + RowDataWithLog rowDataWithLog; + rowDataWithLog.logInfo = optRowDataWithLog.logInfo; + for (const auto &optItem : optRowDataWithLog.optionalData) { + if (!optItem.has_value()) { + continue; + } + rowDataWithLog.rowData.push_back(optItem.value()); + LOGD("type:%d", optItem.value().GetType()); + } + hashKeySet.insert(rowDataWithLog.logInfo.hashKey); + dataList.push_back(rowDataWithLog); + } + LOGD("tableName %s", optTableDataWithLog.tableName.c_str()); + for (const auto &item : syncData_[optTableDataWithLog.tableName]) { + if (hashKeySet.find(item.logInfo.hashKey) == hashKeySet.end()) { + dataList.push_back(item); + } + } + syncData_[optTableDataWithLog.tableName] = dataList; + + return errCode; +} + +int VirtualRelationalVerSyncDBInterface::PutLocalData(const std::vector &dataList, + const std::string &tableName) +{ + for (const auto &item : dataList) { + localData_[tableName].push_back(item); + } + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetSyncData(QueryObject &query, + const SyncTimeRange &timeRange, const DataSizeSpecInfo &dataSizeInfo, + ContinueToken &continueStmtToken, std::vector &entries) const +{ + if (localData_.find(query.GetTableName()) == localData_.end()) { + LOGD("[GetSyncData] No Data Return"); + return E_OK; + } + std::vector dataItemList; + TableDataWithLog tableDataWithLog = {query.GetTableName(), {}}; + for (const auto &data : localData_[query.GetTableName()]) { + if (data.logInfo.timestamp >= timeRange.beginTime && data.logInfo.timestamp < timeRange.endTime) { + tableDataWithLog.dataList.push_back(data); + } + } + int errCode = DataTransformer::TransformTableData(tableDataWithLog, localFieldInfo_, dataItemList); + if (errCode != E_OK) { + return errCode; + } + continueStmtToken = nullptr; + return GetEntriesFromItems(entries, dataItemList); +} + +RelationalSchemaObject VirtualRelationalVerSyncDBInterface::GetSchemaInfo() const +{ + return schemaObj_; +} + +int VirtualRelationalVerSyncDBInterface::GetDatabaseCreateTimeStamp(TimeStamp &outTime) const +{ + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetBatchMetaData(const std::vector &keys, + std::vector &entries) const +{ + int errCode = E_OK; + for (const auto &key : keys) { + Entry entry; + entry.key = key; + errCode = GetMetaData(key, entry.value); + if (errCode != E_OK) { + return errCode; + } + entries.push_back(entry); + } + return errCode; +} + +int VirtualRelationalVerSyncDBInterface::PutBatchMetaData(std::vector &entries) +{ + int errCode = E_OK; + for (const auto &entry : entries) { + errCode = PutMetaData(entry.key, entry.value); + if (errCode != E_OK) { + return errCode; + } + } + return errCode; +} + +std::vector VirtualRelationalVerSyncDBInterface::GetTablesQuery() +{ + return {}; +} + +int VirtualRelationalVerSyncDBInterface::LocalDataChanged(int notifyEvent, std::vector &queryObj) +{ + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::SchemaChanged(int notifyEvent) +{ + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetInterfaceType() const +{ + return SYNC_RELATION; +} + +void VirtualRelationalVerSyncDBInterface::IncRefCount() +{ +} + +void VirtualRelationalVerSyncDBInterface::DecRefCount() +{ +} + +std::vector VirtualRelationalVerSyncDBInterface::GetIdentifier() const +{ + return {}; +} + +void VirtualRelationalVerSyncDBInterface::GetMaxTimeStamp(TimeStamp &stamp) const +{ + for (const auto &item : syncData_) { + for (const auto &rowDataWithLog : item.second) { + if (stamp < rowDataWithLog.logInfo.timestamp) { + stamp = rowDataWithLog.logInfo.timestamp; + } + } + } + LOGD("VirtualSingleVerSyncDBInterface::GetMaxTimeStamp time = %llu", stamp); +} + +int VirtualRelationalVerSyncDBInterface::GetMetaData(const Key &key, Value &value) const +{ + auto iter = metadata_.find(key); + if (iter != metadata_.end()) { + value = iter->second; + return E_OK; + } + return -E_NOT_FOUND; +} + +int VirtualRelationalVerSyncDBInterface::PutMetaData(const Key &key, const Value &value) +{ + metadata_[key] = value; + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::DeleteMetaData(const std::vector &keys) +{ + for (const auto &key : keys) { + (void)metadata_.erase(key); + } + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + size_t prefixKeySize = keyPrefix.size(); + for (auto iter = metadata_.begin();iter != metadata_.end();) { + if (prefixKeySize <= iter->first.size() && + keyPrefix == Key(iter->first.begin(), std::next(iter->first.begin(), prefixKeySize))) { + iter = metadata_.erase(iter); + } else { + ++iter; + } + } + return E_OK; +} + +int VirtualRelationalVerSyncDBInterface::GetAllMetaKeys(std::vector &keys) const +{ + for (auto &iter : metadata_) { + keys.push_back(iter.first); + } + LOGD("GetAllMetaKeys size %d", keys.size()); + return E_OK; +} + +const KvDBProperties &VirtualRelationalVerSyncDBInterface::GetDbProperties() const +{ + return properties_; +} + +void VirtualRelationalVerSyncDBInterface::SetLocalFieldInfo(const std::vector &localFieldInfo) +{ + // sort by dict + std::map infoMap; + for (const auto &item : localFieldInfo) { + infoMap[item.GetFieldName()] = item; + } + for (const auto &item : infoMap) { + localFieldInfo_.push_back(item.second); + } +} + +int VirtualRelationalVerSyncDBInterface::GetAllSyncData(const std::string &tableName, + std::vector &data) +{ + if (syncData_.find(tableName) == syncData_.end()) { + return -E_NOT_FOUND; + } + data = syncData_[tableName]; + return E_OK; +} +} +#endif \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h new file mode 100644 index 0000000000000000000000000000000000000000..a95fd5c8370ded650cc8276c2b20881fe32984f8 --- /dev/null +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_relational_ver_sync_db_interface.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef VIRTUAL_RELATIONAL_VER_SYNC_DB_INTERFACE_H +#define VIRTUAL_RELATIONAL_VER_SYNC_DB_INTERFACE_H +#ifdef RELATIONAL_STORE + +#include "data_transformer.h" +#include "relational_db_sync_interface.h" +#include "sqlite_single_ver_continue_token.h" +#include "relational_schema_object.h" + +namespace DistributedDB { +class VirtualRelationalVerSyncDBInterface : public RelationalDBSyncInterface { +public: + VirtualRelationalVerSyncDBInterface() = default; + ~VirtualRelationalVerSyncDBInterface() override = default; + + int PutSyncDataWithQuery(const QueryObject &query, const std::vector &entries, + const std::string &deviceName) override; + + int PutLocalData(const std::vector &dataList, const std::string &tableName); + + RelationalSchemaObject GetSchemaInfo() const override; + + int GetDatabaseCreateTimeStamp(TimeStamp &outTime) const override; + + int GetBatchMetaData(const std::vector &keys, std::vector &entries) const override; + + int PutBatchMetaData(std::vector &entries) override; + + std::vector GetTablesQuery() override; + + int LocalDataChanged(int notifyEvent, std::vector &queryObj) override; + + int SchemaChanged(int notifyEvent) override; + + int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const override; + + int GetInterfaceType() const override; + + void IncRefCount() override; + + void DecRefCount() override; + + std::vector GetIdentifier() const override; + + void GetMaxTimeStamp(TimeStamp &stamp) const override; + + int GetMetaData(const Key &key, Value &value) const override; + + int PutMetaData(const Key &key, const Value &value) override; + + int DeleteMetaData(const std::vector &keys) override; + + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + + int GetAllMetaKeys(std::vector &keys) const override; + + const KvDBProperties &GetDbProperties() const override; + + void SetLocalFieldInfo(const std::vector &localFieldInfo); + + int GetAllSyncData(const std::string &tableName, std::vector &data); + + int InterceptData(std::vector &entries, + const std::string &sourceID, const std::string &targetID) const override + { + return E_OK; + } + + int CheckAndInitQueryCondition(QueryObject &query) const override + { + return E_OK; + } + +private: + + mutable std::map, std::vector> metadata_; + std::map> syncData_; + mutable std::map> localData_; + std::string schema_; + RelationalSchemaObject schemaObj_; + std::vector localFieldInfo_; + KvDBProperties properties_; + SecurityOption secOption_; +}; +} +#endif +#endif // VIRTUAL_RELATIONAL_VER_SYNC_DB_INTERFACE_H \ No newline at end of file diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp index fd500bb879f64c54ee1a51d7c4cfbda3e3b41efd..37cc0e8c64f48210af751cb48e285b7a26b81a7f 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.cpp @@ -15,17 +15,48 @@ #include "virtual_single_ver_sync_db_Interface.h" -#include #include #include #include "db_errno.h" +#include "generic_single_ver_kv_entry.h" +#include "intercepted_data_impl.h" #include "log_print.h" #include "meta_data.h" +#include "query_object.h" #include "securec.h" -#include "generic_single_ver_kv_entry.h" namespace DistributedDB { +namespace { + int GetEntriesFromItems(std::vector &entries, const std::vector &dataItems) + { + int errCode = E_OK; + for (auto &item : dataItems) { + auto entry = new (std::nothrow) GenericSingleVerKvEntry(); + if (entry == nullptr) { + LOGE("Create entry failed."); + errCode = -E_OUT_OF_MEMORY; + break; + } + DataItem storageItem; + storageItem.key = item.key; + storageItem.value = item.value; + storageItem.flag = item.flag; + storageItem.timeStamp = item.timeStamp; + storageItem.writeTimeStamp = item.writeTimeStamp; + entry->SetEntryData(std::move(storageItem)); + entries.push_back(entry); + } + if (errCode != E_OK) { + for (auto &kvEntry : entries) { + delete kvEntry; + kvEntry = nullptr; + } + entries.clear(); + } + return errCode; + } +} int VirtualSingleVerSyncDBInterface::GetInterfaceType() const { return SYNC_SVD; @@ -61,6 +92,14 @@ int VirtualSingleVerSyncDBInterface::PutMetaData(const Key &key, const Value &va return E_OK; } +int VirtualSingleVerSyncDBInterface::DeleteMetaData(const std::vector &keys) +{ + for (const auto &key : keys) { + (void)metadata_.erase(key); + } + return E_OK; +} + int VirtualSingleVerSyncDBInterface::GetAllMetaKeys(std::vector &keys) const { for (auto iter = metadata_.begin(); iter != metadata_.end(); ++iter) { @@ -87,12 +126,6 @@ void VirtualSingleVerSyncDBInterface::ReleaseContinueToken(ContinueToken& contin return; } -int VirtualSingleVerSyncDBInterface::PutSyncData(std::vector& dataItems, - const std::string &deviceName) -{ - return -E_NOT_SUPPORT; -} - SchemaObject VirtualSingleVerSyncDBInterface::GetSchemaInfo() const { return schemaObj_; @@ -106,12 +139,6 @@ bool VirtualSingleVerSyncDBInterface::CheckCompatible(const std::string& schema) return (schemaObj_.CompareAgainstSchemaString(schema) == -E_SCHEMA_EQUAL_EXACTLY); } -void VirtualSingleVerSyncDBInterface::ReleaseKvEntry(const SingleVerKvEntry *entry) -{ - delete entry; - entry = nullptr; -} - int VirtualSingleVerSyncDBInterface::PutData(const Key &key, const Value &value, const TimeStamp &time, int flag) { VirtualDataItem item; @@ -140,17 +167,17 @@ int VirtualSingleVerSyncDBInterface::RemoveDeviceData(const std::string &deviceN return E_OK; } -int VirtualSingleVerSyncDBInterface::GetSyncData(const Key &key, VirtualDataItem &item) +int VirtualSingleVerSyncDBInterface::GetSyncData(const Key &key, VirtualDataItem &dataItem) { auto iter = std::find_if(dbData_.begin(), dbData_.end(), - [key](VirtualDataItem item) { return item.key == key; }); + [key](const VirtualDataItem& item) { return item.key == key; }); if (iter != dbData_.end()) { - item.key = iter->key; - item.value = iter->value; - item.timeStamp = iter->timeStamp; - item.writeTimeStamp = iter->writeTimeStamp; - item.flag = iter->flag; - item.isLocal = iter->isLocal; + dataItem.key = iter->key; + dataItem.value = iter->value; + dataItem.timeStamp = iter->timeStamp; + dataItem.writeTimeStamp = iter->writeTimeStamp; + dataItem.flag = iter->flag; + dataItem.isLocal = iter->isLocal; return E_OK; } return -E_NOT_FOUND; @@ -166,30 +193,7 @@ int VirtualSingleVerSyncDBInterface::GetSyncData(TimeStamp begin, TimeStamp end, LOGE("[VirtualSingleVerSyncDBInterface][GetSyncData] GetSyncData failed err %d", errCode); return errCode; } - for (auto item : dataItems) { - GenericSingleVerKvEntry *entry = new (std::nothrow) GenericSingleVerKvEntry(); - if (entry == nullptr) { - LOGE("Create entry failed."); - errCode = -E_OUT_OF_MEMORY; - break; - } - DataItem storageItem; - storageItem.key = item.key; - storageItem.value = item.value; - storageItem.flag = item.flag; - storageItem.timeStamp = item.timeStamp; - storageItem.writeTimeStamp = item.writeTimeStamp; - entry->SetEntryData(std::move(storageItem)); - entries.push_back(entry); - } - if (errCode != E_OK) { - for (auto kvEntry : entries) { - delete kvEntry; - kvEntry = nullptr; - } - entries.clear(); - } - return errCode; + return GetEntriesFromItems(entries, dataItems); } int VirtualSingleVerSyncDBInterface::GetSyncDataNext(std::vector &entries, @@ -201,25 +205,6 @@ int VirtualSingleVerSyncDBInterface::GetSyncDataNext(std::vector &entries, - const std::string &deviceName) -{ - std::this_thread::sleep_for(std::chrono::milliseconds(saveDataDelayTime_)); - std::vector dataItems; - for (auto kvEntry : entries) { - auto genricKvEntry = static_cast(kvEntry); - VirtualDataItem item; - genricKvEntry->GetKey(item.key); - genricKvEntry->GetValue(item.value); - item.timeStamp = genricKvEntry->GetTimestamp(); - item.writeTimeStamp = genricKvEntry->GetWriteTimestamp(); - item.flag = genricKvEntry->GetFlag(); - item.isLocal = false; - dataItems.push_back(item); - } - return PutSyncData(dataItems, deviceName); -} - int VirtualSingleVerSyncDBInterface::GetSyncData(TimeStamp begin, TimeStamp end, uint32_t blockSize, std::vector &dataItems, ContinueToken &continueStmtToken) const { @@ -312,4 +297,115 @@ void VirtualSingleVerSyncDBInterface::SetSecurityOption(SecurityOption &option) void VirtualSingleVerSyncDBInterface::NotifyRemotePushFinished(const std::string &targetId) const { } + +int VirtualSingleVerSyncDBInterface::GetDatabaseCreateTimeStamp(TimeStamp &outTime) const +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, + const DataSizeSpecInfo &dataSizeInfo, ContinueToken &continueStmtToken, + std::vector &entries) const +{ + const auto &startKey = query.GetPrefixKey(); + Key endKey = startKey; + endKey.resize(DBConstant::MAX_KEY_SIZE, UCHAR_MAX); + + std::vector dataItems; + for (const auto &data : dbData_) { + // Only get local data. + if (!data.isLocal) { + continue; + } + + if ((data.flag & VirtualDataItem::DELETE_FLAG) != 0) { + if (data.timeStamp >= timeRange.deleteBeginTime && data.timeStamp < timeRange.deleteEndTime) { + dataItems.push_back(data); + } + } else { + if (data.timeStamp >= timeRange.beginTime && data.timeStamp < timeRange.endTime && + data.key >= startKey && data.key <= endKey) { + dataItems.push_back(data); + } + } + } + + LOGD("dataItems size %d", dataItems.size()); + return GetEntriesFromItems(entries, dataItems); +} + +int VirtualSingleVerSyncDBInterface::DeleteMetaDataByPrefixKey(const Key &keyPrefix) const +{ + size_t prefixKeySize = keyPrefix.size(); + for (auto iter = metadata_.begin(); iter != metadata_.end();) { + if (prefixKeySize <= iter->first.size() && + keyPrefix == Key(iter->first.begin(), std::next(iter->first.begin(), prefixKeySize))) { + iter = metadata_.erase(iter); + } else { + ++iter; + } + } + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::GetCompressionAlgo(std::set &algorithmSet) const +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::PutSyncData(const DataItem &item) +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::CheckAndInitQueryCondition(QueryObject &query) const +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::InterceptData(std::vector &entries, + const std::string &sourceID, const std::string &targetID) const +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::PutSyncDataWithQuery(const QueryObject &query, + const std::vector &entries, const std::string &deviceName) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(saveDataDelayTime_)); + std::vector dataItems; + for (auto kvEntry : entries) { + auto genericKvEntry = static_cast(kvEntry); + VirtualDataItem item; + genericKvEntry->GetKey(item.key); + genericKvEntry->GetValue(item.value); + item.timeStamp = genericKvEntry->GetTimestamp(); + item.writeTimeStamp = genericKvEntry->GetWriteTimestamp(); + item.flag = genericKvEntry->GetFlag(); + item.isLocal = false; + dataItems.push_back(item); + } + return PutSyncData(dataItems, deviceName); +} + +int VirtualSingleVerSyncDBInterface::AddSubscribe(const std::string &subscribeId, const QueryObject &query, + bool needCacheSubscribe) +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::RemoveSubscribe(const std::string &subscribeId) +{ + return E_OK; +} + +int VirtualSingleVerSyncDBInterface::RemoveSubscribe(const std::vector &subscribeIds) +{ + return E_OK; +} } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.h index 20b5215ec600ee179e6954e48f32dfe94ecf55bb..6120cef499791832dd7bd173d6e1ce258898e13e 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_single_ver_sync_db_Interface.h @@ -20,6 +20,7 @@ #include #include "single_ver_kvdb_sync_interface.h" +#include "query_object.h" #include "types.h" namespace DistributedDB { @@ -47,6 +48,10 @@ public: int PutMetaData(const Key& key, const Value& value) override; + int DeleteMetaData(const std::vector &keys) override; + // Delete multiple meta data records with key prefix in a transaction. + int DeleteMetaDataByPrefixKey(const Key &keyPrefix) const override; + int GetAllMetaKeys(std::vector& keys) const override; int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &dataItems, @@ -57,15 +62,11 @@ public: void ReleaseContinueToken(ContinueToken& continueStmtToken) const override; - int PutSyncData(std::vector& dataItems, const std::string &deviceName) override; - - void ReleaseKvEntry(const SingleVerKvEntry *entry) override; - void GetMaxTimeStamp(TimeStamp& stamp) const override; int RemoveDeviceData(const std::string &deviceName, bool isNeedNotify) override; - int GetSyncData(const Key& key, VirtualDataItem& item); + int GetSyncData(const Key& key, VirtualDataItem& dataItem); int PutSyncData(const DataItem& item); @@ -74,10 +75,14 @@ public: int GetSyncData(TimeStamp begin, TimeStamp end, std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; + int GetSyncData(QueryObject &query, const SyncTimeRange &timeRange, const DataSizeSpecInfo &dataSizeInfo, + ContinueToken &continueStmtToken, std::vector &entries) const override; + int GetSyncDataNext(std::vector &entries, ContinueToken &continueStmtToken, const DataSizeSpecInfo &dataSizeInfo) const override; - int PutSyncData(const std::vector &entries, const std::string &deviceName) override; + int PutSyncDataWithQuery(const QueryObject &query, const std::vector &entries, + const std::string &deviceName) override; SchemaObject GetSchemaInfo() const override; @@ -97,6 +102,22 @@ public: void NotifyRemotePushFinished(const std::string &targetId) const override; + int GetDatabaseCreateTimeStamp(TimeStamp &outTime) const override; + + int GetCompressionOption(bool &needCompressOnSync, uint8_t &compressionRate) const override; + int GetCompressionAlgo(std::set &algorithmSet) const override; + + // return E_OK if subscribe is legal, ERROR on exception. + int CheckAndInitQueryCondition(QueryObject &query) const override; + + int InterceptData(std::vector &entries, const std::string &sourceID, + const std::string &targetID) const override; + + int AddSubscribe(const std::string &subscribeId, const QueryObject &query, bool needCacheSubscribe) override; + + int RemoveSubscribe(const std::string &subscribeId) override; + + int RemoveSubscribe(const std::vector &subscribeIds) override; private: int GetSyncData(TimeStamp begin, TimeStamp end, uint32_t blockSize, std::vector& dataItems, ContinueToken& continueStmtToken) const; @@ -106,12 +127,12 @@ private: int PutSyncData(std::vector& dataItems, const std::string &deviceName); - std::map, std::vector> metadata_; + mutable std::map, std::vector> metadata_; std::vector dbData_; std::string schema_; SchemaObject schemaObj_; KvDBProperties properties_; - uint64_t saveDataDelayTime_; + uint64_t saveDataDelayTime_ = 0; SecurityOption secOption_; }; } // namespace DistributedDB diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp index 21d10f4455bfb2e30230aec8833443ce62b2590e..37fe299b6c138d164abcbe05a93238145a5230d3 100755 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.cpp @@ -61,6 +61,21 @@ uint32_t VirtualTimeSyncCommunicator::GetCommunicatorMtuSize(const std::string & return GetCommunicatorMtuSize(); } +uint32_t VirtualTimeSyncCommunicator::GetTimeout() const +{ + return 0; +} + +uint32_t VirtualTimeSyncCommunicator::GetTimeout(const std::string &target) const +{ + return 0; +} + +bool VirtualTimeSyncCommunicator::IsDeviceOnline(const std::string &device) const +{ + return true; +} + // Get local target name for identify self int VirtualTimeSyncCommunicator::GetLocalIdentity(std::string &outTarget) const { diff --git a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h index ccc6bd7c18be0404066b39d9dd4afd7e9e2044d3..cc7c46ac3631493656ee4de61aa018378484929f 100644 --- a/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h +++ b/services/distributeddataservice/libs/distributeddb/test/unittest/common/syncer/virtual_time_sync_communicator.h @@ -17,18 +17,17 @@ #define VIRTUAL_TIME_SYNC_COMMUNICATOR_H #include +#include +#include #include #include #include -#include -#include #include "db_types.h" +#include "communicator_aggregator.h" +#include "icommunicator.h" #include "ref_object.h" #include "serial_buffer.h" -#include "icommunicator.h" -#include "communicator_aggregator.h" -#include "vitural_device.h" #include "time_sync.h" namespace DistributedDB { @@ -48,6 +47,13 @@ public: // return maximum allowed data size uint32_t GetCommunicatorMtuSize() const override; uint32_t GetCommunicatorMtuSize(const std::string &target) const override; + + // return timeout + uint32_t GetTimeout() const override; + uint32_t GetTimeout(const std::string &target) const override; + + bool IsDeviceOnline(const std::string &device) const override; + // Get local target name for identify self int GetLocalIdentity(std::string &outTarget) const override; diff --git a/services/distributeddataservice/test/BUILD.gn b/services/distributeddataservice/test/BUILD.gn index c3cfb0f157a62dc3db494edf54a7921886d3aba9..c5c927dc27e7b755c1d1020ed71e8f3977a5700a 100755 --- a/services/distributeddataservice/test/BUILD.gn +++ b/services/distributeddataservice/test/BUILD.gn @@ -14,6 +14,15 @@ import("//build/test.gni") module_output_path = "distributeddatamgr/distributeddb" +mul_sources = [ + "common/distributeddb/src/auto_launch_callback.cpp", + "common/distributeddb/src/delegate_callback.cpp", + "common/distributeddb/src/delegate_kv_mgr_callback.cpp", + "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/kv_store_observer_impl.cpp", + "common/distributeddb/src/kv_store_snapshot_callback.cpp", +] + ############################################################################### config("module_private_config") { visibility = [ ":*" ] @@ -47,6 +56,8 @@ config("module_private_config") { "TESTCASES_USING_GTEST_EXT", "OMIT_JSON", "LOW_LEVEL_MEM_DEV", + "RELEASE_MODE_V2", + "RELEASE_MODE_V3", ] ldflags = [ "-Wl,--exclude-libs,ALL" ] } @@ -57,19 +68,19 @@ ohos_moduletest("DistributeddbKvTransactionTest") { sources = [ "common/distributeddb/src/distributed_test_sysinfo.cpp", - "common/distributeddb/src/distributed_test_tools.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributed_crud_transaction_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_transaction_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -79,19 +90,19 @@ ohos_moduletest("DistributeddbKvTransactionPerfTest") { sources = [ "common/distributeddb/src/distributed_test_sysinfo.cpp", - "common/distributeddb/src/distributed_test_tools.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributed_crud_transaction_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_transaction_perf_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -100,18 +111,18 @@ ohos_moduletest("DistributeddbKvConcurrencyCrudTest") { sources = [ "common/distributeddb/src/distributed_test_sysinfo.cpp", - "common/distributeddb/src/distributed_test_tools.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_concurrency_crud_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -120,18 +131,18 @@ ohos_moduletest("DistributeddbKvBatchCrudTest") { sources = [ "common/distributeddb/src/distributed_test_sysinfo.cpp", - "common/distributeddb/src/distributed_test_tools.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_batch_crud_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -140,18 +151,18 @@ ohos_moduletest("DistributeddbKvCreateTest") { sources = [ "common/distributeddb/src/distributed_test_sysinfo.cpp", - "common/distributeddb/src/distributed_test_tools.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_create_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -160,18 +171,18 @@ ohos_moduletest("DistributeddbKvCrudTest") { sources = [ "common/distributeddb/src/distributed_test_sysinfo.cpp", - "common/distributeddb/src/distributed_test_tools.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_crud_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -180,18 +191,18 @@ ohos_moduletest("DistributeddbKvObserverTest") { sources = [ "common/distributeddb/src/distributed_test_sysinfo.cpp", - "common/distributeddb/src/distributed_test_tools.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_observer_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -200,18 +211,18 @@ ohos_moduletest("DistributeddbKvObserverSnapTest") { sources = [ "common/distributeddb/src/distributed_test_sysinfo.cpp", - "common/distributeddb/src/distributed_test_tools.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_observer_snap_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -219,17 +230,18 @@ ohos_moduletest("DistributeddbKvObserverSnapTest") { ohos_moduletest("DistributeddbKvBackupTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_backup_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -237,18 +249,19 @@ ohos_moduletest("DistributeddbKvBackupTest") { ohos_moduletest("DistributeddbKvRealdelTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "moduletest/common/distributeddb/src/distributeddb_kv_realdel_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -257,19 +270,20 @@ ohos_moduletest("DistributeddbNbCreateTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_create_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -277,19 +291,20 @@ ohos_moduletest("DistributeddbNbCrudTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_crud_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -297,19 +312,20 @@ ohos_moduletest("DistributeddbNbObserverTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_observer_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -318,20 +334,21 @@ ohos_moduletest("DistributeddbNbCursorTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_cursor_test.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_cursor_testcase.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -340,19 +357,20 @@ ohos_moduletest("DistributeddbNbBackupTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_backup_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] @@ -362,19 +380,20 @@ ohos_moduletest("DistributeddbNbBatchCrudTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_batch_crud_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -383,19 +402,20 @@ ohos_moduletest("DistributeddbNbLocalBatchCrudTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_local_batch_crud_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -404,18 +424,19 @@ ohos_moduletest("DistributeddbNbSchemaDbTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_schema_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", - "//third_party/openssl:libcrypto_static", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -424,59 +445,152 @@ ohos_moduletest("DistributeddbNbPredicateQueryTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "moduletest/common/distributeddb/src/distributeddb_nb_predicate_query_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", + "//third_party/sqlite:sqlite", + "//utils/native/base:utils", + ] + external_deps = [ "hiviewdfx_hilog_native:libhilog" ] +} + +ohos_moduletest("DistributeddbNbPredicateQueryExpandTest") { + module_out_path = module_output_path + sources = [ + "common/distributeddb/src/distributeddb_constant.cpp", + "common/distributeddb/src/distributeddb_data_generator.cpp", + "common/distributeddb/src/distributeddb_nb_test_tools.cpp", + "common/distributeddb/src/distributeddb_schema_test_tools.cpp", + "moduletest/common/distributeddb/src/distributeddb_nb_predicate_query_expand_test.cpp", + ] + sources += mul_sources + configs = [ ":module_private_config" ] + + deps = [ + "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", + "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", + ] + external_deps = [ "hiviewdfx_hilog_native:libhilog" ] +} + +ohos_moduletest("DistributeddbNbAutolaunchTest") { + module_out_path = module_output_path + sources = [ + "common/distributeddb/src/distributeddb_constant.cpp", + "common/distributeddb/src/distributeddb_data_generator.cpp", + "common/distributeddb/src/distributeddb_nb_test_tools.cpp", + "common/distributeddb/src/distributeddb_schema_test_tools.cpp", + "moduletest/common/distributeddb/src/distributeddb_nb_autolaunch_test.cpp", + ] + sources += mul_sources + configs = [ ":module_private_config" ] + + deps = [ + "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", + "//third_party/googletest:gtest_main", "//third_party/openssl:libcrypto_static", + "//third_party/sqlite:sqlite", + "//utils/native/base:utils", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } -ohos_moduletest("DistributeddbNbEnableSyncByClosedDbTest") { +ohos_moduletest("DistributedbNbDbDamageTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", - "moduletest/common/distributeddb/src/distributeddb_nb_enable_sync_by_closed_db_test.cpp", + "common/distributeddb/src/distributeddb_schema_test_tools.cpp", + "moduletest/common/distributeddb/src/distributeddb_nb_db_damage_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] + deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", + ] + external_deps = [ "hiviewdfx_hilog_native:libhilog" ] +} + +ohos_moduletest("DistributeddbNbCrudPowerTest") { + module_out_path = module_output_path + sources = [ + "common/distributeddb/src/distributeddb_constant.cpp", + "common/distributeddb/src/distributeddb_data_generator.cpp", + "common/distributeddb/src/distributeddb_nb_test_tools.cpp", + "common/distributeddb/src/distributeddb_schema_test_tools.cpp", + "moduletest/common/distributeddb/src/distributeddb_nb_crud_power_test.cpp", + ] + sources += mul_sources + configs = [ ":module_private_config" ] + + deps = [ + "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", + "//third_party/googletest:gtest_main", "//third_party/openssl:libcrypto_static", + "//third_party/sqlite:sqlite", + "//utils/native/base:utils", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } -ohos_moduletest("DistributeddbNbPredicateQueryExpandTest") { +ohos_moduletest("DistributeddbNbSchemaTest") { module_out_path = module_output_path sources = [ - "common/distributeddb/src/distributed_test_tools.cpp", + "common/distributeddb/src/distributeddb_constant.cpp", "common/distributeddb/src/distributeddb_data_generator.cpp", "common/distributeddb/src/distributeddb_nb_test_tools.cpp", "common/distributeddb/src/distributeddb_schema_test_tools.cpp", - "moduletest/common/distributeddb/src/distributeddb_nb_predicate_query_expand_test.cpp", + "moduletest/common/distributeddb/src/distributeddb_nb_schema_test.cpp", ] + sources += mul_sources configs = [ ":module_private_config" ] deps = [ "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", "//third_party/googletest:gtest_main", + "//third_party/openssl:libcrypto_static", "//third_party/sqlite:sqlite", "//utils/native/base:utils", + ] + external_deps = [ "hiviewdfx_hilog_native:libhilog" ] +} + +ohos_moduletest("DistributeddbNbSchemaUpgradeTest") { + module_out_path = module_output_path + sources = [ + "common/distributeddb/src/distributeddb_constant.cpp", + "common/distributeddb/src/distributeddb_data_generator.cpp", + "common/distributeddb/src/distributeddb_nb_test_tools.cpp", + "common/distributeddb/src/distributeddb_schema_test_tools.cpp", + "moduletest/common/distributeddb/src/distributeddb_nb_schema_upgrade_test.cpp", + ] + sources += mul_sources + configs = [ ":module_private_config" ] + + deps = [ + "//foundation/distributeddatamgr/distributeddatamgr/services/distributeddataservice/libs/distributeddb:distributeddb", + "//third_party/googletest:gtest_main", "//third_party/openssl:libcrypto_static", + "//third_party/sqlite:sqlite", + "//utils/native/base:utils", ] external_deps = [ "hiviewdfx_hilog_native:libhilog" ] } @@ -492,6 +606,7 @@ group("moduletest") { ] deps += [ + ":DistributedbNbDbDamageTest", ":DistributeddbKvBackupTest", ":DistributeddbKvBatchCrudTest", ":DistributeddbKvConcurrencyCrudTest", @@ -502,16 +617,19 @@ group("moduletest") { ":DistributeddbKvRealdelTest", ":DistributeddbKvTransactionPerfTest", ":DistributeddbKvTransactionTest", + ":DistributeddbNbAutolaunchTest", ":DistributeddbNbBackupTest", ":DistributeddbNbBatchCrudTest", ":DistributeddbNbCreateTest", + ":DistributeddbNbCrudPowerTest", ":DistributeddbNbCrudTest", ":DistributeddbNbCursorTest", - ":DistributeddbNbEnableSyncByClosedDbTest", ":DistributeddbNbLocalBatchCrudTest", ":DistributeddbNbObserverTest", ":DistributeddbNbPredicateQueryExpandTest", ":DistributeddbNbPredicateQueryTest", + ":DistributeddbNbSchemaTest", + ":DistributeddbNbSchemaUpgradeTest", ] } ############################################################################### diff --git a/services/distributeddataservice/test/common/distributeddb/include/auto_launch_callback.h b/services/distributeddataservice/test/common/distributeddb/include/auto_launch_callback.h new file mode 100644 index 0000000000000000000000000000000000000000..17e1f409e8e9ad5db1ac139e313bd4fb813e3b9d --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/include/auto_launch_callback.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef AUTO_LAUNCH_CALLBACK_H +#define AUTO_LAUNCH_CALLBACK_H + +#include +#include "auto_launch_export.h" +#include "types_export.h" + +#ifdef RELEASE_MODE_V2 +class AutoLaunchCallback { +public: + AutoLaunchCallback() {} + ~AutoLaunchCallback() {} + + // Delete the copy and assign constructors + AutoLaunchCallback(const AutoLaunchCallback &callback) = delete; + AutoLaunchCallback &operator=(const AutoLaunchCallback &callback) = delete; + AutoLaunchCallback(AutoLaunchCallback &&callback) = delete; + AutoLaunchCallback &operator=(AutoLaunchCallback &&callback) = delete; + + void AutoLaunchNotifier(const std::string &userId, const std::string &appId, const std::string &storeId, + DistributedDB::AutoLaunchStatus status); + int GetStatus(); + void Clear(); +#ifdef RELEASE_MODE_V3 + bool AutoLaunchRequestNotifier(const std::string &identifier, DistributedDB::AutoLaunchParam ¶m); + void AddHashIdentity(const std::string &hashIdentity); + void ClearHashIdentities(); + void SetAutoLaunchParam(DistributedDB::AutoLaunchParam &autoLaunchParam); +#endif + +private: + int realStatus_ = 0; + std::vector hashIdentities_; +#ifdef RELEASE_MODE_V3 + DistributedDB::AutoLaunchParam autoLaunchParam_; +#endif +}; +#endif // end of RELEASE_MODE_V2 +#endif // AUTO_LAUNCH_CALLBACK_H \ No newline at end of file diff --git a/services/distributeddataservice/test/common/distributeddb/include/delegate_callback.h b/services/distributeddataservice/test/common/distributeddb/include/delegate_callback.h new file mode 100644 index 0000000000000000000000000000000000000000..ecb1497db4e3b21175f243db894be0b8cf8a3983 --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/include/delegate_callback.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DELEGATE_CALLBACK_H +#define DELEGATE_CALLBACK_H +#include "kv_store_snapshot_delegate.h" +#include "types.h" +#include "types_export.h" + +// DelegateCallback conclude the Callback implements of function< void(DBStatus, KvStoreSnapshotDelegate*)> +class DelegateCallback { +public: + DelegateCallback() {} + ~DelegateCallback() {} + + // Delete the copy and assign constructors + DelegateCallback(const DelegateCallback &callback) = delete; + DelegateCallback& operator=(const DelegateCallback &callback) = delete; + DelegateCallback(DelegateCallback &&callback) = delete; + DelegateCallback& operator=(DelegateCallback &&callback) = delete; + + void Callback(DistributedDB::DBStatus status, DistributedDB::KvStoreSnapshotDelegate *kvStoreSnapshotDelegate); + void CallbackKv(DistributedDB::DBStatus status, const DistributedDB::Value &value); + + DistributedDB::DBStatus GetStatus(); + const DistributedDB::Value &GetValue(); + + const DistributedDB::KvStoreSnapshotDelegate *GetKvStoreSnapshot() + { + return kvStoreSnapshotDelegate_; + } + +private: + DistributedDB::DBStatus status_ = DistributedDB::DBStatus::INVALID_ARGS; + DistributedDB::Value value_ = {}; + DistributedDB::KvStoreSnapshotDelegate *kvStoreSnapshotDelegate_ = nullptr; +}; + +#endif // DELEGATE_CALLBACK_H \ No newline at end of file diff --git a/services/distributeddataservice/test/common/distributeddb/include/delegate_kv_mgr_callback.h b/services/distributeddataservice/test/common/distributeddb/include/delegate_kv_mgr_callback.h new file mode 100644 index 0000000000000000000000000000000000000000..00d58a11c9ca1a99532d16d2c9e2c8c1b6aa744e --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/include/delegate_kv_mgr_callback.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef DELEGATE_KV_MGR_CALLBACK_H +#define DELEGATE_KV_MGR_CALLBACK_H + +#include "kv_store_delegate.h" +#include "types.h" + +// DelegateKvMgrCallback conclude the Callback implements of function< void(DBStatus, KvStoreDelegate*)> +class DelegateKvMgrCallback { +public: + DelegateKvMgrCallback() {} + ~DelegateKvMgrCallback() {} + + // Delete the copy and assign constructors + DelegateKvMgrCallback(const DelegateKvMgrCallback &callback) = delete; + DelegateKvMgrCallback& operator=(const DelegateKvMgrCallback &callback) = delete; + DelegateKvMgrCallback(DelegateKvMgrCallback &&callback) = delete; + DelegateKvMgrCallback& operator=(DelegateKvMgrCallback &&callback) = delete; + + void Callback(DistributedDB::DBStatus status, DistributedDB::KvStoreDelegate *kvStoreDelegate); + + DistributedDB::DBStatus GetStatus(); + + const DistributedDB::KvStoreDelegate *GetKvStore(); + +private: + DistributedDB::DBStatus status_ = DistributedDB::DBStatus::INVALID_ARGS; + DistributedDB::KvStoreDelegate *kvStoreDelegate_ = nullptr; +}; + +#endif // DELEGATE_KV_MGR_CALLBACK_H \ No newline at end of file diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributed_test_sysinfo.h b/services/distributeddataservice/test/common/distributeddb/include/distributed_test_sysinfo.h index dc89b8ee75b370f7f4332999cada765dc4aa3ab7..417e9979feac72e7ab232f82edb16c1834d84b63 100755 --- a/services/distributeddataservice/test/common/distributeddb/include/distributed_test_sysinfo.h +++ b/services/distributeddataservice/test/common/distributeddb/include/distributed_test_sysinfo.h @@ -16,14 +16,18 @@ #define DISTRIBUTED_TEST_SYSINFO_H #include -#include "platform_specific.h" +#include "distributeddb_log_print.h" +#if defined(RUNNING_ON_LINUX) #include +#elif defined RUNNING_ON_WIN +#endif const uint64_t SYSTEM_INFO_BUFFER_SIZE = 20; const uint64_t PROC_BUFFER_LENGTH = 4096; const uint64_t DEFAULT_INTEVAL = 250000; const uint64_t DEFAULT_COUNT = 5; +#if defined(RUNNING_ON_LINUX) const std::string SYS_MEM_FILE = "/proc/meminfo"; const std::string SYS_CPU_FILE = "/proc/stat"; const std::string POWER_FOLLOW_FILE = "/sys/class/power_supply/Battery/current_now"; @@ -55,6 +59,8 @@ struct CpuOccupy { uint64_t irq_; uint64_t softirq_; }; +#elif defined RUNNING_ON_WIN +#endif enum SeqNo { FIRST = 1, @@ -95,8 +101,11 @@ public: void SaveSecondToFirst(); private: +#if defined(RUNNING_ON_LINUX) MemOccupy memStatFirst_, memStatSecond_; CpuOccupy cpuStatFirst_, cpuStatSecond_; +#elif defined RUNNING_ON_WIN +#endif float cpuStatFirstUsage_, cpuStatSecondUsage_; float powerStatFirst_, powerStatSecond_; float val_; diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributed_test_tools.h b/services/distributeddataservice/test/common/distributeddb/include/distributed_test_tools.h index 37143c0f049fd92788161d6c74a35ae32ac62b31..318b8b00a01fc8ce8721618544cfa6f49f32b00e 100755 --- a/services/distributeddataservice/test/common/distributeddb/include/distributed_test_tools.h +++ b/services/distributeddataservice/test/common/distributeddb/include/distributed_test_tools.h @@ -19,25 +19,23 @@ #include "kv_store_delegate.h" #include "kv_store_delegate_manager.h" +#include "kv_store_observer_impl.h" #include "types.h" #include "distributed_test_sysinfo.h" #include "distributeddb_data_generator.h" #include "log_print.h" -#ifdef TESTCASES_USING_GTEST +#ifdef RUN_MST_ON_TRUNCK // only need it in WAGNER env if run MST #define HWTEST_F(test_case_name, test_name, level) TEST_F(test_case_name, test_name) #endif +#define ULL(x) (static_cast(x)) const int MAX_DIR_LENGTH = 4096; // the max length of directory +const int BUF_LEN = 8192; const static std::string TAG = "DistributedTestTools"; // for log const int AUTHORITY = 0755; const int E_OK = 0; const int E_ERROR = -1; const std::string DIRECTOR = "/data/test/getstub/"; // default work dir. static std::condition_variable g_conditionKvVar; -enum ListType { - INSERT_LIST = 0, - UPDATE_LIST = 1, - DELETE_LIST = 2 -}; struct KvDBParameters { std::string storeId; @@ -106,8 +104,6 @@ const static KvOption g_createLocalDiskEncrypted(true, true, true, DistributedDB DistributedDBDataGenerator::PASSWD_VECTOR_1); static KvOption g_kvOption = g_createKvDiskEncrypted; bool CompareVector(const std::vector& first, const std::vector& second); -bool CompareList(const std::list& retLst, - const std::list& lst); bool CompareEntriesVector(std::vector& retVec, std::vector& expectVec); void PutUniqueKey(std::vector& entryVec, @@ -116,6 +112,8 @@ int Uint8VecToString(std::vector& vec, std::string& str); int GetIntValue(DistributedDB::Value &value); int RemoveDir(const std::string &directory); int SetDir(const std::string &directory, const int authRight = AUTHORITY); +void CopyFile(const std::string &srcFile, const std::string &destFile); +void CopyDir(const std::string &srcDir, const std::string &destDir, const int authRight = AUTHORITY); void CheckFileNumber(const std::string &filePath, int &fileCount); DistributedDB::Value GetValueWithInt(int val); std::vector GenRanKeyVal(int putGetTimes, int keyLength, int valueLength, char val); @@ -406,6 +404,13 @@ public: static bool CalculateTransactionPerformance(PerformanceData &performanceData); static bool CloseAndRelease(DistributedDB::KvStoreDelegateManager *&manager, DistributedDB::KvStoreDelegate *&delegate); + static bool CloseAndRelease(DistributedDB::KvStoreDelegateManager *&manager, + DistributedDB::KvStoreNbDelegate *&delegate); + static bool VerifyDbRecordCnt(DistributedDB::KvStoreNbDelegate *&delegate, unsigned int recordCnt, + bool isLocal = false); + static bool VerifyRecordsInDb(DistributedDB::KvStoreNbDelegate *&delegate, + std::vector &entriesExpected, + const std::vector &keyPrefix = DistributedDBDataGenerator::KEY_EMPTY, bool isLocal = false); static bool GetRecordCntByKey(const std::string &dbName, const std::string &strSql, std::vector &sqlParam, KvOption &option, int &count); static bool QuerySpecifiedData(const std::string &dbName, const std::string &strSql, @@ -413,164 +418,20 @@ public: static bool RepeatCheckAsyncResult(const std::function &inPred, int repeatLimit, uint32_t repeatInterval); static bool CompareKey(const DistributedDB::Entry &entry1, const DistributedDB::Entry &entry2); -}; - -// DelegateCallback conclude the Callback implements of function< void(DBStatus, KvStoreSnapshotDelegate*)> -class DelegateCallback { -public: - DelegateCallback() {} - ~DelegateCallback() {} - - // Delete the copy and assign constructors - DelegateCallback(const DelegateCallback &callback) = delete; - DelegateCallback& operator=(const DelegateCallback &callback) = delete; - DelegateCallback(DelegateCallback &&callback) = delete; - DelegateCallback& operator=(DelegateCallback &&callback) = delete; - - void Callback(DistributedDB::DBStatus status, DistributedDB::KvStoreSnapshotDelegate *kvStoreSnapshotDelegate); - - DistributedDB::DBStatus GetStatus(); - - const DistributedDB::KvStoreSnapshotDelegate *GetKvStoreSnapshot() - { - return kvStoreSnapshotDelegate_; - } - -private: - DistributedDB::DBStatus status_ = DistributedDB::DBStatus::INVALID_ARGS; - DistributedDB::KvStoreSnapshotDelegate *kvStoreSnapshotDelegate_ = nullptr; -}; - -// DelegateKvMgrCallback conclude the Callback implements of function< void(DBStatus, KvStoreDelegate*)> -class DelegateKvMgrCallback { -public: - DelegateKvMgrCallback() {} - ~DelegateKvMgrCallback() {} - - // Delete the copy and assign constructors - DelegateKvMgrCallback(const DelegateKvMgrCallback &callback) = delete; - DelegateKvMgrCallback& operator=(const DelegateKvMgrCallback &callback) = delete; - DelegateKvMgrCallback(DelegateKvMgrCallback &&callback) = delete; - DelegateKvMgrCallback& operator=(DelegateKvMgrCallback &&callback) = delete; - - void Callback(DistributedDB::DBStatus status, DistributedDB::KvStoreDelegate *kvStoreDelegate); - - DistributedDB::DBStatus GetStatus(); - - const DistributedDB::KvStoreDelegate *GetKvStore(); - -private: - DistributedDB::DBStatus status_ = DistributedDB::DBStatus::INVALID_ARGS; - DistributedDB::KvStoreDelegate *kvStoreDelegate_ = nullptr; + static void CopyFile(const std::string &srcFile, const std::string &destFile); }; std::string TransferStringToHashHexString(const std::string &origStr); int RemoveDatabaseDirectory(const std::string &directory); -class KvStoreObserverImpl final : public DistributedDB::KvStoreObserver { -public: - void OnChange(const DistributedDB::KvStoreChangedData &data); - - KvStoreObserverImpl(); - - ~KvStoreObserverImpl(); - - KvStoreObserverImpl(const KvStoreObserverImpl &); - KvStoreObserverImpl& operator=(const KvStoreObserverImpl &); - - const std::list GetInsertList() const; - - const std::list GetUpdateList() const; - - const std::list GetDeleteList() const; - - int GetChanged() const; - - void WaitUntilReachChangeCount(unsigned int countGoal, uint32_t timeout = 0) const; // timeout in second - // timeout in second - void WaitUntilReachRecordCount(unsigned int countExpect, ListType waitWhat, uint32_t timeout = 0) const; - - microClock_type GetOnChangeTime(); - - void Clear(); - - void SetCumulatedFlag(bool isSaveCumulatedData); - - bool GetCumulatedFlag() const; - - const std::list GetCumulatedInsertList() const; - - const std::list GetCumulatedUpdateList() const; - - const std::list GetCumulatedDeleteList() const; - -private: - std::list insertedEntries_ = {}; - std::list updatedEntries_ = {}; - std::list deleteEntries_ = {}; - unsigned int changed_ = 0; - microClock_type onChangeTime_ - = std::chrono::time_point_cast(std::chrono::steady_clock::now()); - bool isSaveCumulatedData_ = false; - std::list cumulatedInsertList_ = {}; - std::list cumulatedUpdateList_ = {}; - std::list cumulatedDeleteList_ = {}; - // For waiting method - mutable std::mutex waitChangeMutex_; - mutable std::condition_variable waitChangeCv_; -}; - bool VerifyObserverResult(const KvStoreObserverImpl &pObserver, - int changedTimes, ListType type, const std::list &lst); + int changedTimes, ListType type, const std::list &lst, + uint32_t timeout = DistributedDBDataGenerator::DistributedDBConstant::THIRTY_MINUTES); bool VerifyObserverResult(const KvStoreObserverImpl &pObserver, - int changedTimes, ListType type, const std::vector &vec); - -class KvStoreSnapshotCallback { -public: - KvStoreSnapshotCallback() {} - ~KvStoreSnapshotCallback() {} - /** - * @tc.steps: step1. Delete the copy and assign constructors. - * @tc.expected: step1. operate successfully. - */ - KvStoreSnapshotCallback(const KvStoreSnapshotCallback &callback) = delete; - KvStoreSnapshotCallback& operator=(const KvStoreSnapshotCallback &callback) = delete; - KvStoreSnapshotCallback(KvStoreSnapshotCallback &&callback) = delete; - KvStoreSnapshotCallback& operator=(KvStoreSnapshotCallback &&callback) = delete; - - void Callback(DistributedDB::DBStatus status, const std::vector &entriesVec); - DistributedDB::DBStatus GetStatus(); - std::vector GetEntries(); - -private: - DistributedDB::DBStatus status_ = DistributedDB::DBStatus::INVALID_ARGS; - std::vector entriesVec_ = {}; -}; - -class AutoLaunchCallback { -public: - AutoLaunchCallback() {} - ~AutoLaunchCallback() {} - - // Delete the copy and assign constructors - AutoLaunchCallback(const AutoLaunchCallback &callback) = delete; - AutoLaunchCallback &operator=(const AutoLaunchCallback &callback) = delete; - AutoLaunchCallback(AutoLaunchCallback &&callback) = delete; - AutoLaunchCallback &operator=(AutoLaunchCallback &&callback) = delete; - - void AutoLaunchNotifier(const std::string &userId, const std::string &appId, const std::string &storeId, - DistributedDB::AutoLaunchStatus status); - bool AutoLaunchRequestNotifier(const std::string &identifier, DistributedDB::AutoLaunchParam ¶m); - int GetStatus(); - void Clear(); - void AddHashIdentity(const std::string &hashIdentity); - void ClearHashIdentities(); - void SetAutoLaunchParam(DistributedDB::AutoLaunchParam &autoLaunchParam); - -private: - int realStatus_ = 0; - std::vector hashIdentities_; - DistributedDB::AutoLaunchParam autoLaunchParam_; -}; + int changedTimes, ListType type, const std::vector &vec, + uint32_t timeout = DistributedDBDataGenerator::DistributedDBConstant::THIRTY_MINUTES); +bool VerifyObserverForSchema(const KvStoreObserverImpl &pObserver, + int changedTimes, ListType type, const std::vector &expectEntry, + uint32_t timeout = DistributedDBDataGenerator::DistributedDBConstant::THIRTY_MINUTES); #endif // DISTRIBUTED_DB_MODULE_TEST_TOOLS_H diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributeddb_constant.h b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_constant.h new file mode 100644 index 0000000000000000000000000000000000000000..c92c2d1ca37eeef90f106b8e79719c0256822d35 --- /dev/null +++ b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_constant.h @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DISTRIBUTEDDB_CONSTANT_DEFINE_H +#define DISTRIBUTEDDB_CONSTANT_DEFINE_H + +#include +#include "types.h" + +namespace DistributedDBDataGenerator { +const int TEST_ID_1 = 1; +const int TEST_ID_2 = 2; +const int TEST_ID_3 = 3; +const int TEST_ID_4 = 4; +const int TEST_ID_5 = 5; +const int TEST_ID_6 = 6; +const int TEST_ID_7 = 7; +const int TEST_ID_8 = 8; +const int TEST_ID_9 = 9; +const int TEST_ID_10 = 10; +const int TEST_ID_11 = 11; +const int TEST_ID_12 = 12; +const int TEST_ID_13 = 13; +const int TEST_ID_14 = 14; +const int TEST_ID_15 = 15; +const int TEST_ID_16 = 16; +const int TEST_ID_17 = 17; +const int TEST_ID_18 = 18; +const int TEST_ID_19 = 19; +const int TEST_ID_20 = 20; +const int TEST_ID_21 = 21; +const int TEST_ID_22 = 22; +const int TEST_ID_23 = 23; +const int TEST_ID_24 = 24; +const int TEST_ID_25 = 25; +const int TEST_ID_26 = 26; +const int TEST_ID_27 = 27; +const int TEST_ID_28 = 28; +const int TEST_ID_29 = 29; +const int TEST_ID_30 = 30; +const int TEST_ID_31 = 31; +const int TEST_ID_32 = 32; +const int TEST_ID_33 = 33; +const int TEST_ID_34 = 34; +const int TEST_ID_35 = 35; +const int TEST_ID_36 = 36; +const int TEST_ID_37 = 37; +const int TEST_ID_38 = 38; +const int TEST_ID_39 = 39; +const int TEST_ID_40 = 40; +const int TEST_ID_41 = 41; +const int TEST_ID_42 = 42; +const int TEST_ID_43 = 43; +const int TEST_ID_44 = 44; +const int TEST_ID_45 = 45; +const int TEST_ID_46 = 46; +const int TEST_ID_47 = 47; +const int TEST_ID_48 = 48; +const int TEST_ID_49 = 49; +const int TEST_ID_50 = 50; +const int TEST_ID_51 = 51; +const int TEST_ID_52 = 52; + +class DistributedDBConstant { +public: + static const std::string NB_DIRECTOR; // default work dir. + static const std::string NB_DATABASE_NAME; + static const std::string NORMAL_COMMON_SCHEMA; + static const std::string COMPATIBLE_FOR_NORMAL_COMMON_SCHEMA; + static const std::string UNCOMPATIBLE_FOR_NORMAL_COMMON_SCHEMA; + // default kvStoreDelegateManager's config. + static const DistributedDB::KvStoreConfig CONFIG; + static const std::string DB_FILE_DOCUMENT; + static constexpr int UNLOCK_ACCESS = 0; + static constexpr int LOCKED_ACCESS = 1; + static constexpr uint32_t THIRTY_MINUTES = 1800; + static constexpr uint32_t WAIT_UNTIL_CALLBACK_COME = 0; + +// ************************ SYNC PACKET ************************************ + static constexpr int ZERO_PACKET = 0; + static constexpr int ONE_PACKET = 1; + static constexpr int TWO_PACKETS = 2; + static constexpr int THREE_PACKETS = 3; + static constexpr int FOUR_PACKETS = 4; + static constexpr int EIGHT_PACKETS = 8; + static constexpr int FIRST_PACKET = 1; + static constexpr int THIRD_PACKET = 3; + static constexpr int FIFTH_PACKET = 5; + static constexpr int EIGHTH_PACKET = 8; + static constexpr uint32_t THROW_PACKET_CNT_LEN = 168; // the length of data packet is greater than 168 + // the length of registering subscriber packet is greater than 111 + static constexpr uint32_t REGISTER_SUBSCRIBER_PACKET_LEN = 111; + +// ************************ MILLISECOND ************************************ + static const unsigned int THREE_S_TO_MS = 3000; + static const unsigned int FIVE_S_TO_MS = 5000; + static const unsigned int EIGHT_S_TO_MS = 8000; + static const unsigned int NINE_S_TO_MS = 9000; + static const unsigned int TEN_S_TO_MS = 10000; + static const unsigned int SIXTY_S_TO_MS = 60000; + +// ************************ compression ratio ************************** + static const unsigned int COMPRESSION_RATIO_ZERO = 0; + static const unsigned int COMPRESSION_RATIO_TEN = 10; + static const unsigned int COMPRESSION_RATIO_TWENTY = 20; + static const unsigned int COMPRESSION_RATIO_THIRTY = 30; + static const unsigned int COMPRESSION_RATIO_FORTY = 40; + static const unsigned int COMPRESSION_RATIO_FIFTY = 50; + static const unsigned int COMPRESSION_RATIO_SIXTY = 60; + static const unsigned int COMPRESSION_RATIO_SEVENTY = 70; + static const unsigned int COMPRESSION_RATIO_EIGHTY = 80; + static const unsigned int COMPRESSION_RATIO_NINETY = 90; + static const unsigned int COMPRESSION_RATIO_ONE_HUNDRED = 100; + static const unsigned int COMPRESSION_RATIO_ONE_HUNDRED_AND_ONE = 101; + +// ************************ record number ************************** + static const unsigned int THIRTY_THOUSAND_RECORDS = 30000; +}; +} // namespace DistributedDBDataGenerator + +#endif // DISTRIBUTEDDB_CONSTANT_DEFINE_H diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributeddb_data_generator.h b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_data_generator.h index d034dd449606ea7ddd6e5b927cb839e42e1db3a4..a6836dfd955b4ca17aaf26f0b49fc88e39660115 100755 --- a/services/distributeddataservice/test/common/distributeddb/include/distributeddb_data_generator.h +++ b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_data_generator.h @@ -23,6 +23,7 @@ #include "kv_store_delegate_manager.h" #include "distributed_test_sysinfo.h" #include "distributeddb_log_print.h" +#include "distributeddb_constant.h" struct EntrySize { unsigned int keySize = 0; @@ -77,6 +78,15 @@ struct LongDefine { char prefix; }; +struct NumberSize { + EntrySize entrySize; + int recordsNumber; + bool isRoundBack = false; + NumberSize() : recordsNumber(0) + { + } +}; + enum class RandType { ALPHA_NUM, ALPHA_NUM_UNDERLINE, @@ -95,17 +105,22 @@ const unsigned int THREE_RECORDS = 3; const unsigned int THREE_PERF_DATA = 3; const unsigned int BATCH_RECORDS = 128; const static int ONE_RECORD = 1; +const static int TWO_RECORDS = 2; const static int FOUR_RECORDS = 4; const static int FIVE_RECORDS = 5; +const static int SIX_RECORDS = 6; +const static int EIGHT_RECORDS = 8; +const static int NINE_RECORDS = 9; const static int TEN_RECORDS = 10; +const static int SIXTEEN_RECORDS = 16; const static int TWENTY_RECORDS = 20; +const static int THIRTY_RECORDS = 30; const static int THIRTYTWO_RECORDS = 32; const static int FORTY_RECORDS = 40; const static int FIFTY_RECORDS = 50; const static int SIXTY_RECORDS = 60; const static int EIGHTY_RECORDS = 80; const static int ONE_HUNDRED_RECORDS = 100; -const static int ONE_HUNDRED_AND_TWENTY_RECORDS = 120; const static int TWO_HUNDREDS_RECORDS = 200; const static int FIVE_HUNDREDS_RECORDS = 500; const static int SIX_HUNDREDS_RECORDS = 600; @@ -115,20 +130,20 @@ const static int TWO_FIVE_ZERO_ZERO_RECORDS = 2500; const static int TWO_FIVE_SIX_ZERO_RECORDS = 2560; const static int FIVE_THOUSANDS_RECORDS = 5000; const static int TEN_THOUSAND_RECORDS = 10000; +const static int FIFTEEN_THOUSAND_RECORDS = 15000; const static int TWENTY_THOUSAND_RECORDS = 20000; +const static int THIRTY_THOUSAND_RECORDS = 30000; +const static int FIFTY_THOUSAND_RECORDS = 50000; const static int HUNDRED_THOUSAND_RECORDS = 100000; const static int FOUR_HUNDRED_THOUSAND_RECORDS = 400000; const static int FIVE_HUNDRED_THOUSAND_RECORDS = 500000; const static int TWO_FIVE_SIX_RECORDS = 256; const static int FIRST_RECORD = 1; const static int SECOND_RECORD = 2; -const static int FOURTH_RECORD = 4; -const static int FORTIETH_RECORD = 40; -const static int EIGHTIETH_RECORD = 80; -const static int THE_HUNDRED_AND_TWENTY_RECORD = 120; const static int DATAS_ACCOUNT = 300; const static int DATA_LEN = 13; const static int NINE_CNT = 9; +const static int TEN_CNT = 10; const static unsigned int TWO_DEVICES = 2; const static unsigned int FOUR_DEVICES = 4; const static int THIRD_FUNC = 3; @@ -138,10 +153,12 @@ const static int THREE_DBS = 3; const static int EIGHT_DBS = 8; const static int TEN_DBS = 10; const static int ELEVEN_DBS = 11; +const static int TWELVE_DBS = 12; const static int FIRST_DB = 1; const static int FIFTH_DB = 5; const static int TENTH_DB = 10; const static int ELEVENTH_DB = 11; +const static int TWELFTH_DB = 12; // ************************ loop times class ************************** const unsigned int MOD_NUM = 2; @@ -158,6 +175,7 @@ const static int MANYTINES = 3; const static int ONE_TIME = 1; const static int TWO_TIMES = 2; const static int FIVE_TIMES = 5; +const static int TEN_TIMES = 10; const static int FIFTY_TIMES = 50; const static int HUNDRED_TIMES = 100; @@ -167,6 +185,7 @@ const unsigned int TWO_SECONDS = 2; const unsigned int THREE_SECONDS = 3; const unsigned int FOUR_SECONDS = 4; const unsigned int FIVE_SECONDS = 5; +const unsigned int SIX_SECONDS = 6; const unsigned int TEN_SECONDS = 10; const unsigned int FIFTEEN_SECONDS = 15; const unsigned int TWENTY_SECONDS = 20; @@ -196,7 +215,7 @@ const unsigned int WAIT_FOR_LAST_SYNC = 500000; const unsigned int WAIT_FOR_TWO_HUNDREDS_MS = 200000; const int FIFTY_MILI_SECONDS = 50; const int HUNDRED_MILLI_SECONDS = 100; -const static int MILLSECONDES_PER_SECOND = 1000; +const static int MILLSECONDS_PER_SECOND = 1000; // ************************ length of key class ************************** const static int KEY_SIX_BYTE = 6; @@ -218,6 +237,7 @@ const unsigned int ONE_M_LONG_STRING = 1048576; // 1M const unsigned int TWO_M_LONG_STRING = 2097152; // 2M const unsigned int FOUR_M_LONG_STRING = 4194304; // 4M const unsigned int TEN_M_LONG_STRING = 10485760; // 10M +const static int VALUE_SIX_BYTE = 6; const static int VALUE_ONE_HUNDRED_BYTE = 100; const static int VALUE_FIVE_HUNDRED_BYTE = 500; const static int VALUE_ONE_K_BYTE = 1024; @@ -252,8 +272,7 @@ const unsigned int NB_OBSERVER_CNT_START = 0; const unsigned int NB_OBSERVER_CNT_END = 4; const unsigned int NB_OPERATION_CNT_START = 0; const unsigned int NB_OPERATION_CNT_END = 5; -const int RAND_BOOL_MIN = 0; -const int RAND_BOOL_MAX = 1; + const int LOCAL_OPER_CNT = 2; const int NATIVE_OPER_CNT = 6; const static int OPER_CNT_START = 0; @@ -277,17 +296,14 @@ const int INDEX_SIXTH = 6; const int INDEX_SEVENTH = 7; const int INDEX_EIGHTTH = 8; const int INDEX_NINTH = 9; +const int INDEX_TENTH = 10; const int INDEX_NINE_NINE_NINTH = 999; +const int INDEX_FIVE_THOUSANDS = 5000; +const int INDEX_TEN_THOUSANDS = 10000; +const int INDEX_FIFTEEN_THOUSANDS = 15000; // ************************ OTHER CLASS ************************************ -const static int ROUND_BACK = 2; -const static int TRUNC_EIGHT = 100000000; - -const unsigned int PIPE_BUFFER = 128; - -const uint8_t ACSIIEND = 255; - -const int TABLE_MAX = 256; +const int TABLE_MAX = 60; const static int ENCRYPT_COUNT = 100; @@ -323,7 +339,7 @@ const std::string STORE_ID_7 = "STORE_ID_7"; const std::string STORE_ID_8 = "STORE_ID_8"; const std::string STORE_ID_9 = "STORE_ID_9"; const std::string STORE_ID_10 = "STORE_ID_10"; -const std::string SCHEMA_STORE_ID_11 = "SCHEMA_STORE_ID_11"; +const std::string JSON_SCHEMA_STORE_ID_11 = "JSON_SCHEMA_STORE_ID_11"; const std::string STORE_ID_PERFORM = "STORE_ID_PERFORM"; const static std::string STORE_ID_SYNC_1 = "SYNC1"; const static std::string STORE_ID_SYNC_2 = "SYNC2"; @@ -386,6 +402,7 @@ const std::vector NULL_K1 = {}; const DistributedDB::Key KEY_EMPTY = { }; const DistributedDB::Key KEY_K = { 'k' }; const DistributedDB::Key KEY_A = { 'a' }; +const DistributedDB::Key KEY_E = { 'k', 'e' }; const DistributedDB::Key OK_KEY_1 = { 'o', 'k' }; const DistributedDB::Value VALUE_1 = { 'v', '1' }; @@ -514,11 +531,9 @@ void GenerateFullAsciiRecords(DistributedDB::Entry &entry); void GenerateBiggistKeyRecords(DistributedDB::Entry &entry); DistributedDB::Entry GenerateFixedLenKVRecord(unsigned int serialNo, - unsigned int keyLen, uint8_t keyFilledChr, - unsigned int valueLen, uint8_t valueFilledChr); + unsigned int keyLen, uint8_t keyFilledChr, unsigned int valueLen, uint8_t valueFilledChr); -void GenerateFixedRecords(std::vector &entries, - std::vector &allKeys, +void GenerateFixedRecords(std::vector &entries, std::vector &allKeys, int recordNum, unsigned int keySize, unsigned int valSize); void GenerateOneRecordForImage(int entryNo, const EntrySize &entrySize, @@ -530,15 +545,20 @@ void GenerateAppointPrefixAndSizeRecord(int recordNo, const EntrySize &entrySize const std::vector &keyPrefix, const std::vector &valPrefix, DistributedDB::Entry &entry); void GenerateAppointPrefixAndSizeRecords(std::vector &entries, const EntrySize &entrySize, int num, const std::vector &keyPrefix = {'k'}, const std::vector &valPrefix = {'v'}); +void GenerateAppointPrefixAndSizeRecords(std::vector &entries, int startpoint, + const NumberSize param, const std::vector &keyPrefix = {'k'}, + const std::vector &valPrefix = {'v'}); int GetRandInt(const int randMin, const int randMax); -void GenerateFixedLenRandRecords(std::vector &entries, - std::vector &allKeys, +void GenerateFixedLenRandRecords(std::vector &entries, std::vector &allKeys, int recordNum, unsigned int keySize, unsigned int valSize); +std::vector GenerateFixedLenRandRecords(std::vector &allKeys, + int recordNum, const EntrySize &entrySize, const std::vector &keyPrefix = {'k'}, + const std::vector &valPrefix = {'v'}); const std::string GetDbType(const int type); -void GenerateRandomRecords(std::vector &entries, EntrySize entrySize, int num); +void GenerateRandomRecords(std::vector &entries, EntrySize &entrySize, int num); void GetLongSchemaDefine(LongDefine ¶m, std::string &longDefine); const std::string SpliceToSchema(const std::string &version, const std::string &mode, const std::string &define, const std::string &index = "", const std::string &skipSize = ""); diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributeddb_log_print.h b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_log_print.h index 4ce2bd14012cb651d30a0992f6bb62763bb11e6d..ff00a8f26d5a7a3894799842997e9a8b79e36ae3 100755 --- a/services/distributeddataservice/test/common/distributeddb/include/distributeddb_log_print.h +++ b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_log_print.h @@ -15,13 +15,30 @@ #ifndef DISTRIBUTED_DB_LOG_PRINT_H #define DISTRIBUTED_DB_LOG_PRINT_H -#if defined USING_PRINTF_LOGGER +#if defined _WIN32 + #ifndef RUNNING_ON_WIN + #define RUNNING_ON_WIN + #endif +#else + #ifndef RUNNING_ON_LINUX + #define RUNNING_ON_LINUX + #endif +#endif + +#if defined USING_LOGCAT_LOGGER + #include +#elif defined _WIN32 + #define USING_PRINTF_LOGGER #include #elif defined USING_HILOG_LOGGER #include "hilog/log.h" #endif -#if defined USING_PRINTF_LOGGER +#if defined USING_LOGCAT_LOGGER + #define MST_LOG(fmt, ...) \ + (void)(__android_log_print(static_cast(ANDROID_LOG_INFO), "DistributedDB[TEST]", \ + "%s: " fmt, __FUNCTION__, ##__VA_ARGS__)) +#elif defined USING_PRINTF_LOGGER #define MST_LOG(fmt, ...) \ (void)(std::printf(fmt"\n", ##__VA_ARGS__)) #elif defined USING_HILOG_LOGGER diff --git a/services/distributeddataservice/test/common/distributeddb/include/distributeddb_nb_test_tools.h b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_nb_test_tools.h index 7035be240302c684c8920ccb6ca229264c76daa8..fdc8b858a60e52de5bedaa3eb327e35b97b9dc04 100755 --- a/services/distributeddataservice/test/common/distributeddb/include/distributeddb_nb_test_tools.h +++ b/services/distributeddataservice/test/common/distributeddb/include/distributeddb_nb_test_tools.h @@ -19,12 +19,12 @@ #include "kv_store_delegate_manager.h" #include "kv_store_nb_delegate.h" #include "kv_store_observer.h" +#ifdef RELEASE_MODE_V2 #include "query.h" +#endif #include "distributeddb_data_generator.h" #include "distributed_test_tools.h" -const std::string NB_DIRECTOR = "/data/test/nbstub/"; // default work dir. - struct DBParameters { std::string storeId; std::string appId; @@ -41,14 +41,38 @@ struct Option { bool isEncryptedDb = false; // whether need encrypt DistributedDB::CipherType cipher = DistributedDB::CipherType::DEFAULT; // cipher type std::vector passwd; // cipher password +#ifdef RELEASE_MODE_V2 std::string schema; bool createDirByStoreIdOnly = false; - Option(bool createIfNecessary1, bool isMemoryDb1, bool isEncryptedDb1, DistributedDB::CipherType cipher1, - std::vector passwd1) - : createIfNecessary(createIfNecessary1), isMemoryDb(isMemoryDb1), isEncryptedDb(isEncryptedDb1), - cipher(cipher1), passwd(passwd1) +#endif // endif of RELEASE_MODE_V2 +#ifdef RELEASE_MODE_V3 + DistributedDB::SecurityOption secOption; + DistributedDB::KvStoreObserver *observer = nullptr; + DistributedDB::Key key; + unsigned int mode = 0; + int conflictType = 0; + DistributedDB::KvStoreNbConflictNotifier notifier = nullptr; + int conflictResolvePolicy = DistributedDB::LAST_WIN; + bool isNeedIntegrityCheck = false; + bool isNeedRmCorruptedDb = false; + bool isNeedCompressOnSync = false; + uint8_t compressionRate = 100; // default compression rate 100%, that means not compressing +#endif // end of RELEASE_MODE_V3 + Option(bool createIfNecessary, bool isMemoryDb, bool isEncryptedDb, DistributedDB::CipherType cipher, + std::vector passwd) + : createIfNecessary(createIfNecessary), isMemoryDb(isMemoryDb), isEncryptedDb(isEncryptedDb), + cipher(cipher), passwd(passwd) { } +#ifdef RELEASE_MODE_V3 + Option(bool createIfNecessary, bool isMemoryDb, bool isEncryptedDb, const DistributedDB::CipherType &cipher, + const std::vector &passwd, const DistributedDB::SecurityOption &secOption, + DistributedDB::KvStoreObserver *observer, const DistributedDB::KvStoreNbConflictNotifier ¬ifier) + : createIfNecessary(createIfNecessary), isMemoryDb(isMemoryDb), isEncryptedDb(isEncryptedDb), + cipher(cipher), passwd(passwd), secOption(secOption), observer(observer), notifier(notifier) + { + } +#endif // endif of RELEASE_MODE_V3 Option() {} }; @@ -80,11 +104,6 @@ enum class ReadOrWriteTag { REGISTER = 3 }; -// default kvStoreDelegateManager's config. -const static DistributedDB::KvStoreConfig CONFIG = { - .dataDir = NB_DIRECTOR -}; - const static DBParameters g_dbParameter1(DistributedDBDataGenerator::STORE_ID_1, DistributedDBDataGenerator::APP_ID_1, DistributedDBDataGenerator::USER_ID_1); const static DBParameters g_dbParameter2(DistributedDBDataGenerator::STORE_ID_2, @@ -119,7 +138,8 @@ const static Option g_ncreateDiskEncrypted(false, false, true, DistributedDB::Ci DistributedDBDataGenerator::PASSWD_VECTOR_1); const static Option g_createMemUnencrypted(true, true, false, DistributedDB::CipherType::DEFAULT, DistributedDBDataGenerator::NULL_PASSWD_VECTOR); -static Option g_option = g_createDiskEncrypted; +static Option g_option = g_createDiskUnencrypted; +const std::vector