From ffa37a5726b405daa19c6acc22b81941bc88ea6a Mon Sep 17 00:00:00 2001 From: songyuhang Date: Wed, 15 Mar 2023 17:04:27 +0800 Subject: [PATCH] full update for performance-optimized --- CMakeLists.txt | 53 +- patch/ceph-global-cache-tls.patch | 74 +- patch/ceph-global-cache.patch | 5893 +++++++++++++++---- patch/globalcache-ceph-adaptor-server.patch | 299 +- src/CMakeLists.txt | 11 +- src/ceph_proxy/CMakeLists.txt | 115 +- src/ceph_proxy/CcmAdaptor.cc | 18 +- src/ceph_proxy/CephExport.h | 28 +- src/ceph_proxy/CephMsgr.cc | 837 +++ src/ceph_proxy/CephMsgr.h | 50 + src/ceph_proxy/CephProxy.cc | 271 +- src/ceph_proxy/CephProxy.h | 106 +- src/ceph_proxy/CephProxyFtds.cc | 76 +- src/ceph_proxy/CephProxyFtds.h | 27 +- src/ceph_proxy/CephProxyInterface.cc | 449 +- src/ceph_proxy/CephProxyInterface.h | 1007 ++-- src/ceph_proxy/CephProxyLog.h | 221 +- src/ceph_proxy/CephProxyOp.cc | 14 +- src/ceph_proxy/CephProxyOp.h | 32 +- src/ceph_proxy/ConfigRead.cc | 131 +- src/ceph_proxy/ConfigRead.h | 33 +- src/ceph_proxy/Gcbufferlist.h | 26 +- src/ceph_proxy/PoolContext.h | 130 +- src/ceph_proxy/RadosMonitor.cc | 120 +- src/ceph_proxy/RadosMonitor.h | 120 +- src/ceph_proxy/RadosWorker.cc | 133 +- src/ceph_proxy/RadosWorker.h | 385 +- src/ceph_proxy/RadosWrapper.cc | 1719 +++--- src/ceph_proxy/RadosWrapper.h | 80 +- src/ceph_proxy/RbdWrapper.cc | 1355 ++--- src/ceph_proxy/utils.h | 131 + src/dependency/include/sa_def.h | 56 +- src/dependency/include/sa_export.h | 36 +- src/server_adaptor/CMakeLists.txt | 13 +- src/server_adaptor/ClassHandler.cc | 428 +- src/server_adaptor/ClassHandler.h | 182 +- src/server_adaptor/class_api.cc | 1151 ++-- src/server_adaptor/client_op_queue.h | 9 +- src/server_adaptor/config_read.cpp | 271 +- src/server_adaptor/config_read.h | 34 +- src/server_adaptor/msg_module.cpp | 45 +- src/server_adaptor/msg_module.h | 11 +- src/server_adaptor/msg_perf_record.cc | 106 +- src/server_adaptor/msg_perf_record.h | 54 +- src/server_adaptor/network_module.cpp | 827 ++- src/server_adaptor/network_module.h | 131 +- src/server_adaptor/objclass.h | 47 +- src/server_adaptor/osa.cpp | 529 +- src/server_adaptor/osa.h | 6 +- src/server_adaptor/sa_ftds_osa.h | 45 +- src/server_adaptor/sa_server_dispatcher.cpp | 66 +- src/server_adaptor/sa_server_dispatcher.h | 67 +- src/server_adaptor/salog.cpp | 144 +- src/server_adaptor/salog.h | 83 +- 54 files changed, 11694 insertions(+), 6591 deletions(-) create mode 100644 src/ceph_proxy/CephMsgr.cc create mode 100644 src/ceph_proxy/CephMsgr.h create mode 100644 src/ceph_proxy/utils.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 97cac18..97b6a24 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,10 +4,10 @@ project(ADAPTOR) set(BASE_DIR_UP ${CMAKE_CURRENT_SOURCE_DIR}/..) set(BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) if (NOT DEFINED CEPH_DIR) - set(CEPH_DIR ${PROJECT_SOURCE_DIR}/../../ceph-14.2.8) + set(CEPH_DIR ${PROJECT_SOURCE_DIR}/../../ceph-14.2.8) endif() if (NOT DEFINED GLOBAL_CACHE_DIR) - set(GLOBAL_CACHE_DIR ${PROJECT_SOURCE_DIR}/../global_cache) + set(GLOBAL_CACHE_DIR ${PROJECT_SOURCE_DIR}/../global_cache) endif() set(GLOBAL_CACHE_INFRAS_LIB_DIR ${GLOBAL_CACHE_DIR}/lib) message(STATUS "CEPH_DIR: " ${CEPH_DIR}) @@ -15,49 +15,50 @@ message(STATUS "GLOBAL_CACHE_DIR: " ${GLOBAL_CACHE_DIR}) message(STATUS "GLOBAL_CACHE_INFRAS_LIB_DIR: " ${GLOBAL_CACHE_INFRAS_LIB_DIR}) include_directories( - ${CEPH_DIR}/src/ - ${CEPH_DIR}/src/boost/ - ${CEPH_DIR}/src/include/ - ${CEPH_DIR}/build/include/ - ${CEPH_DIR}/build/src/include/ - ${BASE_DIR}/src/dependency/include/ + ${CEPH_DIR}/src/ + ${CEPH_DIR}/src/boost/ + ${CEPH_DIR}/src/include/ + ${CEPH_DIR}/build/include/ + ${CEPH_DIR}/build/src/include/ + ${BASE_DIR}/src/dependency/include/ ) get_property(dirs DIRECTORY ${CMAKE_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) message(STATUS "adaptor include directory = ${dirs}") if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE DEBUG) + set(CMAKE_BUILD_TYPE DEBUG) endif() message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") if(NOT DEFINED USE_ASAN) - set(USE_ASAN False) + set(USE_ASAN False) endif() message(STATUS "USE_ASAN: ${USE_ASAN}") + if (${CMAKE_BUILD_TYPE} STREQUAL "RELEASE") - set(COMPILE_FLAGS "-Wl,-z,relro,-z,now -s -fstack-protector-all -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D__linux__ -Wall -Wtype-limits -Wignored-qualifiers -Winit-self -Wpointer-arith -fno-strict-aliasing -fsigned-char -Wno-unknown-pragmas -rdynamic -ftemplate-depth-1024 -Wnon-virtual-dtor -Wno-unknown-pragmas -Wno-ignored-qualifiers -Wstrict-null-sentinel -Woverloaded-virtual -fno-new-ttp-matching -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fdiagnostics-color=auto -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -O2 -DNDEBUG -fPIE -DHAVE_CONFIG_H -D__CEPH__ -D_REENTRANT -D_THREAD_SAFE -D__STDC_FORMAT_MACROS -std=c++1z -ldl -lrt -lresolv -lpthread") + set(COMPILE_FLAGS "-Wl,-z,relro,-z,now -s -fstack-protector-all -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D__linux__ -Wall -Wtype-limits -Wignored-qualifiers -Winit-self -Wpointer-arith -fno-strict-aliasing -fsigned-char -Wno-unknown-pragmas -rdynamic -ftemplate-depth-1024 -Wnon-virtual-dtor -Wno-unknown-pragmas -Wno-ignored-qualifiers -Wstrict-null-sentinel -Woverloaded-virtual -fno-new-ttp-matching -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fdiagnostics-color=auto -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -O2 -DNDEBUG -fPIE -DHAVE_CONFIG_H -D__CEPH__ -D_REENTRANT -D_THREAD_SAFE -D__STDC_FORMAT_MACROS -std=c++1z -ldl -lrt -lresolv -lpthread") else() - set(COMPILE_FLAGS "-Wl,-z,relro,-z,now -ftrapv -fstack-protector-all -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D__linux__ -Wall -Wtype-limits -Wignored-qualifiers -Winit-self -Wpointer-arith -fno-strict-aliasing -fsigned-char -Wno-unknown-pragmas -rdynamic -ftemplate-depth-1024 -Wnon-virtual-dtor -Wno-ignored-qualifiers -Wstrict-null-sentinel -Woverloaded-virtual -fno-new-ttp-matching -DCEPH_DEBUG_MUTEX -fstack-protector-strong -fdiagnostics-color=auto -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -g -O0 -fPIE -DHAVE_CONFIG_H -D__CEPH__ -D_REENTRANT -D_THREAD_SAFE -D__STDC_FORMAT_MACROS -std=c++1z ") + set(COMPILE_FLAGS "-Wl,-z,relro,-z,now -ftrapv -fstack-protector-all -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D__linux__ -Wall -Wtype-limits -Wignored-qualifiers -Winit-self -Wpointer-arith -fno-strict-aliasing -fsigned-char -Wno-unknown-pragmas -rdynamic -ftemplate-depth-1024 -Wnon-virtual-dtor -Wno-ignored-qualifiers -Wstrict-null-sentinel -Woverloaded-virtual -fno-new-ttp-matching -fstack-protector-strong -fdiagnostics-color=auto -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -g -O0 -fPIE -DHAVE_CONFIG_H -D__CEPH__ -D_REENTRANT -D_THREAD_SAFE -D__STDC_FORMAT_MACROS -std=c++1z ") endif() set(CMAKE_CXX_FLAGS "${COMPILE_FLAGS}") if(NOT DEFINED UT) - set(UT False) + set(UT False) endif() message(STATUS "UT: ${UT}") if(NOT DEFINED CLASS_PATH) - set(CLASS_PATH "/opt/gcache/lib") + set(CLASS_PATH "/opt/gcache/lib") endif() message(STATUS "CLASS_PATH: ${CLASS_PATH}") if(NOT DEFINED CPU_TYPE) - set(CPU_TYPE "arm_64") + set(CPU_TYPE "arm_64") endif() message(STATUS "CPU_TYPE: ${CPU_TYPE}") -# +#设置默认的so输出路径 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${BASE_DIR}/build/lib)#.a set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${BASE_DIR}/build/lib)#.so @@ -78,22 +79,24 @@ set(INDEX_STUB_LIBS -lindex_ccdb_client_stub -lkvs_ctrl_stub) set(CACHE_LIBS -lcache) set(CACHE_DEP_LIBS - -ldl -liod -ldpdiagnose -lsecurec -lpatmatch -lftdsclient -ldposax -losax_util - -ldpumm_cmm -ldpumm_mm -ldptracepoint -llwt -ldplog -ldif -lmxml -lupf + -ldl -liod -ldpdiagnose -lsecurec -lpatmatch -lftdsclient -ldposax -losax_util + -ldpumm_cmm -ldpumm_mm -ldptracepoint -llwt -ldplog -ldif -lmxml -lupf ) set(EXTRA_LIBS -laio -lstdc++ -lupf -ldif -llz4 -llwt -lscpart_mgr) set(CLI_LIBS -lcli_server_usr) set(CONF_LIBS -lconfparser) if(${UT}) - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${BASE_DIR}/build/lib) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${BASE_DIR}/build/lib) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${BASE_DIR}/build/ut/cache) - add_subdirectory(test) -endif() - + #设置编译ut时 so和bin文件的输出路径 + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${BASE_DIR}/build/lib) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${BASE_DIR}/build/lib) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${BASE_DIR}/build/ut/cache) + add_subdirectory(src) + add_subdirectory(test) +else() message(STATUS "BASE DIRECTORY: ${BASE_DIR}") message(STATUS "LIBRARY DIRECTORY: ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}") -# +# 引入子目录 add_subdirectory(src) +endif() diff --git a/patch/ceph-global-cache-tls.patch b/patch/ceph-global-cache-tls.patch index bc16652..765af30 100644 --- a/patch/ceph-global-cache-tls.patch +++ b/patch/ceph-global-cache-tls.patch @@ -1,18 +1,18 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index f068082..567ea3f 100644 +index f068082a..8c0c8199 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,8 @@ cmake_minimum_required(VERSION 3.5.1) project(ceph CXX C ASM) set(VERSION 14.2.8) -+link_directories(/opt/gcache_adaptor_compile/third_part/lib/) -+include_directories(/opt/gcache_adaptor_compile/third_part/inc/) ++link_directories(/usr/local/gcache_adaptor/lib/) ++include_directories(/usr/local/gcache_adaptor/inc/) if(POLICY CMP0028) cmake_policy(SET CMP0028 NEW) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index 28ec983..40090cb 100644 +index 28ec9835..40090cb9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -394,10 +394,10 @@ if(WITH_DPDK) @@ -29,7 +29,7 @@ index 28ec983..40090cb 100644 set_target_properties(ceph-common PROPERTIES SOVERSION 0 diff --git a/src/msg/async/PosixStack.cc b/src/msg/async/PosixStack.cc -index e9c8d40..527c635 100644 +index e9c8d404..56fb7025 100644 --- a/src/msg/async/PosixStack.cc +++ b/src/msg/async/PosixStack.cc @@ -13,7 +13,6 @@ @@ -82,13 +82,13 @@ index e9c8d40..527c635 100644 + + if (r == 0) { + if (is_handShaked == false) { -+ int ret = SSL_connect(ssl); -+ if (ret > 0) { -+ is_handShaked = true; -+ ldout(cct, 20) << " TLS Socket " << _fd << " reconnect success" << dendl; -+ } else { -+ ldout(cct, 20) << " Socket " << _fd << " reconnect success, but TLS Socket reconnect failed" << dendl; -+ } ++ int ret = SSL_connect(ssl); ++ if (ret > 0) { ++ is_handShaked = true; ++ ldout(cct, 20) << " TLS Socket " << _fd << " reconnect success"<< dendl; ++ } else { ++ ldout(cct, 20) << " socket " << _fd << " reconnect success, but TLS Socket reconnect failed"<< dendl; ++ } + } + connected = true; + return 1; @@ -154,7 +154,7 @@ index e9c8d40..527c635 100644 + if (read_errno == 11) { + ldout(cct, 4) << "TLS read fail, r = " << r << " errno = " << read_errno << "len = " << len << "fd: " << _fd << "ssl_errno: "<< ssl_errno << " ssl_errno_str: " << error_string < " ${third_part_dir}) +link_directories(${third_part_dir}/lib) +endif() @@ -40,10 +40,10 @@ index 28ec983..bf4aa6e 100644 if(WITH_TESTS) diff --git a/src/client_adaptor/CMakeLists.txt b/src/client_adaptor/CMakeLists.txt new file mode 100644 -index 0000000..aba70f7 +index 00000000..75e0a7a3 --- /dev/null +++ b/src/client_adaptor/CMakeLists.txt -@@ -0,0 +1,17 @@ +@@ -0,0 +1,16 @@ +set(client_adaptor_srcs + ClientAdaptorMsg.cc + ClientAdaptorMgr.cc @@ -59,11 +59,10 @@ index 0000000..aba70f7 +set(client_adaptor_dir ${CEPH_INSTALL_PKGLIBDIR}) +install(TARGETS ceph_client_adaptor_plugin DESTINATION ${client_adaptor_dir}) + -+message(STATUS "Global Cache client-adaptor cmake executing...") -+ ++message(STATUS "Global Cache client-adaptor cmake executing...") diff --git a/src/client_adaptor/ClientAdaptorMgr.cc b/src/client_adaptor/ClientAdaptorMgr.cc new file mode 100644 -index 0000000..900be66 +index 00000000..9a4ed2ce --- /dev/null +++ b/src/client_adaptor/ClientAdaptorMgr.cc @@ -0,0 +1,256 @@ @@ -71,41 +70,41 @@ index 0000000..900be66 +* +* Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. +* -+*/ ++ */ + +#include +#include "ClientAdaptorMgr.h" + +void ClientAdaptorMgr::set_init_flag(bool flag){ -+ init_flag = flag; -+ return; ++ init_flag = flag; ++ return; +} + +const bool ClientAdaptorMgr::is_init_succeed(){ -+ if(init_flag){ -+ return true; -+ } else { -+ return false; -+ } ++ if (init_flag){ ++ return true; ++ } else { ++ return false; ++ } +} + +int32_t CcmPtChangeNotify(PTViewPtEntry *entry, uint32_t entryNum, void *ctx) +{ -+ if (entryNum == 0) { -+ return RET_OK; -+ } -+ std::vector normal_pt; -+ for (uint32_t i = 0; i < entryNum; i++) { -+ if (entry[i].state == CCM_PT_STATE_OK) { -+ normal_pt.push_back(entry[i].ptId); -+ } -+ } -+ if (normal_pt.size() == 0) { -+ return RET_OK; -+ } -+ Objecter *obj = static_cast(ctx); -+ obj->retry_op_submit(normal_pt); ++ if (entryNum == 0) { ++ return RET_OK; ++ } ++ std::vector normal_pt; ++ for (uint32_t i = 0; i < entryNum; i++) { ++ if (entry[i].state == CCM_PT_STATE_OK) { ++ normal_pt.push_back(entry[i].ptId); ++ } ++ } ++ if (normal_pt.size() == 0) { + return RET_OK; ++ } ++ Objecter *obj = static_cast(ctx); ++ obj->retry_op_submit(normal_pt); ++ return RET_OK; +} + +int32_t CcmNodeChangeNotify(int32_t clusterId, NodeInfo *nodeList, uint32_t nodeNum, void *ctx) { @@ -159,85 +158,85 @@ index 0000000..900be66 +} + +/* -+ * Init manager , failed to return !0 -+*/ ++ * Init manager , failed to return !0 ++ */ +int32_t ClientAdaptorCcm::init_mgr(Objecter *obj){ -+ int32_t ret = 0; -+ //Already init -+ if (is_init_succeed()){ -+ if (!ccm_callback_register(obj)) { -+ return RET_CCM_REGISTER_ERROR; -+ } -+ return RET_OK; ++ int32_t ret = 0; ++ // Already init ++ if (is_init_succeed()){ ++ if (!ccm_callback_register(obj)) { ++ return RET_CCM_REGISTER_ERROR; + } ++ return RET_OK; ++ } + -+ ret = OpenBcmInit(); -+ if (ret != 0) { -+ std::cout << __func__ << " ccm agint init failed, ret=" << ret << std::endl; -+ return RET_CCM_AGENT_INIT_ERROR; -+ } ++ ret = OpenBcmInit(); ++ if (ret != 0) { ++ std::cout << __func__ << "ccm agint init failed, ret=" << ret << std::endl; ++ return RET_CCM_AGENT_INIT_ERROR; ++ } + -+ if (!ccm_callback_register(obj)){ -+ return RET_CCM_REGISTER_ERROR; -+ } -+ return RET_OK; ++ if (!ccm_callback_register(obj)) { ++ return RET_CCM_REGISTER_ERROR; ++ } ++ return RET_OK; +} + +int32_t ClientAdaptorCcm::get_pt_num(int32_t clusterId, uint32_t& num){ -+ num = OpenGetTotalPtNum(clusterId); -+ return RET_OK; ++ num = OpenGetTotalPtNum(clusterId); ++ return RET_OK; +} + +int32_t ClientAdaptorCcm::get_pt_entry(int32_t clusterId, uint32_t pt_index, PTViewPtEntry* entry){ -+ if (OpenGetPtEntry(clusterId, pt_index, entry)){ -+ std::cout << __func__ << " Client Adaptor: Get PT entry failed" << std::endl; -+ return RET_CCM_PT_ENTRY_ERROR; -+ } -+ return RET_OK; ++ if (OpenGetPtEntry(clusterId, pt_index, entry)){ ++ std::cout << __func__ << " Client Adaptor: Get PT entry failed" << std::endl; ++ return RET_CCM_PT_ENTRY_ERROR; ++ } ++ return RET_OK; +} + +int32_t ClientAdaptorCcm::get_node_info(int32_t clusterId, uint32_t node_id, NodeInfo* node_info){ -+ using namespace std::chrono; -+ int32_t ret = 0; -+ seconds timeout {50 * 60}; -+ -+ steady_clock::time_point begin = steady_clock::now(); -+ while (true) { -+ ret = OpenAgentGetNodeInfo(clusterId, node_id, node_info); -+ if (ret == 0) { -+ break; -+ } ++ using namespace std::chrono; ++ int32_t ret = 0; ++ seconds timeout {50 * 60}; // 50 minutes ++ ++ steady_clock::time_point begin = steady_clock::now(); ++ while (true) { ++ ret = OpenAgentGetNodeInfo(clusterId, node_id, node_info); ++ if (ret == 0) { ++ break; ++ } + -+ steady_clock::time_point end = steady_clock::now(); -+ seconds time_span = duration_cast(end - begin); ++ steady_clock::time_point end = steady_clock::now(); ++ seconds time_span = duration_cast(end - begin); + -+ if (time_span > timeout) { -+ std::cout << __func__ << " Client Adaptor: get node info failed because of timeout" -+ << ", timeout=" << timeout << std::endl; -+ break; -+ } -+ sleep(1); -+ } -+ if (ret){ -+ std::cout << __func__ << " Client Adaptor: Get node infomation failed" << std::endl; -+ return RET_CCM_NODE_INFO_ERROR; -+ } ++ if (time_span > timeout) { ++ std::cout << __func__ << " Client Adaptor: get node info failed because of timeout" ++ << ", timeout=" << timeout << std::endl; ++ break; ++ } ++ sleep(1); ++ } ++ if (ret){ ++ std::cout << __func__ << " Client Adaptor: Get node infomation failed" << std::endl; ++ return RET_CCM_NODE_INFO_ERROR; ++ } + -+ return RET_OK; ++ return RET_OK; +} + -+bool ClientAdaptorCcm::get_pt_status(int32_t clusterId, uint32_t pt_id) { -+ bool ret = true; -+ PTViewPtEntry pt_entry = { 0 }; -+ if (get_pt_entry(clusterId, pt_id, &pt_entry)) { -+ std::cout << __func__ << " Client Adaptor: Get PT entry failed" << std::endl; -+ return false; -+ } ++bool ClientAdaptorCcm::get_pt_status(int32_t clusterId, uint32_t pt_id){ ++ bool ret = true; ++ PTViewPtEntry pt_entry = { 0 }; ++ if (get_pt_entry(clusterId, pt_id, &pt_entry)){ ++ std::cout << __func__ << " Client Adaptor: Get PT entry failed" << std::endl; ++ return false; ++ } + -+ if (pt_entry.state != CCM_PT_STATE_OK) { -+ return false; -+ } -+ return ret; ++ if (pt_entry.state != CCM_PT_STATE_OK) { ++ return false; ++ } ++ return ret; +} + +int32_t ClientAdaptorCcm::add_snap_to_gc(int64_t md_pool_id, @@ -270,7 +269,7 @@ index 0000000..900be66 + int32_t ret = OpenReleaseImageResource(data_pool_id, image_id.c_str()); + if (ret < 0) { + std::cout << __func__ << " Client Adaptor: Remove image resource failed, image_id=" << image_id -+ << " ret="<< ret << std::endl; ++ << " ret="<< ret << std::endl; + return ret; + } + return 0; @@ -295,7 +294,7 @@ index 0000000..900be66 + info.snapSeq = snap_seq; + int ret = OpenRollbackSnapshot(&info); + if (ret < 0) { -+ std::cout << __func__ << "Client Adaptor: register rollback list failed, snap=" ++ std::cout << __func__ << " Client Adaptor: register rollback list failed, snap=" + << md_pool_id << "-" << data_pool_id << "/" << image_id + << "@" << rb_snap_id << " ret="<< ret << std::endl; + return ret; @@ -307,8 +306,8 @@ index 0000000..900be66 +{ + int ret = OpenImageBusy(data_pool_id, image_id.c_str()); + if (ret < 0) { -+ std::cout << __func__ << "Client Adaptor: query gc rollbacking failed, image=" << md_pool_id -+ << "(" << data_pool_id << ")/" < register_objs; -+ std::map register_node_change_objs; ++ std::map register_objs; ++ std::map register_node_change_objs; + bool ccm_callback_register(Objecter *obj); -+}; ++}; + +class ClientAdaptorLocal : public ClientAdaptorMgr { +public: + ClientAdaptorLocal(){} + ~ClientAdaptorLocal() override {} -+ ++ + int32_t init_mgr(Objecter *obj){ + return 0; -+ } -+ ++ } ++ + int32_t get_pt_num(int32_t clusterId, uint32_t& num){ + num = 10; + return 0; + } -+ ++ + int32_t get_pt_entry(int32_t clusterId, uint32_t pt_index, PTViewPtEntry* entry){ -+ entry->curNodeInfo.nodeId = pt_index % 3; ++ entry->curNodeInfo.nodeId = pt_index % 3; // Assume 3 nodes in one cluster + return 0; + } + @@ -481,14 +480,14 @@ index 0000000..7ea053f + } + + int32_t remove_snap_from_gc(int64_t data_pool_id, const std::string &name_space, -+ const std::string &image_id, uint64_t snap_id) { ++ const std::string &image_id, uint64_t snap_id) { + return 0; + } + -+ int32_t remove_gc_image_resource(const int64_t pool_id, const std::string image_id) { -+ return 0; ++ int32_t remove_gc_image_resource(const int64_t pool_id, const std::string image_id) { ++ return 0; + } -+ int32_t get_node_from_ma(const int64_t pool_id, int32_t *nodeId) { ++ int32_t get_node_from_ma(const int64_t pool_id, int32_t *nodeId) { + return 0; + } + int32_t rollback_gc_snap(int64_t pool_id, int64_t data_pool_id, @@ -508,7 +507,7 @@ index 0000000..7ea053f + const std::string name() override { + return "ClientAdaptorLocal"; + } -+ bool get_pt_status(int32_t clusterId, uint32_t pt_id) { ++ bool get_pt_status(int32_t clusterid, uint32_t pt_id) { + return true; + } + void ccm_deregister(Objecter *obj) {} @@ -520,15 +519,15 @@ index 0000000..7ea053f +#endif diff --git a/src/client_adaptor/ClientAdaptorMsg.cc b/src/client_adaptor/ClientAdaptorMsg.cc new file mode 100644 -index 0000000..ee19bbd +index 00000000..b431167f --- /dev/null +++ b/src/client_adaptor/ClientAdaptorMsg.cc -@@ -0,0 +1,348 @@ +@@ -0,0 +1,363 @@ +/* License:LGPL-2.1 +* +* Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. +* -+*/ ++ */ + +#include +#include @@ -540,8 +539,8 @@ index 0000000..ee19bbd +const int RBD_DATA_OBJECT_NAME_FILTER_LEN = 8; // rbd_data(8 bits) +const int RBD_DATA_OBJECT_NAME_LEN = 27; // rbd_data(8 bits).image_id(n bits).object_index(16 bits) +const string RBD_DATA_OBJECT_NAME = "rbd_data"; -+const int RGW_BUCKET_ID_LEN = 36; // bucket_id(36 bits) -+const int RGW_OBJECT_NAME_LEN = 38; // bucket_id(36 bits)_Object_Name(n bits) ++const int RGW_BUCKET_ID_LEN = 36; // bucket_id(36 bits) ++const int RGW_OBJECT_NAME_LEN = 38; // bucket_id(36 bits)_Object_Name(n bits) +const uint64_t SEGMENT_SIZE = 4194304; +const uint64_t SEGMENT_MASK = 0x3FFFFF; +const int OBJECT_ID_LEN = 16; @@ -555,15 +554,15 @@ index 0000000..ee19bbd + +void ClientAdaptorMsg::push_strategy(Objecter *objecter, uint64_t pool_id, int32_t node_id, std::string oid_name, bufferlist &indata) +{ -+ if (objecter == NULL) { -+ std::cout << __func__ << " Objecter null " << std::endl; ++ if (objecter == NULL){ ++ std::cout << __func__ << " objecter null " << std::endl; + return; + } -+ if (oid_name.size() == 0) { ++ if (oid_name.size() == 0){ + std::cout << __func__ << " oid_name size 0 " << std::endl; + return; + } -+ if (node_id < 0) { ++ if ( node_id < 0){ + std::cout << __func__ << " node id < 0 " << std::endl; + return; + } @@ -578,8 +577,8 @@ index 0000000..ee19bbd + op.indata.append(cls, op.op.cls.class_len); + op.indata.append(method, op.op.cls.method_len); + op.indata.append(indata); -+ Objecter::Op *objecter_op = -+ new Objecter::Op(object_t(oid_name), object_locator_t(), nops, CEPH_OSD_FLAG_EXEC, NULL, NULL, NULL, nullptr); ++ Objecter::Op *objecter_op = ++ new Objecter::Op(object_t(oid_name), object_locator_t(), nops, CEPH_OSD_FLAG_EXEC, NULL, NULL, NULL, nullptr); + objecter_op->target.osd = node_id; + objecter_op->target.base_oloc.pool = pool_id; + objecter_op->target.base_oid.name = oid_name; @@ -591,9 +590,14 @@ index 0000000..ee19bbd +/** + * Maximum string length of the RBD block object name prefix (not including + * null termination). ++ * ++ * v1 format: rb... ++ * v2 format: rbd_data.[.] ++ * ++ * Note: new features might require increasing this maximum prefix length. + */ +bool ClientAdaptorMsg::filter_msg(Objecter::op_target_t *t){ -+ string obj_name = t->base_oid.name; ++ string obj_name = t->base_oid.name; + if (obj_name.size() < RBD_DATA_OBJECT_NAME_LEN) { + return false; + } @@ -601,7 +605,7 @@ index 0000000..ee19bbd + if (obj_name.compare(0, RBD_DATA_OBJECT_NAME_FILTER_LEN, RBD_DATA_OBJECT_NAME) == 0){ + return true; + } -+ ++ + return false; +} + @@ -617,18 +621,23 @@ index 0000000..ee19bbd + return false; +} + ++/* ++ * Hash to get node id. Failed to return !0 ++ */ +int32_t ClientAdaptorMsg::get_node_id(int32_t clusterId, string obj_name, int64_t pool_id, uint32_t& pt_id){ -+ if (obj_name.length() == 0){ -+ std::cout << __func__ << " Client Adaptor: input parameter invalid!" << std::endl; ++ if (obj_name.length() == 0 ){ ++ std::cout << __func__ << " Client Adaptor: input parameter invalid!" << std::endl; + return -RET_CCM_PARAM_ERROR; + } + ++ // Hash obj_name+pool_name to hashed id + string obj_id = to_string(pool_id); + obj_id += '_'; + obj_id += obj_name; + hash hash_str; + uint32_t obj_hashed_id = hash_str(obj_id); + ++ // Calculate node id + uint32_t pt_num = 0; + NodeInfo info = {0}; + @@ -654,15 +663,16 @@ index 0000000..ee19bbd + node_id += clusterId << CLUSTER_ID_OFFSET; + + if (mgr_ref->get_node_info(clusterId, pt_entry.curNodeInfo.nodeId, &info)){ -+ std::cout << __func__ << " Client Adaptor: Get node info failed. node id " << pt_entry.curNodeInfo.nodeId << std::endl; ++ std::cout << __func__ << " Client Adaptor: Get node info failed. node id " << pt_entry.curNodeInfo.nodeId << std::endl; + return -RET_CCM_NODE_INFO_ERROR; + } + if (info.portNum > PORT_SUPPORT_MAX || info.portNum == 0) { -+ std::cout << __func__ << " Client Adaptor: Port number invalid. Port Number: " << info.portNum << std::endl; ++ std::cout << __func__ << " Client Adaptor: Port number invalid. Port Number: " << info.portNum << std::endl; + return -RET_CCM_PORT_NUM_ERROR; + } + uint32_t pt_index = pt_entry.indexInNode; + ++ // port index encode in OSD + node_id += pt_index % info.portNum; + + return node_id; @@ -670,11 +680,12 @@ index 0000000..ee19bbd + +bool ClientAdaptorMsg::valid_ip(string ip_addr) +{ -+ string regStr = "^((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|[1-9])"\ -+ "(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)){3})|(0.0.0.0)$"; -+ regex regIp(regStr); -+ bool matchValue = regex_match(ip_addr, regIp); -+ return matchValue; ++ string regStr = "^((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|[1-9])"\ ++ ++ "(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)){3})|(0.0.0.0)$"; ++ regex regIp(regStr); ++ bool matchValue = regex_match(ip_addr, regIp); ++ return matchValue; +} + +string ClientAdaptorMsg::gc_snap_prefix() @@ -710,6 +721,9 @@ index 0000000..ee19bbd + rd_snap_name.append(to_string(timepoint)); +} + ++/* ++ * Get node ip from MA. Failed to return !0 ++ */ +int32_t ClientAdaptorMsg::get_node_ip(int32_t clusterId, uint32_t node_index, string& node_ip){ + uint32_t node_id = (node_index & NODE_ID_MASK) >> NODE_ID_OFFSET_BIT; + uint32_t port_index = node_index & PORT_INDEX_MASK; @@ -720,13 +734,13 @@ index 0000000..ee19bbd + } + uint32_t port = info.ports[port_index]; + if (port < GC_PORT_MIN || port > GC_PORT_MAX) { -+ return RET_CCM_PORT_NUM_ERROR; ++ return RET_CCM_PORT_NUM_ERROR; + } + + string server_ip = info.publicAddrStr; + + if (!valid_ip(server_ip)) { -+ return RET_CCM_IP_ERROR; ++ return RET_CCM_IP_ERROR; + } + + string addr_str = "tcp://"; @@ -775,13 +789,13 @@ index 0000000..ee19bbd + } + ClientAdaptorMsg *msg_ref = static_cast(params->handle); + Objecter *obj = static_cast(params->ctx); -+ if (msg_ref->is_valid_object(obj) ==false) { -+ return; ++ if (msg_ref->is_valid_object(obj) == false) { ++ return; + } + uint64_t offset = params->offset & SEGMENT_MASK; + int id = params->objId; + uint64_t left = params->len; -+ while(left) { ++ while (left) { + uint64_t max = std::min(SEGMENT_SIZE - offset, left); + + char buff[params->imageIdLen+OBJECT_ID_LEN + 1]; @@ -818,7 +832,7 @@ index 0000000..ee19bbd + + rc = OpenRcacheCeateDasModule(this, dasInstanceParam); + if (rc) { -+ return -1; ++ return -1; + } + initialized = true; + return 0; @@ -830,19 +844,19 @@ index 0000000..ee19bbd + return 0; + DasKvParam *params[op->ops.size()]; + -+ if((op->target.flags & CEPH_OSD_FLAG_WRITE) == CEPH_OSD_FLAG_WRITE) -+ return 0; ++ if ((op->target.flags & CEPH_OSD_FLAG_WRITE) == CEPH_OSD_FLAG_WRITE) ++ return 0; + string obj_name = op->target.base_oid.name; + if (obj_name.compare(0, RBD_DATA_OBJECT_NAME_FILTER_LEN, RBD_DATA_OBJECT_NAME)) + return 0; + std::size_t found = obj_name.find_last_of('.'); -+ if(found == std::string::npos) ++ if (found == std::string::npos) + return -EINVAL; + uint64_t objId = std::stol(obj_name.substr(found+1, OBJECT_ID_LEN), nullptr, 16); + uint64_t ns = ceph_clock_now().to_nsec(); + int i = 0; -+ for(vector::iterator p = op->ops.begin(); p != op->ops.end(); ++p) { -+ if (p->op.op == CEPH_OSD_OP_READ || p->op.op == CEPH_OSD_OP_SPARSE_READ || p->op.op == CEPH_OSD_OP_SYNC_READ) { ++ for (vector::iterator p = op->ops.begin(); p != op->ops.end(); ++p) { ++ if (p->op.op == CEPH_OSD_OP_READ || p->op.op == CEPH_OSD_OP_SPARSE_READ || p->op.op == CEPH_OSD_OP_SYNC_READ) { + params[i] = reinterpret_cast(new char[sizeof(DasKvParam) + found + 1]); + params[i]->offset = p->op.extent.offset; + params[i]->len = p->op.extent.length; @@ -851,7 +865,7 @@ index 0000000..ee19bbd + params[i]->cephPoolId = op->target.base_oloc.pool; + params[i]->algType = DAS_ALG_SEQ; + params[i]->objId = objId; -+ params[i]->imageIdLen =found + 1; ++ params[i]->imageIdLen = found + 1; + params[i]->clusterId = clusterId; + memcpy(params[i]->imageIdBuf, obj_name.c_str(), params[i]->imageIdLen); + params[i]->handle = this; @@ -863,7 +877,7 @@ index 0000000..ee19bbd + if (i) { + int rc = OpenRcachePutDasInfo(params, i); + if (rc) -+ return -1; ++ return -1; + } + return 0; +} @@ -874,7 +888,7 @@ index 0000000..ee19bbd +} diff --git a/src/client_adaptor/ClientAdaptorMsg.h b/src/client_adaptor/ClientAdaptorMsg.h new file mode 100644 -index 0000000..80afa78 +index 00000000..f367dd2a --- /dev/null +++ b/src/client_adaptor/ClientAdaptorMsg.h @@ -0,0 +1,87 @@ @@ -882,7 +896,7 @@ index 0000000..80afa78 +* +* Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. +* -+*/ ++ */ + +#ifndef CLIENT_ADAPTOR_MSG_H +#define CLIENT_ADAPTOR_MSG_H @@ -897,8 +911,8 @@ index 0000000..80afa78 + +class ClientAdaptorMsg { +public: -+ ClientAdaptorMsg(ClientAdaptorMgr* mgr); -+ ~ClientAdaptorMsg() {}; ++ ClientAdaptorMsg(ClientAdaptorMgr* mgr); ++ ~ClientAdaptorMsg() {}; + + void push_strategy(Objecter *objecter, uint64_t pool_id, int32_t node_id, std::string oid_name, bufferlist &indata); + @@ -915,7 +929,7 @@ index 0000000..80afa78 + bool filter_msg(Objecter::op_target_t *t); + + bool filter_msg_by_op(Objecter::Op *op); -+ ++ + const string name() {return "ClientAdaptorMsg";} + + bool is_node(uint32_t index); @@ -933,7 +947,7 @@ index 0000000..80afa78 + bool is_valid_object(Objecter *obj) { + auto it = das_objs.find(obj); + if (it != das_objs.end()) -+ return true; ++ return true; + return false; + } + @@ -947,7 +961,7 @@ index 0000000..80afa78 + +protected: + ClientAdaptorMgr* mgr_ref; -+ ++ // FLAG:20 (true means global cache node) Node ID:4~15 PORT INDEX ID:0~3 + const int FLAG_OFFSET_BIT = 20; + const int CLUSTER_ID_OFFSET = 16; + const int PORT_INDEX_MASK = 0xf; @@ -967,7 +981,7 @@ index 0000000..80afa78 +#endif diff --git a/src/client_adaptor/ClientAdaptorPerf.cc b/src/client_adaptor/ClientAdaptorPerf.cc new file mode 100644 -index 0000000..cea7835 +index 00000000..90f5e1e6 --- /dev/null +++ b/src/client_adaptor/ClientAdaptorPerf.cc @@ -0,0 +1,85 @@ @@ -975,7 +989,7 @@ index 0000000..cea7835 +* +* Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. +* -+*/ ++ */ + +#include "ClientAdaptorPerf.h" +#include "ClientAdaptorPlugin.h" @@ -993,18 +1007,18 @@ index 0000000..cea7835 + gettimeofday(&(op->perf_tick.end), NULL); + return; +} -+ ++ +void ClientAdaptorPerf::record_op(Objecter::Op *op) { + for (vector::iterator p = op->ops.begin(); p != op->ops.end(); ++p) { -+ if (p->op.op == CEPH_OSD_OP_READ || p->op.op == CEPH_OSD_OP_SPARSE_READ || p->op.op == CEPH_OSD_OP_SYNC_READ) { -+ read.op_count++; -+ read.time_cost += (op->perf_tick.end.tv_sec - op->perf_tick.start.tv_sec) * 1000 * 1000 + \ -+ (op->perf_tick.end.tv_usec - op->perf_tick.start.tv_usec); -+ } else if (p->op.op == CEPH_OSD_OP_WRITE || p->op.op == CEPH_OSD_OP_WRITEFULL) { -+ write.op_count++; -+ write.time_cost += (op->perf_tick.end.tv_sec - op->perf_tick.start.tv_sec) * 1000 * 1000 + \ -+ (op->perf_tick.end.tv_usec-op->perf_tick.start.tv_usec); -+ } ++ if (p->op.op == CEPH_OSD_OP_READ || p->op.op == CEPH_OSD_OP_SPARSE_READ || p->op.op == CEPH_OSD_OP_SYNC_READ) { ++ read.op_count++; ++ read.time_cost += (op->perf_tick.end.tv_sec - op->perf_tick.start.tv_sec) * 1000 * 1000 + \ ++ (op->perf_tick.end.tv_usec - op->perf_tick.start.tv_usec); ++ } else if (p->op.op == CEPH_OSD_OP_WRITE || p->op.op == CEPH_OSD_OP_WRITEFULL) { ++ write.op_count++; ++ write.time_cost += (op->perf_tick.end.tv_sec - op->perf_tick.start.tv_sec) * 1000 * 1000 + \ ++ (op->perf_tick.end.tv_usec - op->perf_tick.start.tv_usec); ++ } + } + return; +} @@ -1024,41 +1038,41 @@ index 0000000..cea7835 + float avg_flight = 0; + + while (!tick_done) { -+ sleep (3); -+ read_cnt = plugin->perf_ref->read.op_count; -+ read_cost = plugin->perf_ref->read.time_cost; -+ write_cnt = plugin->perf_ref->write.op_count; -+ write_cost = plugin->perf_ref->write.time_cost; -+ if (read_cnt != 0) { -+ read_lat = read_cost/read_cnt; -+ } -+ if (write_cnt != 0) { -+ write_lat = write_cost/write_cnt; -+ } -+ if ((read_cnt + write_cnt) != 0 ) { -+ avg_flight = (float)total_in_flight / (float)(read_cnt + write_cnt); -+ } -+ outfile << "*************************************************************************" << std::endl; -+ outfile << "PID: " << getpid() << " TID: " << gettid() << std::endl; -+ outfile << " total_count avg_latency(us)" << std::endl; -+ outfile << "read " << setw(16) << read_cnt << " " << setw(16) << read_lat << std::endl; -+ outfile << "write" << setw(16) << write_cnt << " " << setw(16) << write_lat << std::endl; -+ outfile << " total_count average" << std::endl; -+ outfile << "in-flight" << setw(12) << total_in_flight -+ << " " << setw(16) << fixed << setprecision(1) << avg_flight << std::endl; ++ sleep(3); ++ read_cnt = plugin->perf_ref->read.op_count; ++ read_cost = plugin->perf_ref->read.time_cost; ++ write_cnt = plugin->perf_ref->write.op_count; ++ write_cost = plugin->perf_ref->write.time_cost; ++ if (read_cnt != 0) { ++ read_lat = read_cost/read_cnt; ++ } ++ if (write_cnt != 0) { ++ write_lat = write_cost/write_cnt; ++ } ++ if ((read_cnt + write_cnt) != 0) { ++ avg_flight = (float)total_in_flight / (float)(read_cnt + write_cnt); ++ } ++ outfile << "*************************************************************************" << std::endl; ++ outfile << "PID: " << getpid() << " TID: " << gettid() << std::endl; ++ outfile << " total_count avg_latency(us)" << std::endl; ++ outfile << "read " << setw(16) << read_cnt << " " << setw(16) << read_lat << std::endl; ++ outfile << "write" << setw(16) << write_cnt << " " << setw(16) << write_lat << std::endl; ++ outfile << " total_count average" << std::endl; ++ outfile << "in-flight" << setw(12) << total_in_flight ++ << " " << setw(16) << fixed << setprecision(1) << avg_flight << std::endl; + } -+ }; ++ }; +} + +void ClientAdaptorPerf::start_record(ClientAdaptorPlugin* plugin) { -+ std::function perf_thread = create_thread(plugin); -+ threads.push_back(std::thread(perf_thread)); -+ outfile.open("/var/log/ceph/perf_tick.log", ios::out | ios::app); -+ return; ++ std::function perf_thread = create_thread(plugin); ++ threads.push_back(std::thread(perf_thread)); ++ outfile.open("/var/log/ceph/perf_tick.log", ios::out | ios::app); ++ return; +} diff --git a/src/client_adaptor/ClientAdaptorPerf.h b/src/client_adaptor/ClientAdaptorPerf.h new file mode 100644 -index 0000000..f7ee5be +index 00000000..f57b2074 --- /dev/null +++ b/src/client_adaptor/ClientAdaptorPerf.h @@ -0,0 +1,55 @@ @@ -1066,7 +1080,7 @@ index 0000000..f7ee5be +* +* Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. +* -+*/ ++ */ + +#ifndef CLIENT_ADAPTOR_PERF_H +#define CLIENT_ADAPTOR_PERF_H @@ -1088,28 +1102,28 @@ index 0000000..f7ee5be +public: + ClientAdaptorPerf(){} + ~ClientAdaptorPerf(){} -+ -+ ++ // friend class ClientAdaptorPlugin; ++ + struct op_perf_t { + std::atomic op_count{0}; + std::atomic time_cost{0}; + }; + -+void start_tick(Objecter::Op *op); ++ void start_tick(Objecter::Op *op); + -+void end_tick(Objecter::Op *op); -+ -+void record_op(Objecter::Op *op); ++ void end_tick(Objecter::Op *op); ++ ++ void record_op(Objecter::Op *op); + -+void start_record(ClientAdaptorPlugin* plugin); ++ void start_record(ClientAdaptorPlugin* plugin); + -+std::function create_thread(const ClientAdaptorPlugin* plugin); ++ std::function create_thread(const ClientAdaptorPlugin* plugin); + -+const string name() {return "ClientAdaptorPerf";} -+vector threads; -+bool tick_done{false}; -+std::ofstream outfile; -+std::atomic total_in_flight{0}; ++ const string name() {return "ClientAdaptorPerf";} ++ vector threads; ++ bool tick_done{false}; ++ std::ofstream outfile; ++ std::atomic total_in_flight{0}; +private: + struct op_perf_t read; + struct op_perf_t write; @@ -1119,7 +1133,7 @@ index 0000000..f7ee5be +#endif diff --git a/src/client_adaptor/ClientAdaptorPlugin.cc b/src/client_adaptor/ClientAdaptorPlugin.cc new file mode 100644 -index 0000000..c44e348 +index 00000000..88365c25 --- /dev/null +++ b/src/client_adaptor/ClientAdaptorPlugin.cc @@ -0,0 +1,45 @@ @@ -1127,7 +1141,7 @@ index 0000000..c44e348 +* +* Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. +* -+*/ ++ */ + +#include +#include "ClientAdaptorPlugin.h" @@ -1150,10 +1164,10 @@ index 0000000..c44e348 + return CEPH_GIT_NICE_VER; +} + -+ ++// plugin load时调用的初始化函数 +int __ceph_plugin_init(CephContext *cct, -+ const std::string& type, -+ const std::string& name) ++ const std::string& type, ++ const std::string& name) +{ + PluginRegistry *instance = cct->get_plugin_registry(); + if (cct->_conf.get_val("global_cache_debug_mode")){ @@ -1170,7 +1184,7 @@ index 0000000..c44e348 +} diff --git a/src/client_adaptor/ClientAdaptorPlugin.h b/src/client_adaptor/ClientAdaptorPlugin.h new file mode 100644 -index 0000000..fa7d71d +index 00000000..3509005f --- /dev/null +++ b/src/client_adaptor/ClientAdaptorPlugin.h @@ -0,0 +1,39 @@ @@ -1178,7 +1192,7 @@ index 0000000..fa7d71d +* +* Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. +* -+*/ ++ */ + +#ifndef CLIENT_ADAPTOR_PLUGIN_H +#define CLIENT_ADAPTOR_PLUGIN_H @@ -1196,18 +1210,18 @@ index 0000000..fa7d71d + +class ClientAdaptorPlugin : public Plugin { +public: -+ ClientAdaptorPlugin(CephContext* cct, ClientAdaptorMsg* msg, ClientAdaptorMgr* mgr, ++ ClientAdaptorPlugin(CephContext* cct, ClientAdaptorMsg* msg, ClientAdaptorMgr* mgr, + ClientAdaptorPerf* perf) : Plugin(cct), msg_ref(msg), mgr_ref(mgr), perf_ref(perf) + { + } -+ ++ + ~ClientAdaptorPlugin(); + + ClientAdaptorMsg* msg_ref; + ClientAdaptorMgr* mgr_ref; -+ ClientAdaptorPerf* perf_ref; ++ ClientAdaptorPerf* perf_ref; + -+ const string name() { ++ const string name(){ + return "ClientAdaptorPlugin"; + } +}; @@ -1215,24 +1229,24 @@ index 0000000..fa7d71d +#endif diff --git a/src/client_adaptor/open_ccm.h b/src/client_adaptor/open_ccm.h new file mode 100644 -index 0000000..2fefc7e +index 00000000..db87f7d1 --- /dev/null +++ b/src/client_adaptor/open_ccm.h -@@ -0,0 +1,209 @@ -+/* -+ * Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. +@@ -0,0 +1,248 @@ ++/* ++Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. + -+ * Licensed under the Apache License, Version 2.0 (the "License"); -+ * you may not use this file except in compliance with the License. -+ * You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ ++ http://www.apache.org/licenses/LICENSE-2.0 ++ ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. +*/ +#ifndef __CCM_INTERFACE_H__ +#define __CCM_INTERFACE_H__ @@ -1247,32 +1261,35 @@ index 0000000..2fefc7e + +#define PT_VIEW_NODE_MAX_DOMAIN 32 +#define PT_VIEW_MAX_POOL 256 -+#define MAX_CCM_CTRL_NODE_NUM 128 ++#define MAX_CCM_CTRL_NODE_NUM 128 /* 管理节点数量 */ + -+#define MAX_PT_ENTRY 1024 ++#define MAX_PT_ENTRY 1024 /* PT最大数量 */ + -+#define MAX_SERVER_NUM MAX_CCM_CTRL_NODE_NUM ++#define MAX_SERVER_NUM MAX_CCM_CTRL_NODE_NUM /* 节点最大数量 */ + -+#define CCM_MAX_DISK_NUM 1024 -+#define CCM_VNODE_NUM_PER_NODE 8 -+#define MAX_DISK_NUM_PER_NODE 16 -+#define IP_ADDR_LEN (16) -+#define DISK_NAME_LEN (64) -+#define DISK_SN_LEN (64) ++#define CCM_MAX_DISK_NUM 1024 /* 最大磁盘数量 */ ++#define CCM_VNODE_NUM_PER_NODE 8 /* 每个节点上最大的VNODE数量 */ ++#define MAX_DISK_NUM_PER_NODE 16 /* 每个节点上最大的磁盘数量 */ ++#define IP_ADDR_LEN (16) /* IP地址长度 */ ++#define DISK_NAME_LEN (64) /* 磁盘名长度 */ ++#define DISK_SN_LEN (64) /* 磁盘SN长度 */ + -+#define MAX_POOL_NAME_LEN (256) -+#define CCM_MAX_POOL_NUM (4096) -+#define MAX_PORT_NUM (8) ++#define MAX_POOL_NAME_LEN (256) /* POOLName的最大长度 */ ++#define CCM_MAX_POOL_NUM (4096) /* POOL的数量限制 */ ++#define MAX_PORT_NUM (8) /* 每个节点支持的对外监听接口数 */ + -+#define ZK_IP_ADDR_LEN (16) -+#define ZK_DISK_NAME_LEN (64) -+#define ZK_DISK_SN_LEN (64) ++#define ZK_IP_ADDR_LEN (16) ++#define ZK_DISK_NAME_LEN (64) ++#define ZK_DISK_SN_LEN (64) ++#define MAX_CLUSTER_NETWROK_NUM 4 + ++// Upgrade embedding +#define CCM_VERSION_0 0 +#define CCM_VERSION_1 1 +#define CCM_CURRENT_VERSION CCM_VERSION_0 +#define CCM_Upgrade_Embedding_LEN 64 + ++/* Ndoe & Disk Info */ +typedef enum { + NODE_STATE_INVALID = 0, + NODE_STATE_UP = 1, @@ -1304,6 +1321,7 @@ index 0000000..2fefc7e + uint32_t capacity; + uint32_t usedCap; + VdiskState state; ++ VdiskState inOutState; /* 磁盘的IN OUT状态 */ + bool isFirstFormat; +} VdiskInfo; + @@ -1312,24 +1330,25 @@ index 0000000..2fefc7e + uint32_t nodeId; + NodeState state; + uint32_t ipv4addr; -+ char ipv4AddrStr[IP_ADDR_LEN]; -+ char publicAddrStr[IP_ADDR_LEN]; -+ char clusterAddrStr[IP_ADDR_LEN]; -+ int32_t portNum; -+ uint32_t ports[MAX_PORT_NUM]; -+ uint32_t diskNum; -+ VdiskInfo diskList[MAX_DISK_NUM_PER_NODE]; -+ uint64_t version; -+ NodeState inOutState; -+ NodeState runningState; ++ char ipv4AddrStr[IP_ADDR_LEN]; /* 用于管理的IP地址,与zookeeper通信使用的IP地址 */ ++ char publicAddrStr[IP_ADDR_LEN]; /* 前端IO使用的IP地址，即Client与ServerAdaptor一起使用的地址 */ ++ char clusterAddrStr[MAX_CLUSTER_NETWROK_NUM][IP_ADDR_LEN]; /* 后端集群通信IP地址，即Plog 3副本之间使用的IP地址 */ ++ uint32_t clusterIpNum; /* 后端集群的个数 */ ++ int32_t portNum; /* 有效的端口数量 */ ++ uint32_t ports[MAX_PORT_NUM]; /* port */ ++ uint32_t diskNum; /* 磁盘数量 */ ++ VdiskInfo diskList[MAX_DISK_NUM_PER_NODE]; /* 磁盘列表 */ ++ uint64_t version; /* 版本号 */ ++ NodeState inOutState; /* 节点的IN OUT状态 */ ++ NodeState runningState; /* 节点的开工状态 */ +} NodeInfo; + +typedef enum { -+ CCM_PT_STATE_INIT=0, -+ CCM_PT_STATE_OK, -+ CCM_PT_STATE_TRIM, -+ CCM_PT_STATE_REPLAY, -+ CCM_PT_STATE_FAULT, ++ CCM_PT_STATE_INIT = 0, ++ CCM_PT_STATE_OK, // OK状态 ++ CCM_PT_STATE_TRIM, // 迁出状态 ++ CCM_PT_STATE_REPLAY, // 迁入状态 ++ CCM_PT_STATE_FAULT, // 失败状态 +} PtState; + +typedef struct { @@ -1342,11 +1361,11 @@ index 0000000..2fefc7e + uint32_t version; + bool ptChange; + uint32_t birthVersion; -+ uint32_t ptId; -+ uint32_t indexInNode; -+ PtState state; -+ PtNodeInfo curNodeInfo; -+ PtNodeInfo srcNodeInfo; ++ uint32_t ptId; // PTID ++ uint32_t indexInNode; // pt在Node内部编号 ++ PtState state; // pt状态 ++ PtNodeInfo curNodeInfo; // 当前所在节点信息 ++ PtNodeInfo srcNodeInfo; // 初始节点信息 + char reserved[CCM_Upgrade_Embedding_LEN]; +} PtInfo; + @@ -1355,18 +1374,18 @@ index 0000000..2fefc7e + uint32_t globalVersion; + uint32_t ptNum; + char reserved[CCM_Upgrade_Embedding_LEN]; -+ PtInfo ptInfo[0]; ++ PtInfo ptInfo[0]; +} PtView; + +typedef PtInfo PTViewPtEntry; + -+/* NodeView callback */ ++/* NodeView变更通知回调函数 */ +typedef struct { + void *ctx; + int32_t (*notifyNodeChange)(int32_t clusterId, NodeInfo *nodeList, uint32_t nodeNum, void *ctx); +} NodeViewChangeOpHandle; + -+/* PTView callback */ ++/* PTView变更通知回调函数 */ +typedef struct { + void *ctx; + int32_t (*notifyPtChange)(PTViewPtEntry *entry, uint32_t entryNum, void *ctx); @@ -1399,20 +1418,54 @@ index 0000000..2fefc7e + +int32_t OpenGetPtEntry(int32_t clusterId, uint32_t ptId, PTViewPtEntry *entry); + ++/* ++ * 功能描述：获取总的PT的数量 ++ * 参数说明：无参数 ++ * 返回值：返回总的PT的数量 ++ */ +uint32_t OpenGetTotalPtNum(int32_t clusterId); + ++/* ++ * 功能描述：Agent模块获取某个指定的节点的NodeInfo信息 ++ * 参数说明：nodeId: {in}, 节点编号 ++ * nodeInfo: {out}, NodeInfo结构体指针 ++ * 返回值：0表示成功，非0表示失败 ++ */ +int32_t OpenAgentGetNodeInfo(int32_t clusterId, uint32_t nodeId, NodeInfo *nodeInfo); + ++/* ++ * 功能描述：注册PTview更新通知与回调函数 ++ * 参数说明：handle: {in}, PTView更新回调函数，在PTView更之后，则会调用该回调函数 ++ * 返回值：0表示成功，非0表示失败 ++ */ +int32_t OpenRegisterViewChangeNotifyChain(PTViewChangeOpHandle *handle); + ++/* ++ * 功能描述：注册NodeView更新通知与回调函数 ++ * 参数说明：handle: {in}, NodeView更新回调函数，在NodeView变更之后，则会调用该回调函数 ++ * 返回值：0表示成功，非0表示失败 ++ */ +int32_t OpenRegisterNodeViewChangeNotifyChain(NodeViewChangeOpHandle *handle); + +void OpenDeregisterViewChangeNotifyChain(PTViewChangeOpHandle *handle); + +void OpenDeregisterNodeViewChangeNotifyChain(NodeViewChangeOpHandle *handle); + ++/* ++ * 功能描述：创建快照视图 ++ * 参数说明：poolId: {in}, pool的id ++ imageId: {in}, image的id ++ SnapId: {in}, snapshot的id ++ * 返回值：0表示成功，非0表示失败 ++ */ +int32_t OpenCreateSnapshot(int64_t mdPoolId, int64_t dataPoolId, const char *imageId, uint64_t snapId); -+ ++/* ++ * 功能描述：删除快照视图 ++ * 参数说明：poolId: {in}, pool的id ++ imageId: {in}, image的id ++ SnapId: {in}, snapshot的id ++ * 返回值：0表示成功，非0表示失败 ++ */ +int32_t OpenDeleteSnapshot(int64_t dataPoolId, const char *nameSpace, const char *imageId, uint64_t snapId); + +int32_t OpenReleaseImageResource(int64_t poolId, const char *imageId); @@ -1427,28 +1480,27 @@ index 0000000..2fefc7e +} +#endif + -+#endif // __CCM_INTERFACE_H__ -\ No newline at end of file ++#endif diff --git a/src/client_adaptor/open_das.h b/src/client_adaptor/open_das.h -new file mode 100644 -index 0000000..6eac9f7 +new file mode 100755 +index 00000000..77e42672 --- /dev/null +++ b/src/client_adaptor/open_das.h -@@ -0,0 +1,67 @@ -+/* -+ * Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. +@@ -0,0 +1,70 @@ ++/* ++Copyright (c) 2021 Huawei Technologies Co., Ltd All rights reserved. + -+ * Licensed under the Apache License, Version 2.0 (the "License"); -+ * you may not use this file except in compliance with the License. -+ * You may obtain a copy of the License at -+ * -+ * http://www.apache.org/licenses/LICENSE-2.0 -+ * -+ * Unless required by applicable law or agreed to in writing, software -+ * distributed under the License is distributed on an "AS IS" BASIS, -+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+ * See the License for the specific language governing permissions and -+ * limitations under the License. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ ++ http://www.apache.org/licenses/LICENSE-2.0 ++ ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. +*/ +#ifndef OPEN_DAS_H +#define OPEN_DAS_H @@ -1464,7 +1516,7 @@ index 0000000..6eac9f7 + RETURN_DAS_OK = 0, + RETURN_DAS_EMPTY = 1, + RETURN_DAS_DELETING = 2, -+} DAS_RESULT; ++}DAS_RESULT; + +typedef enum TagDasAlgType { + DAS_ALG_SEQ = 0, @@ -1474,37 +1526,113 @@ index 0000000..6eac9f7 +} DasAlgType; + +typedef struct TagDasKvParam { -+ uint64_t offset; -+ uint64_t len; -+ uint8_t opcode; -+ uint64_t timeStamp; -+ int64_t cephPoolId; -+ DasAlgType algType; ++ uint64_t offset; /* offset by byte */ ++ uint64_t len; /* io length by byte */ ++ uint8_t opcode; /* kv opcode 0:read/1:write/ */ ++ uint64_t timeStamp; /* timestamp for test. NS */ ++ int64_t cephPoolId; /* ceph pool id */ ++ DasAlgType algType; /* 入参时填0 */ + uint64_t objId; + uint32_t imageIdLen; + int32_t clusterId; -+ void *ctx; -+ void *handle; ++ void* ctx; /* client adaptor 上下文 */ ++ void* handle; + char imageIdBuf[0]; +} DasKvParam; + ++ +typedef struct TagDasOPS { -+ void (*SubmitDasPrefetch)(DasKvParam* params); ++ void (*SubmitDasPrefetch)(DasKvParam* params); // 同步接口 +} DasOPS; + ++/* Param to create DAS */ +typedef struct TagDasModuleParam { -+ DasOPS *ops; ++ DasOPS *ops; // client adaptor注册 +} DasModuleParam; + -+int32_t OpenRcacheCeateDasModule(void *handle, DasModuleParam *createInstanceParam); ++ ++int32_t OpenRcacheCeateDasModule(void* handle, DasModuleParam *createInstanceParam); + +int32_t OpenRcachePutDasInfo(DasKvParam *params[], uint32_t keyNum); + -+void OpenRcacheExitDasModule(void *handle); -+#endif // OPEN_DAS_H -\ No newline at end of file ++void OpenRcacheExitDasModule(void* handle); ++#endif +diff --git a/src/common/PluginRegistry.cc b/src/common/PluginRegistry.cc +index 2cb7fcee..59707481 100644 +--- a/src/common/PluginRegistry.cc ++++ b/src/common/PluginRegistry.cc +@@ -70,8 +70,11 @@ int PluginRegistry::remove(const std::string& type, const std::string& name) + std::map::iterator j = i->second.find(name); + if (j == i->second.end()) + return -ENOENT; +- ++#ifdef WITH_GLOBAL_CACHE ++ ldout(cct, 5) << __func__ << " " << type << " " << name << dendl; ++#else + ldout(cct, 1) << __func__ << " " << type << " " << name << dendl; ++#endif + void *library = j->second->library; + delete j->second; + dlclose(library); +@@ -91,8 +94,13 @@ int PluginRegistry::add(const std::string& type, + plugins[type].count(name)) { + return -EEXIST; + } ++#ifdef WITH_GLOBAL_CACHE ++ ldout(cct, 5) << __func__ << " " << type << " " << name ++ << " " << plugin << dendl; ++#else + ldout(cct, 1) << __func__ << " " << type << " " << name + << " " << plugin << dendl; ++#endif + plugins[type][name] = plugin; + return 0; + } +@@ -127,8 +135,13 @@ Plugin *PluginRegistry::get(const std::string& type, + ret = j->second; + + out: ++#ifdef WITH_GLOBAL_CACHE ++ ldout(cct, 5) << __func__ << " " << type << " " << name ++ << " = " << ret << dendl; ++#else + ldout(cct, 1) << __func__ << " " << type << " " << name + << " = " << ret << dendl; ++#endif + return ret; + } + +@@ -136,7 +149,11 @@ int PluginRegistry::load(const std::string &type, + const std::string &name) + { + ceph_assert(ceph_mutex_is_locked(lock)); ++#ifdef WITH_GLOBAL_CACHE ++ ldout(cct, 5) << __func__ << " " << type << " " << name << dendl; ++#else + ldout(cct, 1) << __func__ << " " << type << " " << name << dendl; ++#endif + + // std::string fname = cct->_conf->plugin_dir + "/" + type + "/" PLUGIN_PREFIX + // + name + PLUGIN_SUFFIX; +@@ -206,9 +223,15 @@ int PluginRegistry::load(const std::string &type, + } + + plugin->library = library; +- ++#ifdef WITH_GLOBAL_CACHE ++ ldout(cct, 5) << __func__ << ": " << type << " " << name ++ << " loaded and registered" << dendl; ++#else + ldout(cct, 1) << __func__ << ": " << type << " " << name + << " loaded and registered" << dendl; ++#endif ++ ++ + return 0; + } + diff --git a/src/common/options.cc b/src/common/options.cc -index 8135ea8..9d3bb78 100644 +index 8135ea8f..9d3bb781 100644 --- a/src/common/options.cc +++ b/src/common/options.cc @@ -1015,7 +1015,7 @@ std::vector