From c7323487e4cbc504875df60d17d580ef69225d74 Mon Sep 17 00:00:00 2001 From: likun104 Date: Mon, 12 Aug 2024 19:15:16 +0800 Subject: [PATCH] fix memleak and enable ASan --- build.sh | 2 + tf_adapter/tests/CMakeLists.txt | 6 ++ tf_adapter/tests/depends/alog/CMakeLists.txt | 8 ++- tf_adapter/tests/depends/aoe/CMakeLists.txt | 8 ++- .../tests/depends/ascendcl/CMakeLists.txt | 8 ++- .../depends/ascendcl/src/ascendcl_stub.cc | 2 + .../tests/depends/datatransfer/CMakeLists.txt | 8 ++- .../tests/depends/ge_runner/CMakeLists.txt | 7 ++- .../depends/indextransform/CMakeLists.txt | 8 ++- tf_adapter/tests/depends/mmpa/CMakeLists.txt | 8 ++- .../tests/depends/runtime/CMakeLists.txt | 7 ++- tf_adapter/tests/st/CMakeLists.txt | 4 ++ .../testcase/dataset/dataset_test_base.cc | 11 ++-- .../dataset/device_queue_dataset_st.cc | 2 + tf_adapter/tests/ut/CMakeLists.txt | 4 ++ .../testcase/dataset/dataset_test_base.cc | 11 ++-- .../dataset/device_queue_dataset_ut.cc | 2 + .../testcase/dataset/host_queue_dataset_ut.cc | 5 +- tf_adapter/tests/util/sanitizer_options.h | 60 +++++++++++++++++++ tf_adapter_2.x/tests/cmake/aoe/module.cmake | 8 ++- tf_adapter_2.x/tests/st/CMakeLists.txt | 2 +- 21 files changed, 154 insertions(+), 27 deletions(-) create mode 100644 tf_adapter/tests/util/sanitizer_options.h diff --git a/build.sh b/build.sh index 87c1e65a9..b557f98a9 100755 --- a/build.sh +++ b/build.sh @@ -159,6 +159,7 @@ main() { lcov -r coverage/tmp.info '*/tests/*' '*/nlohmann_json-src/*' '*/tensorflow-src/*' \ '*/inc/*' '*/output/*' '*/usr/*' '*/Eigen/*' '*/absl/*' '*/google/*' '*/tensorflow/core/*' \ -o adapter1_coverage.info + export LD_LIBRARY_PATH=${BASE_PATH}/tf_adapter_2.x/tests/build/:$LD_LIBRARY_PATH bash ${BASE_PATH}/tf_adapter_2.x/tests/CI_Build adapter2_ut lcov -o coverage/coverage.info -a ${BASE_PATH}/tf_adapter_2.x/tests/build/ut/ut.coverage -a adapter1_coverage.info fi @@ -181,6 +182,7 @@ main() { lcov -r coverage/tmp.info '*/tests/*' '*/nlohmann_json-src/*' '*/tensorflow-src/*' \ '*/inc/*' '*/output/*' '*/usr/*' '*/Eigen/*' '*/absl/*' '*/google/*' '*/tensorflow/core/*' \ -o adapter1_coverage.info + export LD_LIBRARY_PATH=${BASE_PATH}/tf_adapter_2.x/tests/build/:$LD_LIBRARY_PATH bash ${BASE_PATH}/tf_adapter_2.x/tests/CI_Build adapter2_st lcov -o coverage/coverage.info -a ${BASE_PATH}/tf_adapter_2.x/tests/build/st/st.coverage -a adapter1_coverage.info fi diff --git a/tf_adapter/tests/CMakeLists.txt b/tf_adapter/tests/CMakeLists.txt index 8d335b54a..c63f1ac28 100644 --- a/tf_adapter/tests/CMakeLists.txt +++ b/tf_adapter/tests/CMakeLists.txt @@ -15,6 +15,12 @@ project(tests CXX C) +set(TFADAPTER_COMMON_COMPILE_OPTION -g -O0 -fsanitize=address -fsanitize=leak -fsanitize-recover=address) +set(TFADAPTER_COMMON_LINK_OPTION -fsanitize=address -fsanitize=leak -fsanitize-recover=address) + +string(REPLACE "-s " "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") +string(REPLACE "-s " "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + include(${TFADAPTER_DIR}/cmake/tests/gtest.cmake) include(${TFADAPTER_DIR}/cmake/tests/securec.cmake) include(${TFADAPTER_DIR}/cmake/tests/intf_pub_linux.cmake) diff --git a/tf_adapter/tests/depends/alog/CMakeLists.txt b/tf_adapter/tests/depends/alog/CMakeLists.txt index 52946ba94..a5a676ff7 100644 --- a/tf_adapter/tests/depends/alog/CMakeLists.txt +++ b/tf_adapter/tests/depends/alog/CMakeLists.txt @@ -22,6 +22,12 @@ file(GLOB_RECURSE SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include_directories(${TFADAPTER_DIR}/inc/) add_library(alog_stub SHARED ${SRC_FILES}) + +target_compile_options(alog_stub PRIVATE + ${TFADAPTER_COMMON_COMPILE_OPTION} +) + target_link_libraries(alog_stub PRIVATE $ -) \ No newline at end of file + ${TFADAPTER_COMMON_LINK_OPTION} +) diff --git a/tf_adapter/tests/depends/aoe/CMakeLists.txt b/tf_adapter/tests/depends/aoe/CMakeLists.txt index d2ed742e3..e25b6fe67 100644 --- a/tf_adapter/tests/depends/aoe/CMakeLists.txt +++ b/tf_adapter/tests/depends/aoe/CMakeLists.txt @@ -20,7 +20,13 @@ file(GLOB_RECURSE SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ) add_library(aoe_tuning SHARED ${SRC_FILES}) + +target_compile_options(aoe_tuning PRIVATE + ${TFADAPTER_COMMON_COMPILE_OPTION} +) + target_link_libraries(aoe_tuning PRIVATE $ + ${TFADAPTER_COMMON_LINK_OPTION} ge_runner_stub -) \ No newline at end of file +) diff --git a/tf_adapter/tests/depends/ascendcl/CMakeLists.txt b/tf_adapter/tests/depends/ascendcl/CMakeLists.txt index e1db05d2b..090c4a5d4 100644 --- a/tf_adapter/tests/depends/ascendcl/CMakeLists.txt +++ b/tf_adapter/tests/depends/ascendcl/CMakeLists.txt @@ -25,6 +25,12 @@ include_directories(${TFADAPTER_DIR}/inc/ ) add_library(ascendcl_stub SHARED ${SRC_FILES}) + +target_compile_options(ascendcl_stub PRIVATE + ${TFADAPTER_COMMON_COMPILE_OPTION} +) + target_link_libraries(ascendcl_stub PRIVATE $ -) \ No newline at end of file + ${TFADAPTER_COMMON_LINK_OPTION} +) diff --git a/tf_adapter/tests/depends/ascendcl/src/ascendcl_stub.cc b/tf_adapter/tests/depends/ascendcl/src/ascendcl_stub.cc index e79d1c04c..a6f186817 100644 --- a/tf_adapter/tests/depends/ascendcl/src/ascendcl_stub.cc +++ b/tf_adapter/tests/depends/ascendcl/src/ascendcl_stub.cc @@ -88,6 +88,7 @@ aclError acltdtDestroyChannel(acltdtChannelHandle *handle) { if (handle == nullptr) { return ACL_ERROR_INVALID_PARAM; } + delete handle; return ACL_SUCCESS; } @@ -225,6 +226,7 @@ aclError acltdtReceiveTensor(const acltdtChannelHandle *handle, } } } + dataset->freeSelf = true; return ACL_SUCCESS; } diff --git a/tf_adapter/tests/depends/datatransfer/CMakeLists.txt b/tf_adapter/tests/depends/datatransfer/CMakeLists.txt index e114dce5f..798a214c7 100644 --- a/tf_adapter/tests/depends/datatransfer/CMakeLists.txt +++ b/tf_adapter/tests/depends/datatransfer/CMakeLists.txt @@ -22,6 +22,12 @@ file(GLOB_RECURSE SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include_directories(${TFADAPTER_DIR}/inc/) add_library(datatransfer_stub SHARED ${SRC_FILES}) + +target_compile_options(datatransfer_stub PRIVATE + ${TFADAPTER_COMMON_COMPILE_OPTION} +) + target_link_libraries(datatransfer_stub PRIVATE $ -) \ No newline at end of file + ${TFADAPTER_COMMON_LINK_OPTION} +) diff --git a/tf_adapter/tests/depends/ge_runner/CMakeLists.txt b/tf_adapter/tests/depends/ge_runner/CMakeLists.txt index 3681db1d7..c0e9331bb 100644 --- a/tf_adapter/tests/depends/ge_runner/CMakeLists.txt +++ b/tf_adapter/tests/depends/ge_runner/CMakeLists.txt @@ -23,6 +23,10 @@ set(BASE_DIR ${TOP_DIR}/../) add_library(ge_runner_stub SHARED ${SRC_FILES}) +target_compile_options(ge_runner_stub PRIVATE + ${TFADAPTER_COMMON_COMPILE_OPTION} +) + target_include_directories(ge_runner_stub PRIVATE ${TF_INSTALL_LIB_PATH}/include/ ${TFADAPTER_DIR}/tf_adapter/tests/depends/ascendcl/src @@ -31,4 +35,5 @@ target_include_directories(ge_runner_stub PRIVATE target_link_libraries(ge_runner_stub PRIVATE $ -) \ No newline at end of file + ${TFADAPTER_COMMON_LINK_OPTION} +) diff --git a/tf_adapter/tests/depends/indextransform/CMakeLists.txt b/tf_adapter/tests/depends/indextransform/CMakeLists.txt index 5f10848e4..7dae41354 100644 --- a/tf_adapter/tests/depends/indextransform/CMakeLists.txt +++ b/tf_adapter/tests/depends/indextransform/CMakeLists.txt @@ -22,6 +22,12 @@ file(GLOB_RECURSE SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include_directories(${TFADAPTER_DIR}/inc/) add_library(indextransform_stub SHARED ${SRC_FILES}) + +target_compile_options(indextransform_stub PRIVATE + ${TFADAPTER_COMMON_COMPILE_OPTION} +) + target_link_libraries(indextransform_stub PRIVATE $ -) \ No newline at end of file + ${TFADAPTER_COMMON_LINK_OPTION} +) diff --git a/tf_adapter/tests/depends/mmpa/CMakeLists.txt b/tf_adapter/tests/depends/mmpa/CMakeLists.txt index f5d1a3a53..f95065a0e 100644 --- a/tf_adapter/tests/depends/mmpa/CMakeLists.txt +++ b/tf_adapter/tests/depends/mmpa/CMakeLists.txt @@ -22,9 +22,15 @@ file(GLOB_RECURSE SRC_FILES RELATIVE ${CMAKE_CURRENT_LIST_DIR} include_directories(${TFADAPTER_DIR}/inc) add_library(mmpa_stub SHARED ${SRC_FILES}) + +target_compile_options(mmpa_stub PRIVATE + ${TFADAPTER_COMMON_COMPILE_OPTION} +) + target_link_libraries(mmpa_stub PRIVATE $ + ${TFADAPTER_COMMON_LINK_OPTION} -Wl,--no-as-needed c_sec -Wl,--as-needed -) \ No newline at end of file +) diff --git a/tf_adapter/tests/depends/runtime/CMakeLists.txt b/tf_adapter/tests/depends/runtime/CMakeLists.txt index 82159d1de..9412e3fe9 100644 --- a/tf_adapter/tests/depends/runtime/CMakeLists.txt +++ b/tf_adapter/tests/depends/runtime/CMakeLists.txt @@ -21,6 +21,10 @@ file(GLOB_RECURSE SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} add_library(runtime_stub SHARED ${SRC_FILES}) +target_compile_options(runtime_stub PRIVATE + ${TFADAPTER_COMMON_COMPILE_OPTION} +) + target_include_directories(runtime_stub PRIVATE ${CMAKE_CURRENT_LIST_DIR}/ ${TFADAPTER_DIR}/inc/ @@ -29,4 +33,5 @@ target_include_directories(runtime_stub PRIVATE target_link_libraries(runtime_stub PRIVATE $ -) \ No newline at end of file + ${TFADAPTER_COMMON_LINK_OPTION} +) diff --git a/tf_adapter/tests/st/CMakeLists.txt b/tf_adapter/tests/st/CMakeLists.txt index 850cf56fc..47b92291a 100644 --- a/tf_adapter/tests/st/CMakeLists.txt +++ b/tf_adapter/tests/st/CMakeLists.txt @@ -48,9 +48,11 @@ target_include_directories(tfadapter_stest PRIVATE ${TFADAPTER_DIR}/tf_adapter/tests/depends/runtime/src ${TFADAPTER_DIR}/tf_adapter/tests/depends/ge_runner/src/ ${TFADAPTER_DIR}/tf_adapter/tests/depends/alog/src/ + ${TFADAPTER_DIR}/tf_adapter/tests ) target_compile_options(tfadapter_stest PRIVATE + ${TFADAPTER_COMMON_COMPILE_OPTION} -g --coverage -fprofile-arcs -ftest-coverage -Werror=format -Wno-missing-field-initializers @@ -66,6 +68,7 @@ add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) target_compile_definitions(tfadapter_stest PRIVATE _FORTIFY_SOURCE=2 LOG_CPP + ONLY_COMPILE_OPEN_SRC ) foreach (ST_LINK_FLAG ${ST_LINK_FLAGS}) @@ -79,6 +82,7 @@ add_dependencies(tfadapter_stest aoe_tuning) target_link_libraries(tfadapter_stest PUBLIC $ + ${TFADAPTER_COMMON_LINK_OPTION} gtest gtest_main c_sec mmpa_stub indextransform_stub alog_stub datatransfer_stub ge_runner_stub ascendcl_stub runtime_stub ${PYTHON_LIB_PATH} -lrt -ldl -lgcov ) diff --git a/tf_adapter/tests/st/kernels/testcase/dataset/dataset_test_base.cc b/tf_adapter/tests/st/kernels/testcase/dataset/dataset_test_base.cc index 6a3a0336e..e6040d522 100644 --- a/tf_adapter/tests/st/kernels/testcase/dataset/dataset_test_base.cc +++ b/tf_adapter/tests/st/kernels/testcase/dataset/dataset_test_base.cc @@ -323,11 +323,9 @@ Status DatasetOpsTestBase::RunFunction(const FunctionDef& fdef1, test::function: Status DatasetOpsTestBase::CreateOpKernelContext( OpKernel* kernels, gtl::InlinedVector* inputs, std::unique_ptr* ctx) { params_ = absl::make_unique(); - unique_ptr pas; - pas = std::move(params_); + OpKernelContext::Params *pas = params_.get(); cancellation_manager_ = absl::make_unique(); - unique_ptr cm = std::move(cancellation_manager_); - pas->cancellation_manager = cm.release(); + pas->cancellation_manager = cancellation_manager_.get(); pas->device = device_.get(); pas->frame_iter = FrameAndIter(0, 0); pas->function_library = flr_; @@ -338,8 +336,7 @@ Status DatasetOpsTestBase::CreateOpKernelContext( OpKernel* kernels, gtl::Inline slice_reader_cache_ = absl::make_unique(); pas->slice_reader_cache = slice_reader_cache_.get(); step_container_ = absl::make_unique(0, [](const string&) {}); - unique_ptr stc = std::move(step_container_); - pas->step_container = stc.release(); + pas->step_container = step_container_.get(); allocator_attrs_.clear(); for (int i = 0; i < pas->op_kernel->num_outputs(); i++) { @@ -350,7 +347,7 @@ Status DatasetOpsTestBase::CreateOpKernelContext( OpKernel* kernels, gtl::Inline } pas->output_attr_array = gtl::vector_as_array(&allocator_attrs_); - *ctx = absl::make_unique(pas.release()); + *ctx = absl::make_unique(pas); return Status::OK(); } diff --git a/tf_adapter/tests/st/kernels/testcase/dataset/device_queue_dataset_st.cc b/tf_adapter/tests/st/kernels/testcase/dataset/device_queue_dataset_st.cc index 011627616..6faced707 100644 --- a/tf_adapter/tests/st/kernels/testcase/dataset/device_queue_dataset_st.cc +++ b/tf_adapter/tests/st/kernels/testcase/dataset/device_queue_dataset_st.cc @@ -11,6 +11,7 @@ #include "gtest/gtest.h" #include "tf_adapter/common/adapter_logger.h" +#include "util/sanitizer_options.h" namespace tensorflow { namespace data { @@ -68,6 +69,7 @@ class DeviceQueueDatasetOpTest : public DatasetOpsTestBaseV2 + ${TFADAPTER_COMMON_LINK_OPTION} gtest gtest_main c_sec mmpa_stub indextransform_stub alog_stub datatransfer_stub ge_runner_stub ascendcl_stub runtime_stub ${PYTHON_LIB_PATH} -lrt -ldl -lgcov ) diff --git a/tf_adapter/tests/ut/kernels/testcase/dataset/dataset_test_base.cc b/tf_adapter/tests/ut/kernels/testcase/dataset/dataset_test_base.cc index f6f2ab29f..0ed2a9d25 100644 --- a/tf_adapter/tests/ut/kernels/testcase/dataset/dataset_test_base.cc +++ b/tf_adapter/tests/ut/kernels/testcase/dataset/dataset_test_base.cc @@ -323,11 +323,9 @@ Status DatasetOpsTestBase::RunFunction(const FunctionDef& fdef1, test::function: Status DatasetOpsTestBase::CreateOpKernelContext( OpKernel* kernels, gtl::InlinedVector* inputs, std::unique_ptr* ctx) { params_ = absl::make_unique(); - unique_ptr pas; - pas = std::move(params_); + OpKernelContext::Params *pas = params_.get(); cancellation_manager_ = absl::make_unique(); - unique_ptr cm = std::move(cancellation_manager_); - pas->cancellation_manager = cm.release(); + pas->cancellation_manager = cancellation_manager_.get(); pas->device = device_.get(); pas->frame_iter = FrameAndIter(0, 0); pas->function_library = flr_; @@ -338,8 +336,7 @@ Status DatasetOpsTestBase::CreateOpKernelContext( OpKernel* kernels, gtl::Inline slice_reader_cache_ = absl::make_unique(); pas->slice_reader_cache = slice_reader_cache_.get(); step_container_ = absl::make_unique(0, [](const string&) {}); - unique_ptr stc = std::move(step_container_); - pas->step_container = stc.release(); + pas->step_container = step_container_.get(); allocator_attrs_.clear(); for (int i = 0; i < pas->op_kernel->num_outputs(); i++) { @@ -350,7 +347,7 @@ Status DatasetOpsTestBase::CreateOpKernelContext( OpKernel* kernels, gtl::Inline } pas->output_attr_array = gtl::vector_as_array(&allocator_attrs_); - *ctx = absl::make_unique(pas.release()); + *ctx = absl::make_unique(pas); return Status::OK(); } diff --git a/tf_adapter/tests/ut/kernels/testcase/dataset/device_queue_dataset_ut.cc b/tf_adapter/tests/ut/kernels/testcase/dataset/device_queue_dataset_ut.cc index 674ba9f52..eca33ea1e 100644 --- a/tf_adapter/tests/ut/kernels/testcase/dataset/device_queue_dataset_ut.cc +++ b/tf_adapter/tests/ut/kernels/testcase/dataset/device_queue_dataset_ut.cc @@ -11,6 +11,7 @@ #include "gtest/gtest.h" #include "tf_adapter/common/adapter_logger.h" +#include "util/sanitizer_options.h" namespace tensorflow { namespace data { @@ -68,6 +69,7 @@ class DeviceQueueDatasetOpTest : public DatasetOpsTestBaseV2