From da14cc03f46fd44ec1ecd09a60a54efeef68926e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=A8=E5=85=89?= <18767185082@163.com> Date: Fri, 29 Jul 2022 07:25:45 +0000 Subject: [PATCH 1/2] FixBug for columnar shuffle slice vector in null value buffer situation. --- .../omniop-spark-extension/cpp/src/shuffle/splitter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp b/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp index 5fdff5344..b2a0cf925 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp +++ b/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp @@ -309,7 +309,7 @@ int Splitter::SplitFixedWidthValidityBuffer(VectorBatch& vb){ } // 计算并填充数据 - auto src_addr = const_cast((uint8_t*)((vb.GetVector(col_idx))->GetValueNulls())); + auto src_addr = const_cast((uint8_t*)(VectorHelper::GetNullsAddr(vb.GetVector(col_idx)))); std::fill(std::begin(partition_buffer_idx_offset_), std::end(partition_buffer_idx_offset_), 0); const auto num_rows = vb.GetRowCount(); -- Gitee From 4ff8503e74d9b057f2065b2afbd2c2559dbfc771 Mon Sep 17 00:00:00 2001 From: chen-guang-wang <18767185082@163.com> Date: Fri, 29 Jul 2022 17:53:04 +0800 Subject: [PATCH 2/2] columnar shuffle cpp ut shuflletest link fixup --- .../omniop-spark-extension/cpp/CMakeLists.txt | 1 + .../omniop-spark-extension/cpp/test/CMakeLists.txt | 13 ++++--------- .../cpp/test/shuffle/CMakeLists.txt | 1 + 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/omnioperator/omniop-spark-extension/cpp/CMakeLists.txt b/omnioperator/omniop-spark-extension/cpp/CMakeLists.txt index f400007aa..03cdf8c44 100644 --- a/omnioperator/omniop-spark-extension/cpp/CMakeLists.txt +++ b/omnioperator/omniop-spark-extension/cpp/CMakeLists.txt @@ -16,6 +16,7 @@ configure_file( "${PROJECT_SOURCE_DIR}/config.h" ) +aux_source_directory(${CMAKE_CURRENT_LIST_DIR} ROOT_SRCS) # for header searching include_directories(SYSTEM src) diff --git a/omnioperator/omniop-spark-extension/cpp/test/CMakeLists.txt b/omnioperator/omniop-spark-extension/cpp/test/CMakeLists.txt index a06358d82..432664828 100644 --- a/omnioperator/omniop-spark-extension/cpp/test/CMakeLists.txt +++ b/omnioperator/omniop-spark-extension/cpp/test/CMakeLists.txt @@ -1,3 +1,5 @@ +aux_source_directory(${CMAKE_CURRENT_LIST_DIR} TEST_ROOT_SRCS) + add_subdirectory(shuffle) add_subdirectory(utils) add_subdirectory(tablescan) @@ -5,23 +7,16 @@ add_subdirectory(tablescan) # configure set(TP_TEST_TARGET tptest) set(MY_LINK - utilstest shuffletest + utilstest tablescantest ) # find gtest package find_package(GTest REQUIRED) -set (UT_FILES - tptest.cpp - shuffle/shuffle_test.cpp - tablescan/scan_test.cpp - ) - # compile a executable file -add_executable(${TP_TEST_TARGET} ${UT_FILES}) -add_dependencies(${TP_TEST_TARGET} ${MY_LINK}) +add_executable(${TP_TEST_TARGET} ${ROOT_SRCS} ${TEST_ROOT_SRCS}) # dependent libraries target_link_libraries(${TP_TEST_TARGET} diff --git a/omnioperator/omniop-spark-extension/cpp/test/shuffle/CMakeLists.txt b/omnioperator/omniop-spark-extension/cpp/test/shuffle/CMakeLists.txt index ff8bf512a..7155af40b 100644 --- a/omnioperator/omniop-spark-extension/cpp/test/shuffle/CMakeLists.txt +++ b/omnioperator/omniop-spark-extension/cpp/test/shuffle/CMakeLists.txt @@ -2,6 +2,7 @@ aux_source_directory(${CMAKE_CURRENT_LIST_DIR} SHUFFLE_TESTS_LIST) set(SHUFFLE_TEST_TARGET shuffletest) add_library(${SHUFFLE_TEST_TARGET} STATIC ${SHUFFLE_TESTS_LIST}) target_compile_options(${SHUFFLE_TEST_TARGET} PUBLIC ) +target_link_libraries(${SHUFFLE_TEST_TARGET} utilstest) target_include_directories(${SHUFFLE_TEST_TARGET} PUBLIC ${CMAKE_BINARY_DIR}/src) target_include_directories(${SHUFFLE_TEST_TARGET} PUBLIC /opt/lib/include) target_include_directories(${SHUFFLE_TEST_TARGET} PUBLIC $ENV{JAVA_HOME}/include) -- Gitee