From da14cc03f46fd44ec1ecd09a60a54efeef68926e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=A8=E5=85=89?= <18767185082@163.com> Date: Fri, 29 Jul 2022 07:25:45 +0000 Subject: [PATCH 1/4] FixBug for columnar shuffle slice vector in null value buffer situation. --- .../omniop-spark-extension/cpp/src/shuffle/splitter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp b/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp index 5fdff5344..b2a0cf925 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp +++ b/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp @@ -309,7 +309,7 @@ int Splitter::SplitFixedWidthValidityBuffer(VectorBatch& vb){ } // 计算并填充数据 - auto src_addr = const_cast((uint8_t*)((vb.GetVector(col_idx))->GetValueNulls())); + auto src_addr = const_cast((uint8_t*)(VectorHelper::GetNullsAddr(vb.GetVector(col_idx)))); std::fill(std::begin(partition_buffer_idx_offset_), std::end(partition_buffer_idx_offset_), 0); const auto num_rows = vb.GetRowCount(); -- Gitee From 1591409738b207fe2123a1584958a0d620edee95 Mon Sep 17 00:00:00 2001 From: chen-guang-wang <18767185082@163.com> Date: Sat, 30 Jul 2022 10:02:41 +0800 Subject: [PATCH 2/4] remove path dependency for cpp compile --- omnioperator/omniop-spark-extension/cpp/src/CMakeLists.txt | 2 -- .../omniop-spark-extension/cpp/test/shuffle/CMakeLists.txt | 3 --- .../omniop-spark-extension/cpp/test/tablescan/CMakeLists.txt | 3 --- .../omniop-spark-extension/cpp/test/utils/CMakeLists.txt | 2 -- 4 files changed, 10 deletions(-) diff --git a/omnioperator/omniop-spark-extension/cpp/src/CMakeLists.txt b/omnioperator/omniop-spark-extension/cpp/src/CMakeLists.txt index d233edd19..45f75e7ff 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/CMakeLists.txt +++ b/omnioperator/omniop-spark-extension/cpp/src/CMakeLists.txt @@ -1,4 +1,3 @@ -include_directories(SYSTEM "/opt/lib/include") include_directories(SYSTEM "/user/local/include") set (PROJ_TARGET spark_columnar_plugin) @@ -34,7 +33,6 @@ add_library (${PROJ_TARGET} SHARED ${SOURCE_FILES} ${PROTO_SRCS} ${PROTO_HDRS} $ target_include_directories(${PROJ_TARGET} PUBLIC $ENV{JAVA_HOME}/include) target_include_directories(${PROJ_TARGET} PUBLIC $ENV{JAVA_HOME}/include/linux) target_include_directories(${PROJ_TARGET} PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) -target_include_directories(${PROJ_TARGET} PUBLIC /opt/lib/include) target_link_libraries (${PROJ_TARGET} PUBLIC orc diff --git a/omnioperator/omniop-spark-extension/cpp/test/shuffle/CMakeLists.txt b/omnioperator/omniop-spark-extension/cpp/test/shuffle/CMakeLists.txt index ff8bf512a..178c781c0 100644 --- a/omnioperator/omniop-spark-extension/cpp/test/shuffle/CMakeLists.txt +++ b/omnioperator/omniop-spark-extension/cpp/test/shuffle/CMakeLists.txt @@ -3,8 +3,5 @@ set(SHUFFLE_TEST_TARGET shuffletest) add_library(${SHUFFLE_TEST_TARGET} STATIC ${SHUFFLE_TESTS_LIST}) target_compile_options(${SHUFFLE_TEST_TARGET} PUBLIC ) target_include_directories(${SHUFFLE_TEST_TARGET} PUBLIC ${CMAKE_BINARY_DIR}/src) -target_include_directories(${SHUFFLE_TEST_TARGET} PUBLIC /opt/lib/include) target_include_directories(${SHUFFLE_TEST_TARGET} PUBLIC $ENV{JAVA_HOME}/include) target_include_directories(${SHUFFLE_TEST_TARGET} PUBLIC $ENV{JAVA_HOME}/include/linux) - - diff --git a/omnioperator/omniop-spark-extension/cpp/test/tablescan/CMakeLists.txt b/omnioperator/omniop-spark-extension/cpp/test/tablescan/CMakeLists.txt index ad201c2a6..8ca2b6d59 100644 --- a/omnioperator/omniop-spark-extension/cpp/test/tablescan/CMakeLists.txt +++ b/omnioperator/omniop-spark-extension/cpp/test/tablescan/CMakeLists.txt @@ -8,6 +8,3 @@ target_compile_options(${SCAN_TEST_TARGET} PUBLIC ) target_include_directories(${SCAN_TEST_TARGET} PUBLIC $ENV{JAVA_HOME}/include) target_include_directories(${SCAN_TEST_TARGET} PUBLIC $ENV{JAVA_HOME}/include/linux) -target_include_directories(${SCAN_TEST_TARGET} PUBLIC /opt/lib/include) - - diff --git a/omnioperator/omniop-spark-extension/cpp/test/utils/CMakeLists.txt b/omnioperator/omniop-spark-extension/cpp/test/utils/CMakeLists.txt index 0865325fc..d5ef3a300 100644 --- a/omnioperator/omniop-spark-extension/cpp/test/utils/CMakeLists.txt +++ b/omnioperator/omniop-spark-extension/cpp/test/utils/CMakeLists.txt @@ -1,8 +1,6 @@ aux_source_directory(${CMAKE_CURRENT_LIST_DIR} UTILS_TESTS_LIST) set(UTILS_TEST_TARGET utilstest) add_library(${UTILS_TEST_TARGET} ${UTILS_TESTS_LIST}) -target_include_directories(${UTILS_TEST_TARGET} PUBLIC /opt/lib/include) target_include_directories(${UTILS_TEST_TARGET} PUBLIC ${CMAKE_BINARY_DIR}/src) target_include_directories(${UTILS_TEST_TARGET} PUBLIC $ENV{JAVA_HOME}/include) target_include_directories(${UTILS_TEST_TARGET} PUBLIC $ENV{JAVA_HOME}/include/linux) - -- Gitee From 0008d255259ba6c00bbc896bdac3a8e5cb7e3afc Mon Sep 17 00:00:00 2001 From: chen-guang-wang <18767185082@163.com> Date: Mon, 22 Aug 2022 11:47:37 +0800 Subject: [PATCH 3/4] build.sh add CPLUS_INCLUDE_PATH --- omnioperator/omniop-spark-extension/cpp/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/cpp/build.sh b/omnioperator/omniop-spark-extension/cpp/build.sh index 26f83e2cb..866c021d4 100644 --- a/omnioperator/omniop-spark-extension/cpp/build.sh +++ b/omnioperator/omniop-spark-extension/cpp/build.sh @@ -17,6 +17,8 @@ set -eu +export CPLUS_INCLUDE_PATH=/opt/lib/include:$CPLUS_INCLUDE_PATH + CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) echo $CURRENT_DIR cd ${CURRENT_DIR} @@ -53,4 +55,4 @@ fi make -set +eu \ No newline at end of file +set +eu -- Gitee From 15b93f4087b1e391fc9908b757a6052a1ebfc04d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=A8=E5=85=89?= <18767185082@163.com> Date: Tue, 23 Aug 2022 03:13:05 +0000 Subject: [PATCH 4/4] update omnioperator/omniop-spark-extension/cpp/build.sh. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 王晨光 <18767185082@163.com> --- omnioperator/omniop-spark-extension/cpp/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/cpp/build.sh b/omnioperator/omniop-spark-extension/cpp/build.sh index 866c021d4..bb53aec9b 100644 --- a/omnioperator/omniop-spark-extension/cpp/build.sh +++ b/omnioperator/omniop-spark-extension/cpp/build.sh @@ -17,7 +17,9 @@ set -eu -export CPLUS_INCLUDE_PATH=/opt/lib/include:$CPLUS_INCLUDE_PATH +export OMNI_INCLUDE_PATH=/opt/lib/include +export CPLUS_INCLUDE_PATH=$OMNI_INCLUDE_PATH +echo "OMNI_INCLUDE_PATH=$OMNI_INCLUDE_PATH" CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) echo $CURRENT_DIR -- Gitee