diff --git a/omnioperator/omniop-spark-extension/cpp/src/CMakeLists.txt b/omnioperator/omniop-spark-extension/cpp/src/CMakeLists.txt index 0169e31f9ffa25561562dcc3dce7174387e6c6b1..8d181ad22db0e8fd2b43d827202bba470d90e6e2 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/CMakeLists.txt +++ b/omnioperator/omniop-spark-extension/cpp/src/CMakeLists.txt @@ -14,6 +14,7 @@ set (SOURCE_FILES common/common.cpp jni/SparkJniWrapper.cpp jni/OrcColumnarBatchJniReader.cpp + jni/jni_common.cpp ) #Find required protobuf package diff --git a/omnioperator/omniop-spark-extension/cpp/src/jni/OrcColumnarBatchJniReader.cpp b/omnioperator/omniop-spark-extension/cpp/src/jni/OrcColumnarBatchJniReader.cpp index 31ca0a74e1f4c7ccafe769f29ee29c8e459e2439..11204207b16050211115dd51c75dfe46b2aa68fd 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/jni/OrcColumnarBatchJniReader.cpp +++ b/omnioperator/omniop-spark-extension/cpp/src/jni/OrcColumnarBatchJniReader.cpp @@ -18,6 +18,8 @@ */ #include "OrcColumnarBatchJniReader.h" +#include "jni_common.h" + using namespace omniruntime::vec; using namespace std; using namespace orc; @@ -94,6 +96,7 @@ void JNI_OnUnload(JavaVM *vm, const void *reserved) JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_initializeReader(JNIEnv *env, jobject jObj, jstring path, jobject jsonObj) { + JNI_FUNC_START /* * init logger and jni env method id */ @@ -122,6 +125,7 @@ JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniRe env->ReleaseStringUTFChars(path, pathPtr); orc::Reader *readerNew = reader.release(); return (jlong)(readerNew); + JNI_FUNC_END(runtimeExceptionClass) } bool stringToBool(string boolStr) @@ -173,49 +177,51 @@ int getLiteral(orc::Literal &lit, int leafType, string value) break; } default: { - LogsError("tableScan jni getLiteral unsupported leafType: " + leafType); throw std::runtime_error("tableScan jni getLiteral unsupported leafType: " + leafType); } } return 0; } -int buildLeafs(int leafOp, vector &litList, Literal &lit, string leafNameString, int leafType, +int buildLeaves(PredicateOperatorType leafOp, vector &litList, Literal &lit, string leafNameString, PredicateDataType leafType, SearchArgumentBuilder &builder) { - switch ((PredicateOperatorType)leafOp) { + switch (leafOp) { case PredicateOperatorType::LESS_THAN: { - builder.lessThan(leafNameString, (PredicateDataType)leafType, lit); + builder.lessThan(leafNameString, leafType, lit); break; } case PredicateOperatorType::LESS_THAN_EQUALS: { - builder.lessThanEquals(leafNameString, (PredicateDataType)leafType, lit); + builder.lessThanEquals(leafNameString, leafType, lit); break; } case PredicateOperatorType::EQUALS: { - builder.equals(leafNameString, (PredicateDataType)leafType, lit); + builder.equals(leafNameString, leafType, lit); break; } case PredicateOperatorType::NULL_SAFE_EQUALS: { - builder.nullSafeEquals(leafNameString, (PredicateDataType)leafType, lit); + builder.nullSafeEquals(leafNameString, leafType, lit); break; } case PredicateOperatorType::IS_NULL: { - builder.isNull(leafNameString, (PredicateDataType)leafType); + builder.isNull(leafNameString, leafType); break; } case PredicateOperatorType::IN: { - builder.in(leafNameString, (PredicateDataType)leafType, litList); + builder.in(leafNameString, leafType, litList); break; } + case PredicateOperatorType::BETWEEN: { + throw std::runtime_error("table scan buildLeaves BETWEEN is not supported!"); + } default: { - LogsError("ERROR operator ID"); + throw std::runtime_error("table scan buildLeaves illegal input!"); } } return 1; } -int initLeafs(JNIEnv *env, SearchArgumentBuilder &builder, jobject &jsonExp, jobject &jsonLeaves) +int initLeaves(JNIEnv *env, SearchArgumentBuilder &builder, jobject &jsonExp, jobject &jsonLeaves) { jstring leaf = (jstring)env->CallObjectMethod(jsonExp, jsonMethodString, env->NewStringUTF("leaf")); jobject leafJsonObj = env->CallObjectMethod(jsonLeaves, jsonMethodJsonObj, leaf); @@ -242,7 +248,7 @@ int initLeafs(JNIEnv *env, SearchArgumentBuilder &builder, jobject &jsonExp, job litList.push_back(lit); } } - buildLeafs((int)leafOp, litList, lit, leafNameString, (int)leafType, builder); + buildLeaves((PredicateOperatorType)leafOp, litList, lit, leafNameString, (PredicateDataType)leafType, builder); return 1; } @@ -250,7 +256,7 @@ int initExpressionTree(JNIEnv *env, SearchArgumentBuilder &builder, jobject &jso { int op = env->CallIntMethod(jsonExp, jsonMethodInt, env->NewStringUTF("op")); if (op == (int)(Operator::LEAF)) { - initLeafs(env, builder, jsonExp, jsonLeaves); + initLeaves(env, builder, jsonExp, jsonLeaves); } else { switch ((Operator)op) { case Operator::OR: { @@ -265,6 +271,9 @@ int initExpressionTree(JNIEnv *env, SearchArgumentBuilder &builder, jobject &jso builder.startNot(); break; } + default: { + throw std::runtime_error("tableScan jni initExpressionTree Unsupported op: " + op); + } } jobject childList = env->CallObjectMethod(jsonExp, jsonMethodObj, env->NewStringUTF("child")); int childs = (int)env->CallIntMethod(childList, arrayListSize); @@ -281,6 +290,7 @@ int initExpressionTree(JNIEnv *env, SearchArgumentBuilder &builder, jobject &jso JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_initializeRecordReader(JNIEnv *env, jobject jObj, jlong reader, jobject jsonObj) { + JNI_FUNC_START orc::Reader *readerPtr = (orc::Reader *)reader; // get offset from json obj jlong offset = env->CallLongMethod(jsonObj, jsonMethodLong, env->NewStringUTF("offset")); @@ -318,20 +328,22 @@ JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniRe rowReaderOpts.searchArgument(std::unique_ptr(sargBuilded.release())); } - std::unique_ptr rowReader = readerPtr->createRowReader(rowReaderOpts); return (jlong)(rowReader.release()); + JNI_FUNC_END(runtimeExceptionClass) } JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_initializeBatch(JNIEnv *env, jobject jObj, jlong rowReader, jlong batchSize) { + JNI_FUNC_START orc::RowReader *rowReaderPtr = (orc::RowReader *)(rowReader); uint64_t batchLen = (uint64_t)batchSize; std::unique_ptr batch = rowReaderPtr->createRowBatch(batchLen); orc::ColumnVectorBatch *rtn = batch.release(); return (jlong)rtn; + JNI_FUNC_END(runtimeExceptionClass) } template uint64_t copyFixwidth(orc::ColumnVectorBatch *field) @@ -422,11 +434,11 @@ int copyToOmniVec(orc::TypeKind vcType, int &omniTypeId, uint64_t &omniVecId, or return 1; } -int copyToOmniDecimalVec(int precision, int &ominTypeId, uint64_t &ominVecId, orc::ColumnVectorBatch *field) +int copyToOmniDecimalVec(int precision, int &omniTypeId, uint64_t &omniVecId, orc::ColumnVectorBatch *field) { VectorAllocator *allocator = VectorAllocator::GetGlobalAllocator(); if (precision > 18) { - ominTypeId = static_cast(OMNI_DECIMAL128); + omniTypeId = static_cast(OMNI_DECIMAL128); orc::Decimal128VectorBatch *lvb = dynamic_cast(field); FixedWidthVector *originalVector = new FixedWidthVector(allocator, lvb->numElements); @@ -448,9 +460,9 @@ int copyToOmniDecimalVec(int precision, int &ominTypeId, uint64_t &ominVecId, or originalVector->SetValueNull(i); } } - ominVecId = (uint64_t)originalVector; + omniVecId = (uint64_t)originalVector; } else { - ominTypeId = static_cast(OMNI_DECIMAL64); + omniTypeId = static_cast(OMNI_DECIMAL64); orc::Decimal64VectorBatch *lvb = dynamic_cast(field); FixedWidthVector *originalVector = new FixedWidthVector(allocator, lvb->numElements); for (int i = 0; i < lvb->numElements; i++) { @@ -460,7 +472,7 @@ int copyToOmniDecimalVec(int precision, int &ominTypeId, uint64_t &ominVecId, or originalVector->SetValueNull(i); } } - ominVecId = (uint64_t)originalVector; + omniVecId = (uint64_t)originalVector; } return 1; } @@ -468,6 +480,7 @@ int copyToOmniDecimalVec(int precision, int &ominTypeId, uint64_t &ominVecId, or JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_recordReaderNext(JNIEnv *env, jobject jObj, jlong rowReader, jlong reader, jlong batch, jintArray typeId, jlongArray vecNativeId) { + JNI_FUNC_START orc::RowReader *rowReaderPtr = (orc::RowReader *)rowReader; orc::ColumnVectorBatch *columnVectorBatch = (orc::ColumnVectorBatch *)batch; orc::Reader *readerPtr = (orc::Reader *)reader; @@ -481,25 +494,21 @@ JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniRe for (int id = 0; id < vecCnt; id++) { orc::TypeKind vcType = baseTp.getSubtype(id)->getKind(); int maxLen = baseTp.getSubtype(id)->getMaximumLength(); - int ominTypeId = 0; - uint64_t ominVecId = 0; - try { - if (vcType != orc::TypeKind::DECIMAL) { - copyToOmniVec(vcType, ominTypeId, ominVecId, root->fields[id], maxLen); - } else { - copyToOmniDecimalVec(baseTp.getSubtype(id)->getPrecision(), ominTypeId, ominVecId, - root->fields[id]); - } - } catch (omniruntime::exception::OmniException &e) { - env->ThrowNew(runtimeExceptionClass, e.what()); - return (jlong)batchRowSize; + int omniTypeId = 0; + uint64_t omniVecId = 0; + if (vcType != orc::TypeKind::DECIMAL) { + copyToOmniVec(vcType, omniTypeId, omniVecId, root->fields[id], maxLen); + } else { + copyToOmniDecimalVec(baseTp.getSubtype(id)->getPrecision(), omniTypeId, omniVecId, + root->fields[id]); } - env->SetIntArrayRegion(typeId, id, 1, &ominTypeId); - jlong ominVec = static_cast(ominVecId); - env->SetLongArrayRegion(vecNativeId, id, 1, &ominVec); + env->SetIntArrayRegion(typeId, id, 1, &omniTypeId); + jlong omniVec = static_cast(omniVecId); + env->SetLongArrayRegion(vecNativeId, id, 1, &omniVec); } } return (jlong)batchRowSize; + JNI_FUNC_END(runtimeExceptionClass) } /* @@ -510,9 +519,11 @@ JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniRe JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_recordReaderGetRowNumber( JNIEnv *env, jobject jObj, jlong rowReader) { + JNI_FUNC_START orc::RowReader *rowReaderPtr = (orc::RowReader *)rowReader; uint64_t rownum = rowReaderPtr->getRowNumber(); return (jlong)rownum; + JNI_FUNC_END(runtimeExceptionClass) } /* @@ -523,9 +534,11 @@ JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniRe JNIEXPORT jfloat JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_recordReaderGetProgress( JNIEnv *env, jobject jObj, jlong rowReader) { + JNI_FUNC_START jfloat curProgress = 1; - throw std::runtime_error("recordReaderGetProgress is unsupported"); + env->ThrowNew(runtimeExceptionClass, "recordReaderGetProgress is unsupported"); return curProgress; + JNI_FUNC_END(runtimeExceptionClass) } /* @@ -536,21 +549,23 @@ JNIEXPORT jfloat JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniR JNIEXPORT void JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_recordReaderClose(JNIEnv *env, jobject jObj, jlong rowReader, jlong reader, jlong batchReader) { + JNI_FUNC_START orc::ColumnVectorBatch *columnVectorBatch = (orc::ColumnVectorBatch *)batchReader; if (nullptr == columnVectorBatch) { - throw std::runtime_error("delete nullptr error for batch reader"); + env->ThrowNew(runtimeExceptionClass, "delete nullptr error for batch reader"); } delete columnVectorBatch; orc::RowReader *rowReaderPtr = (orc::RowReader *)rowReader; if (nullptr == rowReaderPtr) { - throw std::runtime_error("delete nullptr error for row reader"); + env->ThrowNew(runtimeExceptionClass, "delete nullptr error for row reader"); } delete rowReaderPtr; orc::Reader *readerPtr = (orc::Reader *)reader; if (nullptr == readerPtr) { - throw std::runtime_error("delete nullptr error for reader"); + env->ThrowNew(runtimeExceptionClass, "delete nullptr error for reader"); } delete readerPtr; + JNI_FUNC_END(runtimeExceptionClass) } /* @@ -561,14 +576,17 @@ JNIEXPORT void JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniRea JNIEXPORT void JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_recordReaderSeekToRow(JNIEnv *env, jobject jObj, jlong rowReader, jlong rowNumber) { + JNI_FUNC_START orc::RowReader *rowReaderPtr = (orc::RowReader *)rowReader; rowReaderPtr->seekToRow((long)rowNumber); + JNI_FUNC_END(runtimeExceptionClass) } JNIEXPORT jobjectArray JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_getAllColumnNames(JNIEnv *env, jobject jObj, jlong reader) { + JNI_FUNC_START orc::Reader *readerPtr = (orc::Reader *)reader; int32_t cols = static_cast(readerPtr->getType().getSubtypeCount()); jobjectArray ret = @@ -577,14 +595,17 @@ Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_getAllColumnNames(J env->SetObjectArrayElement(ret, i, env->NewStringUTF(readerPtr->getType().getFieldName(i).data())); } return ret; + JNI_FUNC_END(runtimeExceptionClass) } JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniReader_getNumberOfRows(JNIEnv *env, jobject jObj, jlong rowReader, jlong batch) { + JNI_FUNC_START orc::RowReader *rowReaderPtr = (orc::RowReader *)rowReader; orc::ColumnVectorBatch *columnVectorBatch = (orc::ColumnVectorBatch *)batch; rowReaderPtr->next(*columnVectorBatch); jlong rows = columnVectorBatch->numElements; return rows; + JNI_FUNC_END(runtimeExceptionClass) } diff --git a/omnioperator/omniop-spark-extension/cpp/src/jni/OrcColumnarBatchJniReader.h b/omnioperator/omniop-spark-extension/cpp/src/jni/OrcColumnarBatchJniReader.h index 4c6eb586363dc72a7af7c6f2eb3f14e4ac0a8563..56c11d49829f9d6e6364d8f42f0237c181ede26c 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/jni/OrcColumnarBatchJniReader.h +++ b/omnioperator/omniop-spark-extension/cpp/src/jni/OrcColumnarBatchJniReader.h @@ -137,16 +137,16 @@ JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_OrcColumnarBatchJniRe int getLiteral(orc::Literal &lit, int leafType, std::string value); -bool stringToBool(std::string boolStr); - -int buildLeafs(int leafOp, std::vector &litList, orc::Literal &lit, std::string leafNameString, int leafType, +int buildLeaves(PredicateOperatorType leafOp, std::vector &litList, orc::Literal &lit, std::string leafNameString, orc::PredicateDataType leafType, orc::SearchArgumentBuilder &builder); +bool stringToBool(std::string boolStr); + int copyToOmniVec(orc::TypeKind vcType, int &omniTypeId, uint64_t &omniVecId, orc::ColumnVectorBatch *field, ...); -int copyToOmniDecimalVec(int precision, int &ominTypeId, uint64_t &ominVecId, orc::ColumnVectorBatch *field); +int copyToOmniDecimalVec(int precision, int &omniTypeId, uint64_t &omniVecId, orc::ColumnVectorBatch *field); -int copyToOmniDecimalVec(int precision, int &ominTypeId, uint64_t &ominVecId, orc::ColumnVectorBatch *field); +int copyToOmniDecimalVec(int precision, int &omniTypeId, uint64_t &omniVecId, orc::ColumnVectorBatch *field); #ifdef __cplusplus } diff --git a/omnioperator/omniop-spark-extension/cpp/src/jni/SparkJniWrapper.cpp b/omnioperator/omniop-spark-extension/cpp/src/jni/SparkJniWrapper.cpp index 49b39abb764a12f599c16ef5240c796f88871161..54af0d239d93c4f9b352412407ebcd11314da4fe 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/jni/SparkJniWrapper.cpp +++ b/omnioperator/omniop-spark-extension/cpp/src/jni/SparkJniWrapper.cpp @@ -31,6 +31,7 @@ static jint JNI_VERSION = JNI_VERSION_1_8; static jclass split_result_class; static jclass runtime_exception_class; +static jclass excepiton_class; static jmethodID split_result_constructor; @@ -76,8 +77,9 @@ Java_com_huawei_boostkit_spark_jni_SparkJniWrapper_nativeMake( jstring compression_type_jstr, jstring data_file_jstr, jint num_sub_dirs, jstring local_dirs_jstr, jlong compress_block_size, jint spill_batch_row, jlong spill_memory_threshold) { + JNI_FUNC_START if (partitioning_name_jstr == nullptr) { - env->ThrowNew(env->FindClass("java/lang/Exception"), + env->ThrowNew(runtime_exception_class, std::string("Short partitioning name can't be null").c_str()); return 0; } @@ -104,12 +106,12 @@ Java_com_huawei_boostkit_spark_jni_SparkJniWrapper_nativeMake( inputDataTypesTmp.inputDataScales = inputDataScales; if (data_file_jstr == nullptr) { - env->ThrowNew(env->FindClass("java/lang/Exception"), + env->ThrowNew(runtime_exception_class, std::string("Shuffle DataFile can't be null").c_str()); return 0; } if (local_dirs_jstr == nullptr) { - env->ThrowNew(env->FindClass("java/lang/Exception"), + env->ThrowNew(runtime_exception_class, std::string("Shuffle DataFile can't be null").c_str()); return 0; } @@ -134,8 +136,6 @@ Java_com_huawei_boostkit_spark_jni_SparkJniWrapper_nativeMake( splitOptions.data_file = std::string(data_file_c); env->ReleaseStringUTFChars(data_file_jstr, data_file_c); - //TODO: memory pool select - auto local_dirs = env->GetStringUTFChars(local_dirs_jstr, JNI_FALSE); setenv("NATIVESQL_SPARK_LOCAL_DIRS", local_dirs, 1); env->ReleaseStringUTFChars(local_dirs_jstr, local_dirs); @@ -161,46 +161,39 @@ Java_com_huawei_boostkit_spark_jni_SparkJniWrapper_nativeMake( splitOptions.thread_id = (int64_t)sid; } - try{ - auto splitter = Splitter::Make(partitioning_name, inputDataTypesTmp, jNumCols, num_partitions, std::move(splitOptions)); - return shuffle_splitter_holder_.Insert(std::shared_ptr(splitter)); - } catch (omniruntime::exception::OmniException & e) { - env->ThrowNew(runtime_exception_class, e.what()); - } + auto splitter = Splitter::Make(partitioning_name, inputDataTypesTmp, jNumCols, num_partitions, std::move(splitOptions)); + return shuffle_splitter_holder_.Insert(std::shared_ptr(splitter)); + JNI_FUNC_END(runtime_exception_class) } JNIEXPORT jlong JNICALL Java_com_huawei_boostkit_spark_jni_SparkJniWrapper_split( JNIEnv *env, jobject jObj, jlong splitter_id, jlong jVecBatchAddress) { + JNI_FUNC_START auto splitter = shuffle_splitter_holder_.Lookup(splitter_id); if (!splitter) { std::string error_message = "Invalid splitter id " + std::to_string(splitter_id); - env->ThrowNew(env->FindClass("java/lang/Exception"), error_message.c_str()); + env->ThrowNew(runtime_exception_class, error_message.c_str()); return -1; } auto vecBatch = (VectorBatch *) jVecBatchAddress; - try { - splitter->Split(*vecBatch); - } catch (omniruntime::exception::OmniException & e) { - env->ThrowNew(runtime_exception_class, e.what()); - } + splitter->Split(*vecBatch); + JNI_FUNC_END(runtime_exception_class) } JNIEXPORT jobject JNICALL Java_com_huawei_boostkit_spark_jni_SparkJniWrapper_stop( JNIEnv* env, jobject, jlong splitter_id) { + JNI_FUNC_START auto splitter = shuffle_splitter_holder_.Lookup(splitter_id); if (!splitter) { std::string error_message = "Invalid splitter id " + std::to_string(splitter_id); - env->ThrowNew(env->FindClass("java/lang/Exception"), error_message.c_str()); - } - try { - splitter->Stop(); - } catch (omniruntime::exception::OmniException & e) { - env->ThrowNew(runtime_exception_class, e.what()); + env->ThrowNew(runtime_exception_class, error_message.c_str()); } + splitter->Stop(); + const auto& partition_length = splitter->PartitionLengths(); auto partition_length_arr = env->NewLongArray(partition_length.size()); auto src = reinterpret_cast(partition_length.data()); @@ -211,15 +204,18 @@ Java_com_huawei_boostkit_spark_jni_SparkJniWrapper_stop( splitter->TotalBytesWritten(), splitter->TotalBytesSpilled(), partition_length_arr); return split_result; + JNI_FUNC_END(runtime_exception_class) } JNIEXPORT void JNICALL Java_com_huawei_boostkit_spark_jni_SparkJniWrapper_close( JNIEnv* env, jobject, jlong splitter_id) { + JNI_FUNC_START auto splitter = shuffle_splitter_holder_.Lookup(splitter_id); if (!splitter) { std::string error_message = "Invalid splitter id " + std::to_string(splitter_id); - env->ThrowNew(env->FindClass("java/lang/Exception"), error_message.c_str()); + env->ThrowNew(runtime_exception_class, error_message.c_str()); } shuffle_splitter_holder_.Erase(splitter_id); + JNI_FUNC_END(runtime_exception_class) } diff --git a/omnioperator/omniop-spark-extension/cpp/src/jni/jni_common.cpp b/omnioperator/omniop-spark-extension/cpp/src/jni/jni_common.cpp new file mode 100644 index 0000000000000000000000000000000000000000..43d8b755774fd167d6b381dd2057049c5acf7d25 --- /dev/null +++ b/omnioperator/omniop-spark-extension/cpp/src/jni/jni_common.cpp @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef THESTRAL_PLUGIN_MASTER_JNI_COMMON_CPP +#define THESTRAL_PLUGIN_MASTER_JNI_COMMON_CPP + +#include "jni_common.h" + +spark::CompressionKind GetCompressionType(JNIEnv* env, jstring codec_jstr) { + auto codec_c = env->GetStringUTFChars(codec_jstr, JNI_FALSE); + auto codec = std::string(codec_c); + auto compression_type = GetCompressionType(codec); + env->ReleaseStringUTFChars(codec_jstr, codec_c); + return compression_type; +} + +jclass CreateGlobalClassReference(JNIEnv* env, const char* class_name) { + jclass local_class = env->FindClass(class_name); + jclass global_class = (jclass)env->NewGlobalRef(local_class); + env->DeleteLocalRef(local_class); + if (global_class == nullptr) { + std::string error_message = "Unable to createGlobalClassReference for" + std::string(class_name); + env->ThrowNew(illegal_access_exception_class, error_message.c_str()); + } + return global_class; +} + +jmethodID GetMethodID(JNIEnv* env, jclass this_class, const char* name, const char* sig) { + jmethodID ret = env->GetMethodID(this_class, name, sig); + if (ret == nullptr) { + std::string error_message = "Unable to find method " + std::string(name) + " within signature" + std::string(sig); + env->ThrowNew(illegal_access_exception_class, error_message.c_str()); + } + + return ret; +} + +#endif //THESTRAL_PLUGIN_MASTER_JNI_COMMON_CPP diff --git a/omnioperator/omniop-spark-extension/cpp/src/jni/jni_common.h b/omnioperator/omniop-spark-extension/cpp/src/jni/jni_common.h index d67561a58648cd26699130bb0a2aa2bfd680c5e3..133dd7b99bc14a4847bbd0991dec5430caa81afa 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/jni/jni_common.h +++ b/omnioperator/omniop-spark-extension/cpp/src/jni/jni_common.h @@ -26,32 +26,19 @@ static jclass illegal_access_exception_class; -spark::CompressionKind GetCompressionType(JNIEnv* env, jstring codec_jstr) { - auto codec_c = env->GetStringUTFChars(codec_jstr, JNI_FALSE); - auto codec = std::string(codec_c); - auto compression_type = GetCompressionType(codec); - env->ReleaseStringUTFChars(codec_jstr, codec_c); - return compression_type; -} - -jclass CreateGlobalClassReference(JNIEnv* env, const char* class_name) { - jclass local_class = env->FindClass(class_name); - jclass global_class = (jclass)env->NewGlobalRef(local_class); - env->DeleteLocalRef(local_class); - if (global_class == nullptr) { - std::string error_message = "Unable to createGlobalClassReference for" + std::string(class_name); - env->ThrowNew(illegal_access_exception_class, error_message.c_str()); - } - return global_class; -} - -jmethodID GetMethodID(JNIEnv* env, jclass this_class, const char* name, const char* sig) { - jmethodID ret = env->GetMethodID(this_class, name, sig); - if (ret == nullptr) { - std::string error_message = "Unable to find method " + std::string(name) + " within signature" + std::string(sig); - env->ThrowNew(illegal_access_exception_class, error_message.c_str()); - } - - return ret; -} +spark::CompressionKind GetCompressionType(JNIEnv* env, jstring codec_jstr); + +jclass CreateGlobalClassReference(JNIEnv* env, const char* class_name); + +jmethodID GetMethodID(JNIEnv* env, jclass this_class, const char* name, const char* sig); + +#define JNI_FUNC_START try { + +#define JNI_FUNC_END(exceptionClass) \ + } \ + catch (const std::exception &e) \ + { \ + env->ThrowNew(exceptionClass, e.what()); \ + } \ + #endif //THESTRAL_PLUGIN_MASTER_JNI_COMMON_H diff --git a/omnioperator/omniop-spark-extension/cpp/test/tablescan/scan_test.cpp b/omnioperator/omniop-spark-extension/cpp/test/tablescan/scan_test.cpp index 37c340a8d203646d9963e54231b931e797d52021..f8a6a6b7f2776f212d7ba6b1c9ee8d9260509116 100644 --- a/omnioperator/omniop-spark-extension/cpp/test/tablescan/scan_test.cpp +++ b/omnioperator/omniop-spark-extension/cpp/test/tablescan/scan_test.cpp @@ -258,29 +258,35 @@ TEST_F(ScanTest, test_copy_shortVec) TEST_F(ScanTest, test_build_leafs) { - int leafOp = 0; std::vector litList; std::string leafNameString; - int leafType = 0; std::unique_ptr builder = orc::SearchArgumentFactory::newBuilder(); (*builder).startAnd(); orc::Literal lit(100L); - - // test equal - buildLeafs(0, litList, lit, "leaf-0", 0, *builder); + // test EQUALS + buildLeaves(PredicateOperatorType::EQUALS, litList, lit, "leaf-0", orc::PredicateDataType::LONG, *builder); // test LESS_THAN - buildLeafs(2, litList, lit, "leaf-1", 0, *builder); + buildLeaves(PredicateOperatorType::LESS_THAN, litList, lit, "leaf-1", orc::PredicateDataType::LONG, *builder); // test LESS_THAN_EQUALS - buildLeafs(3, litList, lit, "leaf-1", 0, *builder); + buildLeaves(PredicateOperatorType::LESS_THAN_EQUALS, litList, lit, "leaf-1", orc::PredicateDataType::LONG, *builder); // test NULL_SAFE_EQUALS - buildLeafs(1, litList, lit, "leaf-1", 0, *builder); + buildLeaves(PredicateOperatorType::NULL_SAFE_EQUALS, litList, lit, "leaf-1", orc::PredicateDataType::LONG, *builder); // test IS_NULL - buildLeafs(6, litList, lit, "leaf-1", 0, *builder); + buildLeaves(PredicateOperatorType::IS_NULL, litList, lit, "leaf-1", orc::PredicateDataType::LONG, *builder); + + // test BETWEEN + std::string tmpStr = ""; + try { + buildLeaves(PredicateOperatorType::BETWEEN, litList, lit, "leaf-1", orc::PredicateDataType::LONG, *builder); + } catch (std::exception &e) { + tmpStr = e.what(); + } + ASSERT_EQ(tmpStr, "table scan buildLeaves BETWEEN is not supported!"); std::string result = ((*builder).end().build())->toString(); std::string buildString = diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReaderTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReaderTest.java index 58cb5e1bc6a95bf72e7cb73364121267024b4b58..99801bcfb86567a5a2cb44dc43e4428496b00ed3 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReaderTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReaderTest.java @@ -118,4 +118,15 @@ public class OrcColumnarBatchJniReaderTest extends TestCase { vecs[1].close(); } + @Test + public void testGetProgress() { + String tmp = ""; + try { + double progressValue = orcColumnarBatchJniReader.getProgress(); + } catch (Exception e) { + tmp = e.getMessage(); + } finally { + assertTrue(tmp.equals("recordReaderGetProgress is unsupported")); + } + } }