diff --git a/.workflow/branch-pipeline.yml b/.workflow/branch-pipeline.yml new file mode 100644 index 0000000000000000000000000000000000000000..9d2a2926f31b55f68083a3cb58c4cb6d6f43a07f --- /dev/null +++ b/.workflow/branch-pipeline.yml @@ -0,0 +1,53 @@ +version: '1.0' +name: branch-pipeline +displayName: BranchPipeline +stages: + - stage: + name: compile + displayName: 编译 + steps: + - step: build@maven + name: build_maven + displayName: Maven 构建 + # 支持6、7、8、9、10、11六个版本 + jdkVersion: 8 + # 支持2.2.1、3.2.5、3.3.9、3.5.2、3.5.3、3.5.4、3.6.1、3.6.3八个版本 + mavenVersion: 3.3.9 + # 构建命令 + commands: + - mvn -B clean package -Dmaven.test.skip=true + # 非必填字段，开启后表示将构建产物暂存，但不会上传到制品库中，7天后自动清除 + artifacts: + # 构建产物名字，作为产物的唯一标识可向下传递，支持自定义，默认为BUILD_ARTIFACT。在下游可以通过${BUILD_ARTIFACT}方式引用来获取构建物地址 + - name: BUILD_ARTIFACT + # 构建产物获取路径，是指代码编译完毕之后构建物的所在路径，如通常jar包在target目录下。当前目录为代码库根目录 + path: + - ./target + - step: publish@general_artifacts + name: publish_general_artifacts + displayName: 上传制品 + # 上游构建任务定义的产物名，默认BUILD_ARTIFACT + dependArtifact: BUILD_ARTIFACT + # 上传到制品库时的制品命名，默认output + artifactName: output + dependsOn: build_maven + - stage: + name: release + displayName: 发布 + steps: + - step: publish@release_artifacts + name: publish_release_artifacts + displayName: '发布' + # 上游上传制品任务的产出 + dependArtifact: output + # 发布制品版本号 + version: '1.0.0.0' + # 是否开启版本号自增，默认开启 + autoIncrement: true +triggers: + push: + branches: + exclude: + - master + include: + - .* diff --git a/.workflow/master-pipeline.yml b/.workflow/master-pipeline.yml new file mode 100644 index 0000000000000000000000000000000000000000..5d926c26f79e41d10427f87bf003ef99fe2fbd79 --- /dev/null +++ b/.workflow/master-pipeline.yml @@ -0,0 +1,51 @@ +version: '1.0' +name: master-pipeline +displayName: MasterPipeline +stages: + - stage: + name: compile + displayName: 编译 + steps: + - step: build@maven + name: build_maven + displayName: Maven 构建 + # 支持6、7、8、9、10、11六个版本 + jdkVersion: 8 + # 支持2.2.1、3.2.5、3.3.9、3.5.2、3.5.3、3.5.4、3.6.1、3.6.3八个版本 + mavenVersion: 3.3.9 + # 构建命令 + commands: + - mvn -B clean package -Dmaven.test.skip=true + # 非必填字段，开启后表示将构建产物暂存，但不会上传到制品库中，7天后自动清除 + artifacts: + # 构建产物名字，作为产物的唯一标识可向下传递，支持自定义，默认为BUILD_ARTIFACT。在下游可以通过${BUILD_ARTIFACT}方式引用来获取构建物地址 + - name: BUILD_ARTIFACT + # 构建产物获取路径，是指代码编译完毕之后构建物的所在路径，如通常jar包在target目录下。当前目录为代码库根目录 + path: + - ./target + - step: publish@general_artifacts + name: publish_general_artifacts + displayName: 上传制品 + # 上游构建任务定义的产物名，默认BUILD_ARTIFACT + dependArtifact: BUILD_ARTIFACT + # 上传到制品库时的制品命名，默认output + artifactName: output + dependsOn: build_maven + - stage: + name: release + displayName: 发布 + steps: + - step: publish@release_artifacts + name: publish_release_artifacts + displayName: '发布' + # 上游上传制品任务的产出 + dependArtifact: output + # 发布制品版本号 + version: '1.0.0.0' + # 是否开启版本号自增，默认开启 + autoIncrement: true +triggers: + push: + branches: + include: + - master diff --git a/.workflow/pr-pipeline.yml b/.workflow/pr-pipeline.yml new file mode 100644 index 0000000000000000000000000000000000000000..3f7579dd405c85f97f77df0357ec4893e616f85f --- /dev/null +++ b/.workflow/pr-pipeline.yml @@ -0,0 +1,40 @@ +version: '1.0' +name: pr-pipeline +displayName: PRPipeline +stages: + - stage: + name: compile + displayName: 编译 + steps: + - step: build@maven + name: build_maven + displayName: Maven 构建 + # 支持6、7、8、9、10、11六个版本 + jdkVersion: 8 + # 支持2.2.1、3.2.5、3.3.9、3.5.2、3.5.3、3.5.4、3.6.1、3.6.3八个版本 + mavenVersion: 3.3.9 + # 构建命令 + commands: + - mvn -B clean package -Dmaven.test.skip=true + # 非必填字段，开启后表示将构建产物暂存，但不会上传到制品库中，7天后自动清除 + artifacts: + # 构建产物名字，作为产物的唯一标识可向下传递，支持自定义，默认为BUILD_ARTIFACT。在下游可以通过${BUILD_ARTIFACT}方式引用来获取构建物地址 + - name: BUILD_ARTIFACT + # 构建产物获取路径，是指代码编译完毕之后构建物的所在路径，如通常jar包在target目录下。当前目录为代码库根目录 + path: + - ./target + - step: publish@general_artifacts + name: publish_general_artifacts + displayName: 上传制品 + # 上游构建任务定义的产物名，默认BUILD_ARTIFACT + dependArtifact: BUILD_ARTIFACT + # 构建产物制品库，默认default，系统默认创建 + artifactRepository: default + # 上传到制品库时的制品命名，默认output + artifactName: output + dependsOn: build_maven +triggers: + pr: + branches: + include: + - master diff --git a/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp b/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp index 1dd18929b5cc94c99a15ba5b7ded8e7a007c0f2d..0774bb9ff0886c9c1dad5e7e892339ba470a0bee 100644 --- a/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp +++ b/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp @@ -356,6 +356,69 @@ std::unique_ptr CopyFixedWidth(orc::ColumnVectorBatch *field) return newVector; } + +template +std::unique_ptr CopyVarForOmniDouble(orc::ColumnVectorBatch *field) +{ + using T = typename NativeType::type; + ORC_TYPE *lvb = dynamic_cast(field); + auto numElements = lvb->numElements; + auto values = lvb->data.data(); + auto notNulls = lvb->notNull.data(); + auto newVector = std::make_unique>>(numElements); + auto newVectorPtr = newVector.get(); + // Check ColumnVectorBatch has null or not firstly + if (lvb->hasNulls) { + for (uint i = 0; i < numElements; i++) { + if (!notNulls[i]) { + newVectorPtr->SetNull(i); + } else { + auto data = std::strtod(value[i], NULL); + newVectorPtr->SetValue(i, data); + } + } + } else { + for (uint i = 0; i < numElements; i++) { + auto data = std::strtod(value[i], NULL); + newVectorPtr->SetValue(i, data); + } + } + return newVector; +} + + +template +std::unique_ptr CopyLongForOmniVar(orc::ColumnVectorBatch *field) +{ + using T = typename NativeType::type; + ORC_TYPE *lvb = dynamic_cast(field); + auto numElements = lvb->numElements; + auto values = lvb->data.data(); + auto notNulls = lvb->notNull.data(); + auto newVector = std::make_unique>>(numElements); + auto newVectorPtr = newVector.get(); + // Check ColumnVectorBatch has null or not firstly + if (lvb->hasNulls) { + for (uint i = 0; i < numElements; i++) { + if (!notNulls[i]) { + newVectorPtr->SetNull(i); + } else { + auto str = std::to_string(values[i]); + auto data = std::string_view(str); + newVectorPtr->SetValue(i, data); + } + } + } else { + for (uint i = 0; i < numElements; i++) { + auto str = std::to_string(values[i]); + auto data = std::string_view(str); + newVectorPtr->SetValue(i, data); + } + } + newVectorPtr->SetValues(0, values, numElements); + return newVector; +} + template std::unique_ptr CopyOptimizedForInt64(orc::ColumnVectorBatch *field) { @@ -445,6 +508,32 @@ std::unique_ptr CopyCharType(orc::ColumnVectorBatch *field) return newVector; } +std::unique_ptr CopyDoubleToOmniDecimal64Vec(orc::ColumnVectorBatch *field) +{ + orc::DoubleVectorBatch *lvb = dynamic_cast(field); + auto numElements = lvb->numElements; + auto values = lvb->data.data(); + auto notNulls = lvb->notNull.data(); + auto newVector = std::make_unique>(numElements); + auto newVectorPtr = newVector.get(); + if (lvb->hasNulls) { + for (uint i = 0; i < numElements; i++) { + if (notNulls[i]) { + omniruntime::type::Decimal64 d(values[i]); + newVectorPtr->SetValue(i, d.GetValue()); + } else { + newVectorPtr->SetNull(i); + } + } + } else { + for (uint i = 0; i < numElements; i++) { + omniruntime::type::Decimal64 d(values[i]); + newVectorPtr->SetValue(i, d.GetValue()); + } + } + return newVector; +} + std::unique_ptr CopyToOmniDecimal128Vec(orc::ColumnVectorBatch *field) { orc::Decimal128VectorBatch *lvb = dynamic_cast(field); @@ -532,6 +621,8 @@ std::unique_ptr DealLongVectorBatch(DataTypeId id, orc::ColumnVector return CopyFixedWidth(field); case omniruntime::type::OMNI_DATE64: return CopyOptimizedForInt64(field); + case omniruntime::type::OMNI_VARCHAR: + return CopyLongForOmniVarr(field); default: { throw std::runtime_error("DealLongVectorBatch not support for type: " + id); } @@ -542,6 +633,8 @@ std::unique_ptr DealDoubleVectorBatch(DataTypeId id, orc::ColumnVect switch (id) { case omniruntime::type::OMNI_DOUBLE: return CopyOptimizedForInt64(field); + case omniruntime::type::OMNI_DECIMAL64: + return CopyDoubleToOmniDecimal64Vec(field); default: { throw std::runtime_error("DealDoubleVectorBatch not support for type: " + id); } @@ -592,6 +685,9 @@ std::unique_ptr CopyToOmniVec(const orc::Type *type, int omniTypeId, return CopyCharType(field); case orc::TypeKind::STRING: case orc::TypeKind::VARCHAR: + if (dataTypeId == OMNI_DOUBLE) { + return CopyVarForOmniDouble(field); + } if (dataTypeId != OMNI_VARCHAR) { throw std::runtime_error("Cannot transfer to other OMNI_TYPE but VARCHAR for orc string/varchar"); }