From a02f002fd73edb5724a6a959dfbb015674c4aae1 Mon Sep 17 00:00:00 2001 From: xiongyutian Date: Thu, 18 Jan 2024 21:51:51 +0800 Subject: [PATCH] adapt new datatype Timestamp --- .../cpp/src/jni/OrcColumnarBatchJniReader.cpp | 4 ++++ .../cpp/src/parquet/ParquetTypedRecordReader.cpp | 3 +++ .../cpp/src/shuffle/splitter.cpp | 4 ++++ .../spark/jni/OrcColumnarBatchScanReader.java | 1 + .../spark/jni/ParquetColumnarBatchScanReader.java | 2 ++ .../spark/serialize/ShuffleDataSerializer.java | 4 ++++ .../sql/execution/vectorized/OmniColumnVector.java | 14 ++++++++------ .../scala/com/huawei/boostkit/spark/Constant.scala | 1 + .../spark/expression/OmniExpressionAdaptor.scala | 9 ++++++--- .../boostkit/spark/util/OmniAdaptorUtil.scala | 2 +- .../joins/ColumnarSortMergeJoinExec.scala | 2 +- .../spark/sql/execution/util/MergeIterator.scala | 4 ++-- .../spark/ColumnShuffleCompressionTest.java | 1 + .../spark/ColumnShuffleDiffPartitionTest.java | 1 + .../boostkit/spark/ColumnShuffleDiffRowVBTest.java | 1 + .../boostkit/spark/ColumnShuffleGBSizeTest.java | 1 + .../boostkit/spark/ColumnShuffleNullTest.java | 1 + .../huawei/boostkit/spark/ColumnShuffleTest.java | 6 ++++++ 18 files changed, 48 insertions(+), 13 deletions(-) diff --git a/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp b/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp index bb6efe749..a93c5f8e7 100644 --- a/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp +++ b/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp @@ -525,6 +525,10 @@ int CopyToOmniVec(const orc::Type *type, int &omniTypeId, uint64_t &omniVecId, o omniTypeId = static_cast(OMNI_LONG); omniVecId = CopyOptimizedForInt64(field); break; + case orc::TypeKind::TIMESTAMP: + omniTypeId = static_cast(OMNI_TIMESTAMP); + omniVecId = CopyOptimizedForInt64(field); + break; case orc::TypeKind::DOUBLE: omniTypeId = static_cast(OMNI_DOUBLE); omniVecId = CopyOptimizedForInt64(field); diff --git a/omnioperator/omniop-native-reader/cpp/src/parquet/ParquetTypedRecordReader.cpp b/omnioperator/omniop-native-reader/cpp/src/parquet/ParquetTypedRecordReader.cpp index 6251044a8..8fc203ddb 100644 --- a/omnioperator/omniop-native-reader/cpp/src/parquet/ParquetTypedRecordReader.cpp +++ b/omnioperator/omniop-native-reader/cpp/src/parquet/ParquetTypedRecordReader.cpp @@ -332,6 +332,9 @@ std::shared_ptr MakeRecordReader(const ColumnDescriptor* descr case ::arrow::Type::INT64: { return std::make_shared>(descr, leaf_info, pool); } + case ::arrow::Type::TIMESTAMP: { + return std::make_shared>(descr, leaf_info, pool); + } case ::arrow::Type::DATE64: { return std::make_shared>(descr, leaf_info, pool); diff --git a/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp b/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp index 14b65f06e..66236eeb0 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp +++ b/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp @@ -485,6 +485,10 @@ void Splitter::ToSplitterTypeId(int num_cols) CastOmniToShuffleType(OMNI_LONG, SHUFFLE_8BYTE); break; } + case OMNI_TIMESTAMP: { + CastOmniToShuffleType(OMNI_TIMESTAMP, SHUFFLE_8BYTE); + break; + } case OMNI_DOUBLE: { CastOmniToShuffleType(OMNI_DOUBLE, SHUFFLE_8BYTE); break; diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchScanReader.java b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchScanReader.java index 227a00e15..25e43adb7 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchScanReader.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchScanReader.java @@ -289,6 +289,7 @@ public class OrcColumnarBatchScanReader { break; } case OMNI_LONG: + case OMNI_TIMESTAMP: case OMNI_DECIMAL64: { vecList[i] = new LongVec(vecNativeIds[nativeGetId]); break; diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/ParquetColumnarBatchScanReader.java b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/ParquetColumnarBatchScanReader.java index 5275a8ecf..005a3e120 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/ParquetColumnarBatchScanReader.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/ParquetColumnarBatchScanReader.java @@ -72,6 +72,8 @@ public class ParquetColumnarBatchScanReader { DataType type = types.get(i); if (type instanceof LongType) { vecList[i] = new LongVec(vecNativeIds[i]); + } else if (type instanceof TimestampType) { + vecList[i] = new LongVec(vecNativeIds[i]); } else if (type instanceof BooleanType) { vecList[i] = new BooleanVec(vecNativeIds[i]); } else if (type instanceof ShortType) { diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/serialize/ShuffleDataSerializer.java b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/serialize/ShuffleDataSerializer.java index 9f6cadf70..c429380a0 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/serialize/ShuffleDataSerializer.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/serialize/ShuffleDataSerializer.java @@ -70,6 +70,10 @@ public class ShuffleDataSerializer { type = DataTypes.LongType; vec = new LongVec(vecSize); break; + case VEC_TYPE_TIMESTAMP: + type = DataTypes.TimestampType; + vec = new LongVec(vecSize); + break; case VEC_TYPE_DATE64: type = DataTypes.DateType; vec = new LongVec(vecSize); diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/org/apache/spark/sql/execution/vectorized/OmniColumnVector.java b/omnioperator/omniop-spark-extension/java/src/main/java/org/apache/spark/sql/execution/vectorized/OmniColumnVector.java index 3ba4c6e06..411f26b5a 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/org/apache/spark/sql/execution/vectorized/OmniColumnVector.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/org/apache/spark/sql/execution/vectorized/OmniColumnVector.java @@ -95,7 +95,7 @@ public class OmniColumnVector extends WritableColumnVector { return dictionaryData; } - if (type instanceof LongType) { + if (type instanceof LongType || type instanceof TimestampType) { return longDataVec; } else if (type instanceof BooleanType || type instanceof NullType) { return booleanDataVec; @@ -130,7 +130,7 @@ public class OmniColumnVector extends WritableColumnVector { public void setVec(Vec vec) { if (vec instanceof DictionaryVec) { dictionaryData = (DictionaryVec) vec; - } else if (type instanceof LongType) { + } else if (type instanceof LongType || type instanceof TimestampType) { this.longDataVec = (LongVec) vec; } else if (type instanceof DecimalType) { if (DecimalType.is64BitDecimalType(type)) { @@ -217,7 +217,7 @@ public class OmniColumnVector extends WritableColumnVector { } else { return decimal128DataVec.hasNull(); } - } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type)) { + } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { return longDataVec.hasNull(); } else if (type instanceof FloatType) { return false; @@ -259,7 +259,7 @@ public class OmniColumnVector extends WritableColumnVector { } else { decimal128DataVec.setNull(rowId); } - } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type)) { + } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { longDataVec.setNull(rowId); } else if (type instanceof FloatType) { return; @@ -294,7 +294,7 @@ public class OmniColumnVector extends WritableColumnVector { } else { decimal128DataVec.setNulls(rowId, nullValue, 0, count); } - } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type)) { + } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { longDataVec.setNulls(rowId, nullValue, 0, count); } else if (type instanceof FloatType) { return; @@ -329,7 +329,7 @@ public class OmniColumnVector extends WritableColumnVector { } else { return decimal128DataVec.isNull(rowId); } - } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type)) { + } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { return longDataVec.isNull(rowId); } else if (type instanceof FloatType) { return false; @@ -854,6 +854,8 @@ public class OmniColumnVector extends WritableColumnVector { } } else if (type instanceof LongType) { longDataVec = new LongVec(newCapacity); + } else if (type instanceof TimestampType) { + longDataVec = new LongVec(newCapacity); } else if (type instanceof FloatType) { throw new UnsupportedOperationException("reserveInternal is not supported for type:" + type); } else if (type instanceof DoubleType) { diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/Constant.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/Constant.scala index 9d7f844bc..f1d354e29 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/Constant.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/Constant.scala @@ -29,6 +29,7 @@ object Constant { val OMNI_SHOR_TYPE: String = DataTypeId.OMNI_SHORT.ordinal().toString val OMNI_INTEGER_TYPE: String = DataTypeId.OMNI_INT.ordinal().toString val OMNI_LONG_TYPE: String = DataTypeId.OMNI_LONG.ordinal().toString + val OMNI_TIMESTAMP_TYPE: String = DataTypeId.OMNI_TIMESTAMP.ordinal().toString val OMNI_DOUBLE_TYPE: String = DataTypeId.OMNI_DOUBLE.ordinal().toString val OMNI_BOOLEAN_TYPE: String = DataTypeId.OMNI_BOOLEAN.ordinal().toString val OMNI_DATE_TYPE: String = DataTypeId.OMNI_DATE32.ordinal().toString diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index 1ad38834a..52cc93362 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -20,8 +20,8 @@ package com.huawei.boostkit.spark.expression import scala.collection.mutable.ArrayBuffer -import com.huawei.boostkit.spark.Constant.{DEFAULT_STRING_TYPE_LENGTH, IS_CHECK_OMNI_EXP, OMNI_BOOLEAN_TYPE, OMNI_DATE_TYPE, OMNI_DECIMAL128_TYPE, OMNI_DECIMAL64_TYPE, OMNI_DOUBLE_TYPE, OMNI_INTEGER_TYPE, OMNI_LONG_TYPE, OMNI_SHOR_TYPE, OMNI_VARCHAR_TYPE} -import nova.hetu.omniruntime.`type`.{BooleanDataType, DataTypeSerializer, Date32DataType, Decimal128DataType, Decimal64DataType, DoubleDataType, IntDataType, LongDataType, ShortDataType, VarcharDataType} +import com.huawei.boostkit.spark.Constant.{DEFAULT_STRING_TYPE_LENGTH, IS_CHECK_OMNI_EXP, OMNI_BOOLEAN_TYPE, OMNI_DATE_TYPE, OMNI_DECIMAL128_TYPE, OMNI_DECIMAL64_TYPE, OMNI_DOUBLE_TYPE, OMNI_INTEGER_TYPE, OMNI_LONG_TYPE, OMNI_TIMESTAMP_TYPE, OMNI_SHOR_TYPE, OMNI_VARCHAR_TYPE} +import nova.hetu.omniruntime.`type`.{BooleanDataType, DataTypeSerializer, Date32DataType, Decimal128DataType, Decimal64DataType, DoubleDataType, IntDataType, LongDataType, TimestampDataType, ShortDataType, VarcharDataType} import nova.hetu.omniruntime.constants.FunctionType import nova.hetu.omniruntime.constants.FunctionType.{OMNI_AGGREGATION_TYPE_AVG, OMNI_AGGREGATION_TYPE_COUNT_ALL, OMNI_AGGREGATION_TYPE_COUNT_COLUMN, OMNI_AGGREGATION_TYPE_FIRST_INCLUDENULL, OMNI_AGGREGATION_TYPE_FIRST_IGNORENULL, OMNI_AGGREGATION_TYPE_MAX, OMNI_AGGREGATION_TYPE_MIN, OMNI_AGGREGATION_TYPE_SUM, OMNI_WINDOW_TYPE_RANK, OMNI_WINDOW_TYPE_ROW_NUMBER} import nova.hetu.omniruntime.constants.JoinType._ @@ -874,6 +874,7 @@ object OmniExpressionAdaptor extends Logging { case ShortType => OMNI_SHOR_TYPE case IntegerType => OMNI_INTEGER_TYPE case LongType => OMNI_LONG_TYPE + case TimestampType => OMNI_TIMESTAMP_TYPE case DoubleType => OMNI_DOUBLE_TYPE case BooleanType => OMNI_BOOLEAN_TYPE case StringType => OMNI_VARCHAR_TYPE @@ -911,6 +912,8 @@ object OmniExpressionAdaptor extends Logging { IntDataType.INTEGER case LongType => LongDataType.LONG + case TimestampType => + TimestampDataType.TIMESTAMP case DoubleType => DoubleDataType.DOUBLE case BooleanType => @@ -935,7 +938,7 @@ object OmniExpressionAdaptor extends Logging { val metadata = attr.metadata val omniDataType: String = sparkTypeToOmniExpType(dataType) dataType match { - case ShortType | IntegerType | LongType | DoubleType | BooleanType | DateType => + case ShortType | IntegerType | LongType | DoubleType | TimestampType | BooleanType | DateType => "{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":%s,\"colVal\":%d}" .format(omniDataType, colVal) case StringType => diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/util/OmniAdaptorUtil.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/util/OmniAdaptorUtil.scala index ed99f6b43..0fad75e13 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/util/OmniAdaptorUtil.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/util/OmniAdaptorUtil.scala @@ -65,7 +65,7 @@ object OmniAdaptorUtil { def transColumnVector(columnVector: ColumnVector, columnSize : Int): Vec = { val datatype: DataType = columnVector.dataType() val vec: Vec = datatype match { - case LongType => + case LongType | TimestampType => val vec = new LongVec(columnSize) val values = new Array[Long](columnSize) for (i <- 0 until columnSize) { diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/joins/ColumnarSortMergeJoinExec.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/joins/ColumnarSortMergeJoinExec.scala index 6718e5e7f..d31d493ab 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/joins/ColumnarSortMergeJoinExec.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/joins/ColumnarSortMergeJoinExec.scala @@ -435,7 +435,7 @@ case class ColumnarSortMergeJoinExec( vecs(i) = types(i).getId match { case DataType.DataTypeId.OMNI_INT | DataType.DataTypeId.OMNI_DATE32 => new IntVec(0) - case DataType.DataTypeId.OMNI_LONG | DataType.DataTypeId.OMNI_DECIMAL64 => + case DataType.DataTypeId.OMNI_LONG | DataType.DataTypeId.OMNI_TIMESTAMP | DataType.DataTypeId.OMNI_DECIMAL64 => new LongVec(0) case DataType.DataTypeId.OMNI_DOUBLE => new DoubleVec(0) diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/util/MergeIterator.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/util/MergeIterator.scala index 017eaba23..ea37d48c7 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/util/MergeIterator.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/util/MergeIterator.scala @@ -27,7 +27,7 @@ import nova.hetu.omniruntime.`type`.{DataType, VarcharDataType} import nova.hetu.omniruntime.vector.{BooleanVec, Decimal128Vec, DoubleVec, IntVec, LongVec, ShortVec, VarcharVec, Vec, VecBatch} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.vectorized.OmniColumnVector -import org.apache.spark.sql.types.{BooleanType, DateType, DecimalType, DoubleType, IntegerType, LongType, ShortType, StringType, StructType} +import org.apache.spark.sql.types.{BooleanType, DateType, DecimalType, DoubleType, IntegerType, LongType, TimestampType, ShortType, StringType, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch class MergeIterator(iter: Iterator[ColumnarBatch], localSchema: StructType, @@ -46,7 +46,7 @@ class MergeIterator(iter: Iterator[ColumnarBatch], localSchema: StructType, val vecs = new Array[Vec](schema.fields.length) schema.fields.zipWithIndex.foreach { case (field, index) => field.dataType match { - case LongType => + case LongType | TimestampType => vecs(index) = new LongVec(columnSize) case DateType | IntegerType => vecs(index) = new IntVec(columnSize) diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleCompressionTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleCompressionTest.java index d95be1883..b814ff22c 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleCompressionTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleCompressionTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffPartitionTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffPartitionTest.java index c8fd47413..982d90aa0 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffPartitionTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffPartitionTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffRowVBTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffRowVBTest.java index dc53fda8a..7b1568864 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffRowVBTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffRowVBTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleGBSizeTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleGBSizeTest.java index 2ef81ac49..90c74633a 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleGBSizeTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleGBSizeTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleNullTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleNullTest.java index 98fc18dd8..0b2bf2e31 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleNullTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleNullTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleTest.java index 8be5702df..d5b4e474d 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleTest.java @@ -28,6 +28,7 @@ import nova.hetu.omniruntime.type.Decimal64DataType; import nova.hetu.omniruntime.type.DoubleDataType; import nova.hetu.omniruntime.type.IntDataType; import nova.hetu.omniruntime.type.LongDataType; +import nova.hetu.omniruntime.type.TimestampDataType; import nova.hetu.omniruntime.type.VarcharDataType; import nova.hetu.omniruntime.vector.Decimal128Vec; import nova.hetu.omniruntime.vector.DoubleVec; @@ -76,6 +77,10 @@ abstract class ColumnShuffleTest { types[i] = Date64DataType.DATE64; break; } + case OMNI_TIMESTAMP: { + types[i] = TimestampDataType.TIMESTAMP; + break; + } case OMNI_DECIMAL64: { types[i] = Decimal64DataType.DECIMAL64; break; @@ -118,6 +123,7 @@ abstract class ColumnShuffleTest { break; } case OMNI_LONG: + case OMNI_TIMESTAMP: case OMNI_DECIMAL64: case OMNI_DATE64: { tmpVec = new LongVec(rowNum); -- Gitee