diff --git a/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp b/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp index 9fa980fba9ce9608d30ee3f3e3e6ea1e9eff213b..c0cd5cf696fbf10a78e72b46ae6424732e2f6c21 100644 --- a/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp +++ b/omnioperator/omniop-native-reader/cpp/src/jni/OrcColumnarBatchJniReader.cpp @@ -523,6 +523,10 @@ int CopyToOmniVec(const orc::Type *type, int &omniTypeId, uint64_t &omniVecId, o omniTypeId = static_cast(OMNI_LONG); omniVecId = CopyOptimizedForInt64(field); break; + case orc::TypeKind::TIMESTAMP: + omniTypeId = static_cast(OMNI_TIMESTAMP); + omniVecId = CopyOptimizedForInt64(field); + break; case orc::TypeKind::DOUBLE: omniTypeId = static_cast(OMNI_DOUBLE); omniVecId = CopyOptimizedForInt64(field); diff --git a/omnioperator/omniop-native-reader/cpp/src/parquet/ParquetTypedRecordReader.cpp b/omnioperator/omniop-native-reader/cpp/src/parquet/ParquetTypedRecordReader.cpp index 6251044a85da44926b049f313bef92813e69552a..8fc203ddbec67e1198c5a3fadd51efc22c28db54 100644 --- a/omnioperator/omniop-native-reader/cpp/src/parquet/ParquetTypedRecordReader.cpp +++ b/omnioperator/omniop-native-reader/cpp/src/parquet/ParquetTypedRecordReader.cpp @@ -332,6 +332,9 @@ std::shared_ptr MakeRecordReader(const ColumnDescriptor* descr case ::arrow::Type::INT64: { return std::make_shared>(descr, leaf_info, pool); } + case ::arrow::Type::TIMESTAMP: { + return std::make_shared>(descr, leaf_info, pool); + } case ::arrow::Type::DATE64: { return std::make_shared>(descr, leaf_info, pool); diff --git a/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp b/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp index f0a9b3e76ab9956581a13744b16851cd4f34d040..ac1b22742126be70475fe4109b987421154a6fa1 100644 --- a/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp +++ b/omnioperator/omniop-spark-extension/cpp/src/shuffle/splitter.cpp @@ -484,6 +484,10 @@ void Splitter::ToSplitterTypeId(int num_cols) CastOmniToShuffleType(OMNI_LONG, SHUFFLE_8BYTE); break; } + case OMNI_TIMESTAMP: { + CastOmniToShuffleType(OMNI_TIMESTAMP, SHUFFLE_8BYTE); + break; + } case OMNI_DOUBLE: { CastOmniToShuffleType(OMNI_DOUBLE, SHUFFLE_8BYTE); break; diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchScanReader.java b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchScanReader.java index 9a657ecd5e0e0e3093a6a11a20fef68a26e1bd0d..237ebdfae1e8dfaa72f1a7a7107f356531cc6a23 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchScanReader.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchScanReader.java @@ -271,6 +271,7 @@ public class OrcColumnarBatchScanReader { break; } case OMNI_LONG: + case OMNI_TIMESTAMP: case OMNI_DECIMAL64: { vecList[i] = new LongVec(vecNativeIds[nativeGetId]); break; diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/ParquetColumnarBatchScanReader.java b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/ParquetColumnarBatchScanReader.java index 85cd70cb9d9756b182ff44bba3ab8f22e4c51745..a28144d2d640988527f3c9754816b79608c24e05 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/ParquetColumnarBatchScanReader.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/ParquetColumnarBatchScanReader.java @@ -26,6 +26,7 @@ import org.apache.spark.sql.types.BooleanType; import org.apache.spark.sql.types.ByteType; import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.DateType; +import org.apache.spark.sql.types.TimestampType; import org.apache.spark.sql.types.DecimalType; import org.apache.spark.sql.types.DoubleType; import org.apache.spark.sql.types.IntegerType; @@ -78,6 +79,8 @@ public class ParquetColumnarBatchScanReader { DataType type = types.get(i); if (type instanceof LongType) { vecList[i] = new LongVec(vecNativeIds[i]); + } else if (type instanceof TimestampType) { + vecList[i] = new LongVec(vecNativeIds[i]); } else if (type instanceof BooleanType) { vecList[i] = new BooleanVec(vecNativeIds[i]); } else if (type instanceof ShortType) { diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/serialize/ShuffleDataSerializer.java b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/serialize/ShuffleDataSerializer.java index 9f6cadf70b7348a1696e140a13887c87018af4b2..c429380a0656444fdd4901e9467c3d93cfc5f2c4 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/serialize/ShuffleDataSerializer.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/serialize/ShuffleDataSerializer.java @@ -70,6 +70,10 @@ public class ShuffleDataSerializer { type = DataTypes.LongType; vec = new LongVec(vecSize); break; + case VEC_TYPE_TIMESTAMP: + type = DataTypes.TimestampType; + vec = new LongVec(vecSize); + break; case VEC_TYPE_DATE64: type = DataTypes.DateType; vec = new LongVec(vecSize); diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/org/apache/spark/sql/execution/vectorized/OmniColumnVector.java b/omnioperator/omniop-spark-extension/java/src/main/java/org/apache/spark/sql/execution/vectorized/OmniColumnVector.java index 4a84c2a7ccb57ec2737a3b95a9c35ecc97d764b1..3d07b2dec0515dacdc6739b17f2b2dc96d1d3bff 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/org/apache/spark/sql/execution/vectorized/OmniColumnVector.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/org/apache/spark/sql/execution/vectorized/OmniColumnVector.java @@ -95,7 +95,7 @@ public class OmniColumnVector extends WritableColumnVector { return dictionaryData; } - if (type instanceof LongType) { + if (type instanceof LongType || type instanceof TimestampType) { return longDataVec; } else if (type instanceof BooleanType || type instanceof NullType) { return booleanDataVec; @@ -130,7 +130,7 @@ public class OmniColumnVector extends WritableColumnVector { public void setVec(Vec vec) { if (vec instanceof DictionaryVec) { dictionaryData = (DictionaryVec) vec; - } else if (type instanceof LongType) { + } else if (type instanceof LongType || type instanceof TimestampType) { this.longDataVec = (LongVec) vec; } else if (type instanceof DecimalType) { if (DecimalType.is64BitDecimalType(type)) { @@ -217,7 +217,7 @@ public class OmniColumnVector extends WritableColumnVector { } else { return decimal128DataVec.hasNull(); } - } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type)) { + } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { return longDataVec.hasNull(); } else if (type instanceof FloatType) { return false; @@ -259,7 +259,7 @@ public class OmniColumnVector extends WritableColumnVector { } else { decimal128DataVec.setNull(rowId); } - } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type)) { + } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { longDataVec.setNull(rowId); } else if (type instanceof FloatType) { return; @@ -294,7 +294,7 @@ public class OmniColumnVector extends WritableColumnVector { } else { decimal128DataVec.setNulls(rowId, nullValue, 0, count); } - } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type)) { + } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { longDataVec.setNulls(rowId, nullValue, 0, count); } else if (type instanceof FloatType) { return; @@ -329,7 +329,7 @@ public class OmniColumnVector extends WritableColumnVector { } else { return decimal128DataVec.isNull(rowId); } - } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type)) { + } else if (type instanceof LongType || DecimalType.is64BitDecimalType(type) || type instanceof TimestampType) { return longDataVec.isNull(rowId); } else if (type instanceof FloatType) { return false; @@ -837,6 +837,8 @@ public class OmniColumnVector extends WritableColumnVector { } } else if (type instanceof LongType) { longDataVec = new LongVec(newCapacity); + } else if (type instanceof TimestampType) { + longDataVec = new LongVec(newCapacity); } else if (type instanceof FloatType) { throw new UnsupportedOperationException("reserveInternal is not supported for type:" + type); } else if (type instanceof DoubleType) { diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/Constant.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/Constant.scala index 9d7f844bcc19601ac065083b988085c340631ad3..f1d354e29f3980c43e36b7f8ff2ec043849081ef 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/Constant.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/Constant.scala @@ -29,6 +29,7 @@ object Constant { val OMNI_SHOR_TYPE: String = DataTypeId.OMNI_SHORT.ordinal().toString val OMNI_INTEGER_TYPE: String = DataTypeId.OMNI_INT.ordinal().toString val OMNI_LONG_TYPE: String = DataTypeId.OMNI_LONG.ordinal().toString + val OMNI_TIMESTAMP_TYPE: String = DataTypeId.OMNI_TIMESTAMP.ordinal().toString val OMNI_DOUBLE_TYPE: String = DataTypeId.OMNI_DOUBLE.ordinal().toString val OMNI_BOOLEAN_TYPE: String = DataTypeId.OMNI_BOOLEAN.ordinal().toString val OMNI_DATE_TYPE: String = DataTypeId.OMNI_DATE32.ordinal().toString diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index 11e1ae448754bdc2d398c93df6cda1b7b508a8f1..d0d39e2cbca5c78ec7441820d927fea59f7c5e51 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -20,8 +20,8 @@ package com.huawei.boostkit.spark.expression import scala.collection.mutable.ArrayBuffer -import com.huawei.boostkit.spark.Constant.{DEFAULT_STRING_TYPE_LENGTH, IS_CHECK_OMNI_EXP, OMNI_BOOLEAN_TYPE, OMNI_DATE_TYPE, OMNI_DECIMAL128_TYPE, OMNI_DECIMAL64_TYPE, OMNI_DOUBLE_TYPE, OMNI_INTEGER_TYPE, OMNI_LONG_TYPE, OMNI_SHOR_TYPE, OMNI_VARCHAR_TYPE} -import nova.hetu.omniruntime.`type`.{BooleanDataType, DataTypeSerializer, Date32DataType, Decimal128DataType, Decimal64DataType, DoubleDataType, IntDataType, LongDataType, ShortDataType, VarcharDataType} +import com.huawei.boostkit.spark.Constant.{DEFAULT_STRING_TYPE_LENGTH, IS_CHECK_OMNI_EXP, OMNI_BOOLEAN_TYPE, OMNI_DATE_TYPE, OMNI_DECIMAL128_TYPE, OMNI_DECIMAL64_TYPE, OMNI_DOUBLE_TYPE, OMNI_INTEGER_TYPE, OMNI_LONG_TYPE, OMNI_TIMESTAMP_TYPE, OMNI_SHOR_TYPE, OMNI_VARCHAR_TYPE} +import nova.hetu.omniruntime.`type`.{BooleanDataType, DataTypeSerializer, Date32DataType, Decimal128DataType, Decimal64DataType, DoubleDataType, IntDataType, LongDataType, TimestampDataType, ShortDataType, VarcharDataType} import nova.hetu.omniruntime.constants.FunctionType import nova.hetu.omniruntime.constants.FunctionType.{OMNI_AGGREGATION_TYPE_AVG, OMNI_AGGREGATION_TYPE_COUNT_ALL, OMNI_AGGREGATION_TYPE_COUNT_COLUMN, OMNI_AGGREGATION_TYPE_FIRST_INCLUDENULL, OMNI_AGGREGATION_TYPE_FIRST_IGNORENULL, OMNI_AGGREGATION_TYPE_MAX, OMNI_AGGREGATION_TYPE_MIN, OMNI_AGGREGATION_TYPE_SUM, OMNI_WINDOW_TYPE_RANK, OMNI_WINDOW_TYPE_ROW_NUMBER} import nova.hetu.omniruntime.constants.JoinType._ @@ -507,7 +507,7 @@ object OmniExpressionAdaptor extends Logging { case _ => "{\"exprType\":\"UNARY\",\"returnType\":%s,\"operator\":\"not\",\"expr\":%s}".format( sparkTypeToOmniExpJsonType(BooleanType), - rewriteToOmniJsonExpressionLiteral(not.child, exprsIndexMap)) + rewriteToOmniJsonExpressionLiteral(not.child, exprsIndexMap)) } case isnotnull: IsNotNull => ("{\"exprType\":\"UNARY\",\"returnType\":%s, \"operator\":\"not\"," @@ -614,7 +614,7 @@ object OmniExpressionAdaptor extends Logging { rewriteToOmniJsonExpressionLiteral(round.child, exprsIndexMap), rewriteToOmniJsonExpressionLiteral(round.scale, exprsIndexMap)) case attr: Attribute => toOmniJsonAttribute(attr, exprsIndexMap(attr.exprId)) - + case inStr: StringInstr => ("{\"exprType\":\"FUNCTION\",\"returnType\":%s,\"function_name\":\"instr\", \"arguments\":[%s,%s]}") .format(sparkTypeToOmniExpJsonType(inStr.dataType), @@ -792,7 +792,7 @@ object OmniExpressionAdaptor extends Logging { OMNI_AGGREGATION_TYPE_FIRST_IGNORENULL case First(_, false) => checkFirstParamType(agg) - OMNI_AGGREGATION_TYPE_FIRST_INCLUDENULL + OMNI_AGGREGATION_TYPE_FIRST_INCLUDENULL case _ => throw new UnsupportedOperationException(s"Unsupported aggregate function: $agg") } } @@ -826,6 +826,7 @@ object OmniExpressionAdaptor extends Logging { case ShortType => OMNI_SHOR_TYPE case IntegerType => OMNI_INTEGER_TYPE case LongType => OMNI_LONG_TYPE + case TimestampType => OMNI_TIMESTAMP_TYPE case DoubleType => OMNI_DOUBLE_TYPE case BooleanType => OMNI_BOOLEAN_TYPE case StringType => OMNI_VARCHAR_TYPE @@ -863,6 +864,8 @@ object OmniExpressionAdaptor extends Logging { IntDataType.INTEGER case LongType => LongDataType.LONG + case TimestampType => + TimestampDataType.TIMESTAMP case DoubleType => DoubleDataType.DOUBLE case BooleanType => @@ -887,7 +890,7 @@ object OmniExpressionAdaptor extends Logging { val metadata = attr.metadata val omniDataType: String = sparkTypeToOmniExpType(dataType) dataType match { - case ShortType | IntegerType | LongType | DoubleType | BooleanType | DateType => + case ShortType | IntegerType | LongType | DoubleType | TimestampType | BooleanType | DateType => "{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":%s,\"colVal\":%d}" .format(omniDataType, colVal) case StringType => diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/util/OmniAdaptorUtil.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/util/OmniAdaptorUtil.scala index ed99f6b4311a48492438095a87d450f7d9d89a5a..0fad75e135b14d1f74f8a6369d475c8335aa462b 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/util/OmniAdaptorUtil.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/util/OmniAdaptorUtil.scala @@ -65,7 +65,7 @@ object OmniAdaptorUtil { def transColumnVector(columnVector: ColumnVector, columnSize : Int): Vec = { val datatype: DataType = columnVector.dataType() val vec: Vec = datatype match { - case LongType => + case LongType | TimestampType => val vec = new LongVec(columnSize) val values = new Array[Long](columnSize) for (i <- 0 until columnSize) { diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/joins/ColumnarSortMergeJoinExec.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/joins/ColumnarSortMergeJoinExec.scala index 4a91517d5abf7122b7bcafcc4bfc3036a292e875..f66a1bbad09dae1211587ac23cf561a627b219bc 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/joins/ColumnarSortMergeJoinExec.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/joins/ColumnarSortMergeJoinExec.scala @@ -258,7 +258,7 @@ case class ColumnarSortMergeJoinExec( vecs(i) = types(i).getId match { case DataType.DataTypeId.OMNI_INT | DataType.DataTypeId.OMNI_DATE32 => new IntVec(0) - case DataType.DataTypeId.OMNI_LONG | DataType.DataTypeId.OMNI_DECIMAL64 => + case DataType.DataTypeId.OMNI_LONG | DataType.DataTypeId.OMNI_TIMESTAMP | DataType.DataTypeId.OMNI_DECIMAL64 => new LongVec(0) case DataType.DataTypeId.OMNI_DOUBLE => new DoubleVec(0) diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/util/MergeIterator.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/util/MergeIterator.scala index 6236aefee7c8836c8a244e34e004da8f641f6e84..d28893e332eac8c070c6f1d66276a476341f0817 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/util/MergeIterator.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/util/MergeIterator.scala @@ -27,7 +27,7 @@ import nova.hetu.omniruntime.`type`.{DataType, VarcharDataType} import nova.hetu.omniruntime.vector.{BooleanVec, Decimal128Vec, DoubleVec, IntVec, LongVec, ShortVec, VarcharVec, Vec, VecBatch} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.vectorized.OmniColumnVector -import org.apache.spark.sql.types.{BooleanType, DateType, DecimalType, DoubleType, IntegerType, LongType, ShortType, StringType, StructType} +import org.apache.spark.sql.types.{BooleanType, DateType, DecimalType, DoubleType, IntegerType, LongType, TimestampType, ShortType, StringType, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch class MergeIterator(iter: Iterator[ColumnarBatch], localSchema: StructType, @@ -46,7 +46,7 @@ class MergeIterator(iter: Iterator[ColumnarBatch], localSchema: StructType, val vecs = new Array[Vec](schema.fields.length) schema.fields.zipWithIndex.foreach { case (field, index) => field.dataType match { - case LongType => + case LongType | TimestampType => vecs(index) = new LongVec(columnSize) case DateType | IntegerType => vecs(index) = new IntVec(columnSize) diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleCompressionTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleCompressionTest.java index d95be18832b926500b599821b6b6fd0baa8861c5..b814ff22ccb6500ec46be3bec31f2e6b130214a6 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleCompressionTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleCompressionTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffPartitionTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffPartitionTest.java index c8fd474137a93ea8831d3dc3ab432e409018cc55..982d90aa0d2f8cf7fbb102080ee7396900f88aa7 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffPartitionTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffPartitionTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffRowVBTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffRowVBTest.java index dc53fda8a1a04a15bf7ffb9919926d4812208fc0..7b15688649e81fc3705fdb1f4ebd97567ed3a926 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffRowVBTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleDiffRowVBTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleGBSizeTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleGBSizeTest.java index 2ef81ac49e545aa617136b9d4f3e7e769ea34652..90c74633a2d947640fb1a027f40e0b92b067e7a6 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleGBSizeTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleGBSizeTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleNullTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleNullTest.java index 98fc18dd8f3237928cc066887e6fcb2205686692..0b2bf2e31b5a430e1faf911288fe4321ab53c905 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleNullTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleNullTest.java @@ -30,6 +30,7 @@ import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DECIMAL64; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_DOUBLE; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_INT; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_LONG; +import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_TIMESTAMP; import static nova.hetu.omniruntime.type.DataType.DataTypeId.OMNI_VARCHAR; import nova.hetu.omniruntime.type.DataTypeSerializer; import nova.hetu.omniruntime.vector.VecBatch; diff --git a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleTest.java b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleTest.java index 8be5702dfbabc5bc847e4ebe547d1d4dfa243e6f..d5b4e474dd16f018fcb3522c1ee735ca07b5c0d0 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleTest.java +++ b/omnioperator/omniop-spark-extension/java/src/test/java/com/huawei/boostkit/spark/ColumnShuffleTest.java @@ -28,6 +28,7 @@ import nova.hetu.omniruntime.type.Decimal64DataType; import nova.hetu.omniruntime.type.DoubleDataType; import nova.hetu.omniruntime.type.IntDataType; import nova.hetu.omniruntime.type.LongDataType; +import nova.hetu.omniruntime.type.TimestampDataType; import nova.hetu.omniruntime.type.VarcharDataType; import nova.hetu.omniruntime.vector.Decimal128Vec; import nova.hetu.omniruntime.vector.DoubleVec; @@ -76,6 +77,10 @@ abstract class ColumnShuffleTest { types[i] = Date64DataType.DATE64; break; } + case OMNI_TIMESTAMP: { + types[i] = TimestampDataType.TIMESTAMP; + break; + } case OMNI_DECIMAL64: { types[i] = Decimal64DataType.DECIMAL64; break; @@ -118,6 +123,7 @@ abstract class ColumnShuffleTest { break; } case OMNI_LONG: + case OMNI_TIMESTAMP: case OMNI_DECIMAL64: case OMNI_DATE64: { tmpVec = new LongVec(rowNum);