diff --git a/omnidata/omnidata-openlookeng-connector/connector/pom.xml b/omnidata/omnidata-openlookeng-connector/connector/pom.xml index 7f9e6c4bc1dd92230a18868c9681ce3463756202..1f4c602db2c80eb7a46ead54164dd61566f88bfc 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/pom.xml +++ b/omnidata/omnidata-openlookeng-connector/connector/pom.xml @@ -760,6 +760,23 @@ + + com.mycila + license-maven-plugin + 2.3 + + + ${air.main.basedir}/src/main/resource/license/license-header.txt + ${air.main.basedir}/src/main/resource/license/license-header-alternate-2010.txt + ${air.main.basedir}/src/main/resource/license/license-header-alternate-2012.txt + ${air.main.basedir}/src/main/resource/license/license-header-alternate-2020.txt + ${air.main.basedir}/src/main/resource/license/license-header-alternate-2021.txt + ${air.main.basedir}/src/main/resource/license/license-header-alternate-2022.txt + ${air.main.basedir}/src/main/resource/license/license-header-alternate-2022-2022.txt + ${air.main.basedir}/src/main/resource/license/license-header-third.txt + + + diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java index e72e2cad802d24afd6f6ac1e7297610267f57403..9af74c7504ff28f6cb9e89badbc7103e1a0dad3e 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/HivePageSourceProvider.java @@ -16,7 +16,6 @@ package io.prestosql.plugin.hive; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.huawei.boostkit.omnidata.decode.impl.OpenLooKengDeserializer; import com.huawei.boostkit.omnidata.model.Predicate; import com.huawei.boostkit.omnidata.model.TaskSource; import com.huawei.boostkit.omnidata.model.datasource.DataSource; @@ -31,6 +30,7 @@ import io.prestosql.plugin.hive.HiveBucketing.BucketingVersion; import io.prestosql.plugin.hive.coercions.HiveCoercer; import io.prestosql.plugin.hive.omnidata.OmniDataNodeManager; import io.prestosql.plugin.hive.omnidata.OmniDataNodeStatus; +import io.prestosql.plugin.hive.omnidata.decode.impl.OpenLooKengDeserializer; import io.prestosql.plugin.hive.orc.OrcConcatPageSource; import io.prestosql.plugin.hive.util.IndexCache; import io.prestosql.spi.Page; diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/AbstractDecoding.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/AbstractDecoding.java new file mode 100644 index 0000000000000000000000000000000000000000..a1b7d5a1e223f0bf4c3f8b438c6fc2a2f7be8455 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/AbstractDecoding.java @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode; + +import com.huawei.boostkit.omnidata.exception.OmniDataException; +import io.airlift.slice.SliceInput; +import io.prestosql.plugin.hive.omnidata.decode.type.DecimalDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.DecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.LongToByteDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.LongToFloatDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.LongToIntDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.LongToShortDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.TimestampDecodeType; +import io.prestosql.spi.type.DateType; +import io.prestosql.spi.type.RowType; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.stream.IntStream; + +/** + * Abstract decoding + * + * @param decoding type + * @since 2022-07-18 + */ +public abstract class AbstractDecoding + implements Decoding +{ + private static final Map DECODE_METHODS; + + static { + DECODE_METHODS = new HashMap<>(); + Method[] methods = Decoding.class.getDeclaredMethods(); + for (Method method : methods) { + if (method.isAnnotationPresent(Decode.class)) { + DECODE_METHODS.put(method.getAnnotation(Decode.class).value(), method); + } + } + } + + private Method getDecodeMethod(String decodeName) + { + return DECODE_METHODS.get(decodeName); + } + + private String getDecodeName(SliceInput input) + { + int length = input.readInt(); + byte[] bytes = new byte[length]; + input.readBytes(bytes); + + return new String(bytes, StandardCharsets.UTF_8); + } + + private Optional typeToDecodeName(DecodeType type) + { + Class javaType = null; + if (type.getJavaType().isPresent()) { + javaType = type.getJavaType().get(); + } + if (javaType == double.class) { + return Optional.of("DOUBLE_ARRAY"); + } + else if (javaType == float.class) { + return Optional.of("FLOAT_ARRAY"); + } + else if (javaType == int.class) { + return Optional.of("INT_ARRAY"); + } + else if (javaType == long.class) { + return Optional.of("LONG_ARRAY"); + } + else if (javaType == byte.class) { + return Optional.of("BYTE_ARRAY"); + } + else if (javaType == boolean.class) { + return Optional.of("BOOLEAN_ARRAY"); + } + else if (javaType == short.class) { + return Optional.of("SHORT_ARRAY"); + } + else if (javaType == String.class) { + return Optional.of("VARIABLE_WIDTH"); + } + else if (javaType == RowType.class) { + return Optional.of("ROW"); + } + else if (javaType == DateType.class) { + return Optional.of("DATE"); + } + else if (javaType == LongToIntDecodeType.class) { + return Optional.of("LONG_TO_INT"); + } + else if (javaType == LongToShortDecodeType.class) { + return Optional.of("LONG_TO_SHORT"); + } + else if (javaType == LongToByteDecodeType.class) { + return Optional.of("LONG_TO_BYTE"); + } + else if (javaType == LongToFloatDecodeType.class) { + return Optional.of("LONG_TO_FLOAT"); + } + else if (javaType == DecimalDecodeType.class) { + return Optional.of("DECIMAL"); + } + else if (javaType == TimestampDecodeType.class) { + return Optional.of("TIMESTAMP"); + } + else { + return Optional.empty(); + } + } + + private boolean[] getIsNullValue(byte value) + { + boolean[] isNullValue = new boolean[8]; + isNullValue[0] = ((value & 0b1000_0000) != 0); + isNullValue[1] = ((value & 0b0100_0000) != 0); + isNullValue[2] = ((value & 0b0010_0000) != 0); + isNullValue[3] = ((value & 0b0001_0000) != 0); + isNullValue[4] = ((value & 0b0000_1000) != 0); + isNullValue[5] = ((value & 0b0000_0100) != 0); + isNullValue[6] = ((value & 0b0000_0010) != 0); + isNullValue[7] = ((value & 0b0000_0001) != 0); + + return isNullValue; + } + + @Override + public T decode(Optional type, SliceInput sliceInput) + { + try { + String decodeName = getDecodeName(sliceInput); + if (type.isPresent()) { + Optional decodeNameOpt = typeToDecodeName(type.get()); + if ("DECIMAL".equals(decodeNameOpt.orElse(decodeName)) && !"RLE".equals(decodeName)) { + Method method = getDecodeMethod("DECIMAL"); + return (T) method.invoke(this, type, sliceInput, decodeName); + } + if (!"RLE".equals(decodeName)) { + decodeName = decodeNameOpt.orElse(decodeName); + } + } + Method method = getDecodeMethod(decodeName); + return (T) method.invoke(this, type, sliceInput); + } + catch (IllegalAccessException | InvocationTargetException e) { + throw new OmniDataException("decode failed " + e.getMessage()); + } + } + + /** + * decode Null Bits + * + * @param sliceInput sliceInput + * @param positionCount positionCount + * @return decode boolean[] + * @since 2022-07-18 + */ + public Optional decodeNullBits(SliceInput sliceInput, int positionCount) + { + if (!sliceInput.readBoolean()) { + return Optional.empty(); + } + + // read null bits 8 at a time + boolean[] valueIsNull = new boolean[positionCount]; + for (int position = 0; position < (positionCount & ~0b111); position += 8) { + boolean[] nextEightValue = getIsNullValue(sliceInput.readByte()); + int finalPosition = position; + IntStream.range(0, 8).forEach(pos -> valueIsNull[finalPosition + pos] = nextEightValue[pos]); + } + + // read last null bits + if ((positionCount & 0b111) > 0) { + byte value = sliceInput.readByte(); + int maskInt = 0b1000_0000; + for (int pos = positionCount & ~0b111; pos < positionCount; pos++) { + valueIsNull[pos] = ((value & maskInt) != 0); + maskInt >>>= 1; + } + } + + return Optional.of(valueIsNull); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/Decode.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/Decode.java new file mode 100644 index 0000000000000000000000000000000000000000..efea3a4904d5ff4df9570fb6c495e8751fc1735d --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/Decode.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@BindingAnnotation +public @interface Decode +{ + String value(); +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/Decoding.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/Decoding.java new file mode 100644 index 0000000000000000000000000000000000000000..3f14fc7f467885f095bb3c59b272aefed38b6325 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/Decoding.java @@ -0,0 +1,264 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode; + +import io.airlift.slice.SliceInput; +import io.prestosql.plugin.hive.omnidata.decode.type.DecodeType; + +import java.lang.reflect.InvocationTargetException; +import java.util.Optional; + +/** + * Decode Slice to type + * + * @param + * @since 2022-07-18 + */ +public interface Decoding +{ + /** + * decode + * + * @param type decode type + * @param sliceInput content + * @return T + */ + T decode(Optional type, SliceInput sliceInput); + + /** + * decode array type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("ARRAY") + T decodeArray(Optional type, SliceInput sliceInput); + + /** + * decode byte array type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("BYTE_ARRAY") + T decodeByteArray(Optional type, SliceInput sliceInput); + + /** + * decode boolean array type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("BOOLEAN_ARRAY") + T decodeBooleanArray(Optional type, SliceInput sliceInput); + + /** + * decode int array type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("INT_ARRAY") + T decodeIntArray(Optional type, SliceInput sliceInput); + + /** + * decode int128 array type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("INT128_ARRAY") + T decodeInt128Array(Optional type, SliceInput sliceInput); + + /** + * decode short array type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("SHORT_ARRAY") + T decodeShortArray(Optional type, SliceInput sliceInput); + + /** + * decode long array type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("LONG_ARRAY") + T decodeLongArray(Optional type, SliceInput sliceInput); + + /** + * decode float array type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("FLOAT_ARRAY") + T decodeFloatArray(Optional type, SliceInput sliceInput); + + /** + * decode double array type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("DOUBLE_ARRAY") + T decodeDoubleArray(Optional type, SliceInput sliceInput); + + /** + * decode map type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("MAP") + T decodeMap(Optional type, SliceInput sliceInput); + + /** + * decode map element type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("MAP_ELEMENT") + T decodeSingleMap(Optional type, SliceInput sliceInput); + + /** + * decode variable width type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("VARIABLE_WIDTH") + T decodeVariableWidth(Optional type, SliceInput sliceInput); + + /** + * decode dictionary type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("DICTIONARY") + T decodeDictionary(Optional type, SliceInput sliceInput); + + /** + * decode rle type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + * @throws InvocationTargetException throw invocation target exception + * @throws IllegalAccessException throw illegal access exception + */ + @Decode("RLE") + T decodeRunLength(Optional type, SliceInput sliceInput) + throws InvocationTargetException, IllegalAccessException; + + /** + * decode row type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("ROW") + T decodeRow(Optional type, SliceInput sliceInput); + + /** + * decode date type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("DATE") + T decodeDate(Optional type, SliceInput sliceInput); + + /** + * decode long to int type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("LONG_TO_INT") + T decodeLongToInt(Optional type, SliceInput sliceInput); + + /** + * decode long to short type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("LONG_TO_SHORT") + T decodeLongToShort(Optional type, SliceInput sliceInput); + + /** + * decode long to byte type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("LONG_TO_BYTE") + T decodeLongToByte(Optional type, SliceInput sliceInput); + + /** + * decode long to float type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("LONG_TO_FLOAT") + T decodeLongToFloat(Optional type, SliceInput sliceInput); + + /** + * decode decimal type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @param decodeType storage type of decimal + * @return T + */ + @Decode("DECIMAL") + T decodeDecimal(Optional type, SliceInput sliceInput, String decodeType); + + /** + * decode timestamp type + * + * @param type type of data to decode + * @param sliceInput data to decode + * @return T + */ + @Decode("TIMESTAMP") + T decodeTimestamp(Optional type, SliceInput sliceInput); +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/impl/OpenLooKengDecoding.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/impl/OpenLooKengDecoding.java new file mode 100644 index 0000000000000000000000000000000000000000..66cf8fc9afc6eee5aaafc2eab1b5bba442c0212e --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/impl/OpenLooKengDecoding.java @@ -0,0 +1,292 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.impl; + +import io.airlift.slice.Slice; +import io.airlift.slice.SliceInput; +import io.airlift.slice.Slices; +import io.prestosql.plugin.hive.omnidata.decode.AbstractDecoding; +import io.prestosql.plugin.hive.omnidata.decode.type.ArrayDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.DecodeType; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.ByteArrayBlock; +import io.prestosql.spi.block.DictionaryBlock; +import io.prestosql.spi.block.DictionaryId; +import io.prestosql.spi.block.Int128ArrayBlock; +import io.prestosql.spi.block.IntArrayBlock; +import io.prestosql.spi.block.LongArrayBlock; +import io.prestosql.spi.block.RunLengthEncodedBlock; +import io.prestosql.spi.block.ShortArrayBlock; +import io.prestosql.spi.block.VariableWidthBlock; + +import java.util.Optional; +import java.util.stream.IntStream; + +import static io.airlift.slice.SizeOf.SIZE_OF_INT; +import static io.prestosql.spi.block.ArrayBlock.fromElementBlock; +import static io.prestosql.spi.block.RowBlock.fromFieldBlocks; + +/** + * Decode data to block + * + * @since 2022-07-18 + */ +public class OpenLooKengDecoding + extends AbstractDecoding> +{ + @Override + public Block decodeArray(Optional type, SliceInput sliceInput) + { + Optional elementType = Optional.empty(); + if (type.isPresent()) { + if (type.get() instanceof ArrayDecodeType) { + ArrayDecodeType arrayDecodeType = (ArrayDecodeType) type.get(); + elementType = Optional.of((arrayDecodeType).getElementType()); + } + } + Block values = decode(elementType, sliceInput); + int positionCount = sliceInput.readInt(); + int[] offsets = new int[positionCount + 1]; + sliceInput.readBytes(Slices.wrappedIntArray(offsets)); + boolean[] valueIsNull = decodeNullBits(sliceInput, positionCount).orElseGet(() -> new boolean[positionCount]); + + return fromElementBlock(positionCount, Optional.ofNullable(valueIsNull), offsets, values); + } + + @Override + public Block decodeByteArray(Optional type, SliceInput sliceInput) + { + int positionCount = sliceInput.readInt(); + boolean[] valueIsNull = decodeNullBits(sliceInput, positionCount).orElse(null); + byte[] values = new byte[positionCount]; + + IntStream.range(0, positionCount) + .forEach( + position -> { + if (valueIsNull == null || !valueIsNull[position]) { + values[position] = sliceInput.readByte(); + } + }); + + return new ByteArrayBlock(positionCount, Optional.ofNullable(valueIsNull), values); + } + + @Override + public Block decodeBooleanArray(Optional type, SliceInput sliceInput) + { + return decodeByteArray(type, sliceInput); + } + + @Override + public Block decodeIntArray(Optional type, SliceInput sliceInput) + { + int posCount = sliceInput.readInt(); + boolean[] valueIsNull = decodeNullBits(sliceInput, posCount).orElse(null); + int[] values = new int[posCount]; + + for (int position = 0; position < posCount; position++) { + if (valueIsNull == null || !valueIsNull[position]) { + values[position] = sliceInput.readInt(); + } + } + + return new IntArrayBlock(posCount, Optional.ofNullable(valueIsNull), values); + } + + @Override + public Block decodeInt128Array(Optional type, SliceInput sliceInput) + { + int posCount = sliceInput.readInt(); + boolean[] valueIsNull = decodeNullBits(sliceInput, posCount).orElse(null); + long[] values = new long[posCount * 2]; + + for (int position = 0; position < posCount; position++) { + if (valueIsNull == null || !valueIsNull[position]) { + values[position * 2] = sliceInput.readLong(); + values[(position * 2) + 1] = sliceInput.readLong(); + } + } + + return new Int128ArrayBlock(posCount, Optional.ofNullable(valueIsNull), values); + } + + @Override + public Block decodeShortArray(Optional type, SliceInput sliceInput) + { + int posCount = sliceInput.readInt(); + boolean[] valueIsNull = decodeNullBits(sliceInput, posCount).orElse(null); + short[] values = new short[posCount]; + + for (int position = 0; position < posCount; position++) { + if (valueIsNull == null || !valueIsNull[position]) { + values[position] = sliceInput.readShort(); + } + } + + return new ShortArrayBlock(posCount, Optional.ofNullable(valueIsNull), values); + } + + @Override + public Block decodeLongArray(Optional type, SliceInput sliceInput) + { + int posCount = sliceInput.readInt(); + boolean[] valueIsNull = decodeNullBits(sliceInput, posCount).orElse(null); + long[] values = new long[posCount]; + + for (int position = 0; position < posCount; position++) { + if (valueIsNull == null || !valueIsNull[position]) { + values[position] = sliceInput.readLong(); + } + } + + return new LongArrayBlock(posCount, Optional.ofNullable(valueIsNull), values); + } + + @Override + public Block decodeFloatArray(Optional type, SliceInput sliceInput) + { + return decodeLongArray(type, sliceInput); + } + + @Override + public Block decodeDoubleArray(Optional type, SliceInput sliceInput) + { + return decodeLongArray(type, sliceInput); + } + + @Override + public Block decodeMap(Optional type, SliceInput sliceInput) + { + throw new UnsupportedOperationException(); + } + + @Override + public Block decodeSingleMap(Optional type, SliceInput sliceInput) + { + throw new UnsupportedOperationException(); + } + + @Override + public Block decodeVariableWidth(Optional type, SliceInput sliceInput) + { + int posCount = sliceInput.readInt(); + int[] offsets = new int[posCount + 1]; + + sliceInput.readBytes(Slices.wrappedIntArray(offsets), SIZE_OF_INT, posCount * SIZE_OF_INT); + + boolean[] valueIsNull = decodeNullBits(sliceInput, posCount).orElse(null); + + int blockSize = sliceInput.readInt(); + Slice slice = sliceInput.readSlice(blockSize); + + return new VariableWidthBlock(posCount, slice, offsets, Optional.ofNullable(valueIsNull)); + } + + @Override + public Block decodeDictionary(Optional type, SliceInput sliceInput) + { + int posCount = sliceInput.readInt(); + + Block dictionaryBlock = decode(type, sliceInput); + + int[] ids = new int[posCount]; + sliceInput.readBytes(Slices.wrappedIntArray(ids)); + + long mostSignificantBits = sliceInput.readLong(); + long leastSignificantBits = sliceInput.readLong(); + long sequenceId = sliceInput.readLong(); + + // We always compact the dictionary before we send it. However, dictionaryBlock comes from sliceInput, which may + // over-retain memory. + // As a result, setting dictionaryIsCompacted to true is not appropriate here. + // over-retains memory. + return new DictionaryBlock<>( + posCount, + dictionaryBlock, + ids, + false, + new DictionaryId(mostSignificantBits, leastSignificantBits, sequenceId)); + } + + @Override + public Block decodeRunLength(Optional type, SliceInput sliceInput) + { + int posCount = sliceInput.readInt(); + + Block values = decode(type, sliceInput); + + return new RunLengthEncodedBlock<>(values, posCount); + } + + @Override + public Block decodeRow(Optional type, SliceInput sliceInput) + { + int numFields = sliceInput.readInt(); + Block[] fieldBlocks = new Block[numFields]; + for (int i = 0; i < numFields; i++) { + fieldBlocks[i] = decode(type, sliceInput); + } + + int positionCount = sliceInput.readInt(); + int[] fieldBlockOffsets = new int[positionCount + 1]; + sliceInput.readBytes(Slices.wrappedIntArray(fieldBlockOffsets)); + boolean[] rowIsNull = decodeNullBits(sliceInput, positionCount).orElseGet(() -> new boolean[positionCount]); + + return fromFieldBlocks(positionCount, Optional.of(rowIsNull), fieldBlocks); + } + + @Override + public Block decodeDate(Optional type, SliceInput sliceInput) + { + throw new UnsupportedOperationException(); + } + + @Override + public Block decodeLongToInt(Optional type, SliceInput sliceInput) + { + throw new UnsupportedOperationException(); + } + + @Override + public Block decodeLongToShort(Optional type, SliceInput sliceInput) + { + throw new UnsupportedOperationException(); + } + + @Override + public Block decodeLongToByte(Optional type, SliceInput sliceInput) + { + throw new UnsupportedOperationException(); + } + + @Override + public Block decodeLongToFloat(Optional type, SliceInput sliceInput) + { + throw new UnsupportedOperationException(); + } + + @Override + public Block decodeDecimal(Optional type, SliceInput sliceInput, String decodeType) + { + throw new UnsupportedOperationException(); + } + + @Override + public Block decodeTimestamp(Optional type, SliceInput sliceInput) + { + throw new UnsupportedOperationException(); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/impl/OpenLooKengDeserializer.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/impl/OpenLooKengDeserializer.java new file mode 100644 index 0000000000000000000000000000000000000000..4cbd9963d1b4b4a59e773d2415b558cfd1250b35 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/impl/OpenLooKengDeserializer.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.impl; + +import com.huawei.boostkit.omnidata.decode.Deserializer; +import io.airlift.compress.Decompressor; +import io.airlift.compress.zstd.ZstdDecompressor; +import io.airlift.slice.Slice; +import io.airlift.slice.SliceInput; +import io.airlift.slice.Slices; +import io.hetu.core.transport.execution.buffer.SerializedPage; +import io.prestosql.spi.Page; +import io.prestosql.spi.block.Block; + +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; + +/** + * Deserialize block + * + * @since 2022-07-18 + */ +public class OpenLooKengDeserializer + implements Deserializer +{ + private final Decompressor decompressor; + private final OpenLooKengDecoding decoding; + + /** + * Constructor of deserialize block + */ + public OpenLooKengDeserializer() + { + decoding = new OpenLooKengDecoding(); + decompressor = new ZstdDecompressor(); + } + + /** + * Decompress serialized page + * + * @param page page need decompress + * @param decompressor decompressor + * @return Slice decompressed + */ + public static Slice decompressPage(SerializedPage page, Decompressor decompressor) + { + if (!page.isCompressed()) { + return page.getSlice(); + } + Slice slice = page.getSlice(); + int uncompressedSize = page.getUncompressedSizeInBytes(); + byte[] decompressed = new byte[uncompressedSize]; + if (slice.getBase() instanceof byte[]) { + byte[] sliceBase = (byte[]) slice.getBase(); + checkState( + decompressor.decompress( + sliceBase, + (int) (slice.getAddress() - ARRAY_BYTE_BASE_OFFSET), + slice.length(), + decompressed, + 0, + uncompressedSize) == uncompressedSize); + } + + return Slices.wrappedBuffer(decompressed); + } + + @Override + public Page deserialize(SerializedPage page) + { + checkArgument(page != null, "page is null"); + + if (page.isEncrypted()) { + throw new UnsupportedOperationException("unsupported encrypted page."); + } + + Slice slice = decompressPage(page, decompressor); + SliceInput input = slice.getInput(); + int numberOfBlocks = input.readInt(); + Block[] blocks = new Block[numberOfBlocks]; + for (int i = 0; i < blocks.length; i++) { + blocks[i] = decoding.decode(Optional.empty(), input); + } + + return new Page(page.getPositionCount(), page.getPageMetadata(), blocks); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/ArrayDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/ArrayDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..be2650369e1e0af5600e5d70249e933769b748a2 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/ArrayDecodeType.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Array decode type + * + * @param decode type + * @since 2022-07-18 + */ +public class ArrayDecodeType + implements DecodeType +{ + private final T elementType; + + public ArrayDecodeType(T elementType) + { + this.elementType = elementType; + } + + public T getElementType() + { + return elementType; + } + + @Override + public Optional> getJavaType() + { + return Optional.empty(); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/BooleanDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/BooleanDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..e1d5fe64a4f615c5a488d24790390dd8d1976db9 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/BooleanDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Boolean decode type + * + * @since 2022-07-18 + */ +public class BooleanDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(boolean.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/ByteDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/ByteDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..33fe178b76ac641eaac2a7e83be9374493ccb900 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/ByteDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Byte decode type + * + * @since 2022-07-18 + */ +public class ByteDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(byte.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DateDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DateDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..d173d70f91a6ac326ae464b9080bbca5f4172ddd --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DateDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Date Decode Type + * + * @since 2022-07-18 + */ +public class DateDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(io.prestosql.spi.type.DateType.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DecimalDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DecimalDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..779e93e384b3e861bf41f1b761a3ff699e2689a6 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DecimalDecodeType.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Decimal decode type + * + * @since 2022-07-18 + */ +public class DecimalDecodeType + implements DecodeType +{ + private final int precision; + private final int scale; + + public DecimalDecodeType(int precision, int scale) + { + this.precision = precision; + this.scale = scale; + } + + public int getPrecision() + { + return precision; + } + + public int getScale() + { + return scale; + } + + @Override + public Optional> getJavaType() + { + return Optional.of(DecimalDecodeType.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/decode/impl/OpenLooKengDeserializer.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DecodeType.java similarity index 62% rename from omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/decode/impl/OpenLooKengDeserializer.java rename to omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DecodeType.java index 275327c376cb1245fba06e0d3883cbe843a42042..47dc317e408ba1d290fa7c62927f5a05b3ecf6d1 100644 --- a/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/decode/impl/OpenLooKengDeserializer.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DecodeType.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -12,9 +12,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.huawei.boostkit.omnidata.decode.impl; -import com.huawei.boostkit.omnidata.decode.Deserializer; +package io.prestosql.plugin.hive.omnidata.decode.type; -public class OpenLooKengDeserializer implements Deserializer { +import java.util.Optional; + +/** + * Dcode java type + * + * @since 2022-07-18 + */ +public interface DecodeType +{ + /** + * get java class type + * + * @return class type + */ + Optional> getJavaType(); } diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DoubleDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DoubleDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..b1a4b789ec2b598419b125b2a1577540e8b949f2 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/DoubleDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Double decode type + * + * @since 2022-07-18 + */ +public class DoubleDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(double.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/FloatDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/FloatDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..fdd9f725cdd83bca7b7ea366f67863907d44962a --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/FloatDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Float decode type + * + * @since 2022-07-18 + */ +public class FloatDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(float.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/IntDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/IntDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..129760ad46ed62fba774575aa76a3de165aa3f10 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/IntDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Int decode type + * + * @since 2022-07-18 + */ +public class IntDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(int.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..af78cdcff0dcff5c9670068e68c48b0699a49201 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Long decode type + * + * @since 2022-07-18 + */ +public class LongDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(long.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToByteDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToByteDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..7c87a0578fd8ed0c87c33c58a219f821db665f17 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToByteDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Long To Byte decode + * + * @since 2022-07-18 + */ +public class LongToByteDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(LongToByteDecodeType.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToFloatDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToFloatDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..f073f157e2f74d9c97e120894d68fa90ba309868 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToFloatDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Double To Float decode + * + * @since 2022-07-18 + */ +public class LongToFloatDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(LongToFloatDecodeType.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToIntDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToIntDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..a78dab75241099548f923ca84fda3ef47c5b646c --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToIntDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Long To Int decode + * + * @since 2022-07-18 + */ +public class LongToIntDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(LongToIntDecodeType.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToShortDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToShortDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..9c0491e1dfcaed735e69f0e6e0bfb02cf8cd1bde --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/LongToShortDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * 功能描述 + * + * @since 2022-07-18 + */ +public class LongToShortDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(LongToShortDecodeType.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/MapDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/MapDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..887689ebe2e96b401d204a658077c6175264464b --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/MapDecodeType.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * MapDecode type + * + * @param k + * @param v + * @since 2022-07-18 + */ +public class MapDecodeType + implements DecodeType +{ + private final K keyType; + private final V valueType; + + public MapDecodeType(K keyType, V valueType) + { + this.keyType = keyType; + this.valueType = valueType; + } + + @Override + public Optional> getJavaType() + { + return Optional.empty(); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/RowDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/RowDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..d91008e406357857fdcb8c45cee72342bc61644e --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/RowDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Row decode type + * + * @since 2022-07-18 + */ +public class RowDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(io.prestosql.spi.type.RowType.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/ShortDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/ShortDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..6256173124bf52e15a8d3de77ec79d153346372d --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/ShortDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Short decode type + * + * @since 2022-07-18 + */ +public class ShortDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(short.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/TimestampDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/TimestampDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..375e75769b82728cd58af32367126ae8d7ec4812 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/TimestampDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Timestamp decode type + * + * @since 2022-07-18 + */ +public class TimestampDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(TimestampDecodeType.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/VarcharDecodeType.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/VarcharDecodeType.java new file mode 100644 index 0000000000000000000000000000000000000000..42f2f2f55676c746fa3d410034ffc62e405bac65 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/omnidata/decode/type/VarcharDecodeType.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import java.util.Optional; + +/** + * Varchar decode type + * + * @since 2022-07-18 + */ +public class VarcharDecodeType + implements DecodeType +{ + @Override + public Optional> getJavaType() + { + return Optional.of(String.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java index abfc416e7ea564866b9e645580dfa7dd07574675..b106d365b1aba21b6872af5985fe1290230ef0c4 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/orc/OrcPageSourceFactory.java @@ -17,7 +17,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.util.concurrent.UncheckedExecutionException; -import com.huawei.boostkit.omnidata.decode.impl.OpenLooKengDeserializer; import com.huawei.boostkit.omnidata.model.Predicate; import com.huawei.boostkit.omnidata.model.TaskSource; import com.huawei.boostkit.omnidata.model.datasource.DataSource; @@ -51,6 +50,7 @@ import io.prestosql.plugin.hive.HivePushDownPageSource; import io.prestosql.plugin.hive.HiveSessionProperties; import io.prestosql.plugin.hive.HiveType; import io.prestosql.plugin.hive.HiveUtil; +import io.prestosql.plugin.hive.omnidata.decode.impl.OpenLooKengDeserializer; import io.prestosql.plugin.hive.orc.OrcPageSource.ColumnAdaptation; import io.prestosql.spi.Page; import io.prestosql.spi.PrestoException; diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/parquet/ParquetPageSourceFactory.java b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/parquet/ParquetPageSourceFactory.java index 639b0ca18ba79e9688dee512ed4fc86f66b36827..dbccc94163cd357cdc2dc4f6c71e1b90d68b64d2 100644 --- a/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/parquet/ParquetPageSourceFactory.java +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/java/io/prestosql/plugin/hive/parquet/ParquetPageSourceFactory.java @@ -16,7 +16,6 @@ package io.prestosql.plugin.hive.parquet; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.huawei.boostkit.omnidata.decode.impl.OpenLooKengDeserializer; import com.huawei.boostkit.omnidata.model.TaskSource; import com.huawei.boostkit.omnidata.model.datasource.DataSource; import com.huawei.boostkit.omnidata.reader.DataReader; @@ -39,6 +38,7 @@ import io.prestosql.plugin.hive.HivePageSourceFactory; import io.prestosql.plugin.hive.HivePartitionKey; import io.prestosql.plugin.hive.HivePushDownPageSource; import io.prestosql.plugin.hive.HiveSessionProperties; +import io.prestosql.plugin.hive.omnidata.decode.impl.OpenLooKengDeserializer; import io.prestosql.spi.Page; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ConnectorPageSource; diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/main/resource/license/license-header-alternate-2022-2022.txt b/omnidata/omnidata-openlookeng-connector/connector/src/main/resource/license/license-header-alternate-2022-2022.txt new file mode 100644 index 0000000000000000000000000000000000000000..586419df16f9e4ccb846e33f55342e26286a2a47 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/main/resource/license/license-header-alternate-2022-2022.txt @@ -0,0 +1,12 @@ +Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/omnidata/decode/DeserializerTestUtils.java b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/omnidata/decode/DeserializerTestUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..3f1508870e42f1cea9a0e5b7134b528eb1bdcc26 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/omnidata/decode/DeserializerTestUtils.java @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2018-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode; + +import com.google.common.collect.ImmutableList; +import com.huawei.boostkit.omnidata.exception.OmniDataException; +import io.prestosql.metadata.Metadata; +import io.prestosql.spi.PageBuilder; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.BlockBuilder; +import io.prestosql.spi.block.RowBlockBuilder; +import io.prestosql.spi.block.SingleRowBlockWriter; +import io.prestosql.spi.type.MapType; +import io.prestosql.spi.type.RowType; +import io.prestosql.spi.type.StandardTypes; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeSignatureParameter; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static io.airlift.slice.Slices.wrappedBuffer; +import static io.prestosql.metadata.MetadataManager.createTestMetadataManager; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TinyintType.TINYINT; +import static io.prestosql.spi.type.VarcharType.VARCHAR; + +/** + * Deserializer Test Utils + * + * @since 2022-07-18 + */ +public class DeserializerTestUtils +{ + public static final Metadata METADATA = createTestMetadataManager(); + + private DeserializerTestUtils() + {} + + /** + * Return a map type + * @param keyType keyType + * @param valueType valueType + * @return type + */ + public static MapType mapType(Type keyType, Type valueType) + { + Type type = METADATA.getFunctionAndTypeManager() + .getParameterizedType(StandardTypes.MAP, + ImmutableList.of(TypeSignatureParameter.of(keyType.getTypeSignature()), + TypeSignatureParameter.of(valueType.getTypeSignature()))); + if (type instanceof MapType) { + return (MapType) type; + } + throw new OmniDataException("Except Map type"); + } + + /** + * create Long sequence block + * + * @param start start + * @param end end + * @return block + */ + public static Block createLongSequenceBlock(int start, int end) + { + BlockBuilder builder = BIGINT.createFixedSizeBlockBuilder(end - start); + + for (int i = start; i < end; i++) { + BIGINT.writeLong(builder, i); + } + + return builder.build(); + } + + /** + * Get Test PageBuilder + * + * @return pageBuilder + */ + public static PageBuilder getTestPageBuilder() + { + // generate rowType + List fieldTypes = new ArrayList<>(); + fieldTypes.add(DOUBLE); + fieldTypes.add(BIGINT); + RowType rowType = RowType.anonymous(fieldTypes); + //generate a page + ImmutableList.Builder typesBuilder = ImmutableList.builder(); + typesBuilder.add(INTEGER, DOUBLE, BOOLEAN, BIGINT, VARCHAR, SMALLINT, DATE, TINYINT, REAL, rowType); + + ImmutableList types = typesBuilder.build(); + PageBuilder pageBuilder = new PageBuilder(types); + + fillUpPageBuilder(pageBuilder); + + // RowType + BlockBuilder builder = pageBuilder.getBlockBuilder(9); + if (!(builder instanceof RowBlockBuilder)) { + throw new OmniDataException("Except RowBlockBuilder but found " + builder.getClass()); + } + RowBlockBuilder rowBlockBuilder = (RowBlockBuilder) builder; + SingleRowBlockWriter singleRowBlockWriter = rowBlockBuilder.beginBlockEntry(); + DOUBLE.writeDouble(singleRowBlockWriter, 1.0); + BIGINT.writeLong(singleRowBlockWriter, 1); + rowBlockBuilder.closeEntry(); + + singleRowBlockWriter = rowBlockBuilder.beginBlockEntry(); + singleRowBlockWriter.appendNull(); + singleRowBlockWriter.appendNull(); + rowBlockBuilder.closeEntry(); + + pageBuilder.declarePositions(2); + + return pageBuilder; + } + + private static void fillUpPageBuilder(PageBuilder pageBuilder) + { + BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); + blockBuilder.writeInt(1); + blockBuilder.appendNull(); + + // DOUBLE + blockBuilder = pageBuilder.getBlockBuilder(1); + blockBuilder.writeLong(Double.doubleToLongBits(1.0)); + blockBuilder.appendNull(); + + // BOOLEAN false + blockBuilder = pageBuilder.getBlockBuilder(2); + blockBuilder.writeByte(0); + blockBuilder.appendNull(); + + // LONG + blockBuilder = pageBuilder.getBlockBuilder(3); + blockBuilder.writeLong(1); + blockBuilder.appendNull(); + + // VARCHAR + blockBuilder = pageBuilder.getBlockBuilder(4); + blockBuilder.writeBytes(wrappedBuffer("test".getBytes(StandardCharsets.UTF_8)), 0, "test".length()); + blockBuilder.closeEntry(); + blockBuilder.appendNull(); + + // SMALLINT + blockBuilder = pageBuilder.getBlockBuilder(5); + blockBuilder.writeShort(1); + blockBuilder.appendNull(); + + // DATE + blockBuilder = pageBuilder.getBlockBuilder(6); + blockBuilder.writeInt(1); + blockBuilder.appendNull(); + + // TINYINT + blockBuilder = pageBuilder.getBlockBuilder(7); + blockBuilder.writeByte(1); + blockBuilder.appendNull(); + + // REAL + blockBuilder = pageBuilder.getBlockBuilder(8); + blockBuilder.writeInt(Float.floatToIntBits((float) 1.0)); + blockBuilder.appendNull(); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/omnidata/decode/TestOpenLooKengDeserializer.java b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/omnidata/decode/TestOpenLooKengDeserializer.java new file mode 100644 index 0000000000000000000000000000000000000000..b70349af7d54b913ec2cd3031081627e370d2eda --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/omnidata/decode/TestOpenLooKengDeserializer.java @@ -0,0 +1,334 @@ +/* + * Copyright (C) 2018-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode; + +import com.google.common.collect.ImmutableList; +import com.huawei.boostkit.omnidata.exception.OmniDataException; +import com.huawei.boostkit.omnidata.serialize.OmniDataBlockEncodingSerde; +import io.airlift.compress.Decompressor; +import io.airlift.compress.zstd.ZstdDecompressor; +import io.airlift.slice.Slice; +import io.airlift.slice.SliceInput; +import io.hetu.core.transport.execution.buffer.PagesSerde; +import io.hetu.core.transport.execution.buffer.PagesSerdeFactory; +import io.hetu.core.transport.execution.buffer.SerializedPage; +import io.prestosql.plugin.hive.omnidata.decode.impl.OpenLooKengDecoding; +import io.prestosql.plugin.hive.omnidata.decode.impl.OpenLooKengDeserializer; +import io.prestosql.plugin.hive.omnidata.decode.type.ArrayDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.DateDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.DecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.LongDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.LongToByteDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.LongToFloatDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.LongToIntDecodeType; +import io.prestosql.plugin.hive.omnidata.decode.type.LongToShortDecodeType; +import io.prestosql.spi.Page; +import io.prestosql.spi.PageBuilder; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.BlockBuilder; +import io.prestosql.spi.block.DictionaryBlock; +import io.prestosql.spi.block.Int128ArrayBlock; +import io.prestosql.spi.block.LongArrayBlock; +import io.prestosql.spi.block.RunLengthEncodedBlock; +import io.prestosql.spi.block.SingleMapBlock; +import io.prestosql.spi.type.ArrayType; +import io.prestosql.spi.type.MapType; +import io.prestosql.spi.type.Type; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Optional; + +import static io.prestosql.plugin.hive.omnidata.decode.DeserializerTestUtils.METADATA; +import static io.prestosql.plugin.hive.omnidata.decode.DeserializerTestUtils.createLongSequenceBlock; +import static io.prestosql.plugin.hive.omnidata.decode.DeserializerTestUtils.getTestPageBuilder; +import static io.prestosql.plugin.hive.omnidata.decode.DeserializerTestUtils.mapType; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +/** + * Block deserializer test + * + * @since 2022-07-19 + */ +public class TestOpenLooKengDeserializer +{ + /** + * Test all types + */ + @Test + public void testAllSupportTypesDeserializer() + { + Page page = getTestPageBuilder().build(); + + // serialize page + PagesSerdeFactory factory = new PagesSerdeFactory(new OmniDataBlockEncodingSerde(), false); + PagesSerde pagesSerde = factory.createPagesSerde(); + SerializedPage serializedPage = pagesSerde.serialize(page); + + // deserialize page + OpenLooKengDeserializer deserializer = new OpenLooKengDeserializer(); + Page deserializedPage = deserializer.deserialize(serializedPage); + + assertEquals(2, deserializedPage.getPositionCount()); + assertEquals(10, deserializedPage.getChannelCount()); + + assertEquals(deserializedPage.getBlock(0).getSizeInBytes(), 10); + assertEquals(deserializedPage.getBlock(1).getSizeInBytes(), 18); + assertEquals(deserializedPage.getBlock(2).getSizeInBytes(), 4); + assertEquals(deserializedPage.getBlock(3).getSizeInBytes(), 18); + assertEquals(deserializedPage.getBlock(4).getSizeInBytes(), 14); + assertEquals(deserializedPage.getBlock(5).getSizeInBytes(), 6); + assertEquals(deserializedPage.getBlock(6).getSizeInBytes(), 10); + assertEquals(deserializedPage.getBlock(7).getSizeInBytes(), 4); + assertEquals(deserializedPage.getBlock(8).getSizeInBytes(), 10); + assertEquals(deserializedPage.getBlock(9).getSizeInBytes(), 46); + } + + /** + * Test Compressed deserializer + */ + @Test + public void testCompressedDeserializer() + { + long[] values = new long[] {1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 0}; + boolean[] valueIsNull = new boolean[] {false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, true}; + LongArrayBlock longArrayBlock = new LongArrayBlock(16, Optional.of(valueIsNull), values); + Page longArrayPage = new Page(longArrayBlock); + + // serialize page + PagesSerdeFactory factory = new PagesSerdeFactory(new OmniDataBlockEncodingSerde(), true); + PagesSerde pagesSerde = factory.createPagesSerde(); + SerializedPage serializedPage = pagesSerde.serialize(longArrayPage); + + // deserialize page + DecodeType[] decodeTypes = new DecodeType[] { + new LongToIntDecodeType(), new LongToShortDecodeType(), new LongToByteDecodeType(), + new LongToFloatDecodeType(), new DateDecodeType() + }; + + OpenLooKengDecoding blockDecoding = new OpenLooKengDecoding(); + Decompressor decompressor = new ZstdDecompressor(); + int failedTimes = 0; + for (int i = 0; i < decodeTypes.length; i++) { + // to decode type + Slice slice = OpenLooKengDeserializer.decompressPage(serializedPage, decompressor); + SliceInput input = slice.getInput(); + int numberOfBlocks = input.readInt(); + Block[] blocks = new Block[numberOfBlocks]; + try { + blocks[0] = blockDecoding.decode(Optional.of(decodeTypes[i]), input); + } + catch (OmniDataException e) { + failedTimes++; + } + } + + assertEquals(failedTimes, decodeTypes.length); + } + + /** + * Test Int128Type deserializer + */ + @Test + public void testInt128TypeDeserializer() + { + Int128ArrayBlock int128ArrayBlock = + new Int128ArrayBlock(0, Optional.empty(), new long[0]); + Page int128ArrayPage = new Page(int128ArrayBlock); + + // serialize page + PagesSerdeFactory factory = new PagesSerdeFactory(new OmniDataBlockEncodingSerde(), false); + PagesSerde pagesSerde = factory.createPagesSerde(); + SerializedPage serializedPage = pagesSerde.serialize(int128ArrayPage); + + // deserialize page + OpenLooKengDeserializer deserializer = new OpenLooKengDeserializer(); + + // generate exception + Page page = deserializer.deserialize(serializedPage); + + assertEquals(0, page.getSizeInBytes()); + } + + /** + * Test RunLengthType deserializer + */ + @Test + public void testRunLengthTypeDeserializer() + { + RunLengthEncodedBlock runLengthEncodedBlock = + new RunLengthEncodedBlock<>(createLongSequenceBlock(4, 5), 100); + Page runLengthPage = new Page(runLengthEncodedBlock); + + // serialize page + PagesSerdeFactory factory = new PagesSerdeFactory(new OmniDataBlockEncodingSerde(), false); + PagesSerde pagesSerde = factory.createPagesSerde(); + SerializedPage serializedPage = pagesSerde.serialize(runLengthPage); + + // deserialize page + OpenLooKengDeserializer deserializer = new OpenLooKengDeserializer(); + + // generate ColumnVector + Page page = deserializer.deserialize(serializedPage); + + assertEquals(9, page.getSizeInBytes()); + } + + /** + * Test ArrayType deserializer + */ + @Test + public void testArrayTypeDeserializer() + { + // generate a page + ImmutableList.Builder typeBuilder = ImmutableList.builder(); + typeBuilder.add(new ArrayType<>(BIGINT)); + + ImmutableList types = typeBuilder.build(); + PageBuilder pageBuilder = new PageBuilder(types); + BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); + BlockBuilder elementBlockBuilder = BIGINT.createBlockBuilder(null, 2); + for (int i = 0; i < 2; i++) { + BIGINT.writeLong(elementBlockBuilder, 1); + } + blockBuilder.appendStructure(elementBlockBuilder.build()); + + pageBuilder.declarePositions(1); + Page page = pageBuilder.build(); + + // serialize page + PagesSerdeFactory factory = new PagesSerdeFactory(new OmniDataBlockEncodingSerde(), false); + PagesSerde pagesSerde = factory.createPagesSerde(); + SerializedPage serializedPage = pagesSerde.serialize(page); + + //deserialize page + DecodeType[] decodeTypes = new DecodeType[] {new ArrayDecodeType<>(new LongDecodeType())}; + DecodeType firstType = decodeTypes[0]; + if (firstType instanceof ArrayDecodeType) { + assertEquals(((ArrayDecodeType) firstType).getElementType().getClass(), LongDecodeType.class); + } + else { + throw new OmniDataException("except arrayType"); + } + OpenLooKengDeserializer deserializer = new OpenLooKengDeserializer(); + + // generate exception + Page deserialized = deserializer.deserialize(serializedPage); + + assertEquals(23, deserialized.getSizeInBytes()); + } + + /** + * Test MapType deserializer + */ + @Test(expectedExceptions = {UnsupportedOperationException.class, RuntimeException.class}) + public void testMapTypeDeserializer() + { + // generate a page + ImmutableList.Builder typeBuilder = ImmutableList.builder(); + MapType mapType = mapType(BIGINT, BIGINT); + + typeBuilder.add(mapType); + + ImmutableList types = typeBuilder.build(); + PageBuilder pageBuilder = new PageBuilder(types); + BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); + blockBuilder.appendNull(); + + pageBuilder.declarePositions(1); + Page page = pageBuilder.build(); + + // serialize page + PagesSerdeFactory factory = new PagesSerdeFactory(METADATA.getFunctionAndTypeManager().getBlockEncodingSerde(), + false); + PagesSerde pagesSerde = factory.createPagesSerde(); + SerializedPage serializedPage = pagesSerde.serialize(page); + + // deserialize page + OpenLooKengDeserializer deserializer = new OpenLooKengDeserializer(); + + // generate exception + deserializer.deserialize(serializedPage); + } + + /** + * Test SingleMapType deserializer + */ + @Test(expectedExceptions = {UnsupportedOperationException.class, RuntimeException.class}) + public void testSingleMapTypeDeserializer() + { + // generate a page + ImmutableList.Builder typeBuilder = ImmutableList.builder(); + MapType mapType = mapType(BIGINT, BIGINT); + + typeBuilder.add(mapType); + + ImmutableList types = typeBuilder.build(); + PageBuilder pageBuilder = new PageBuilder(types); + BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); + blockBuilder.appendNull(); + + pageBuilder.declarePositions(1); + Page page = pageBuilder.build(); + Block block = page.getBlock(0); + + Block elementBlock = mapType.getObject(block, 0); + assertTrue(elementBlock instanceof SingleMapBlock); + + Page singleMapPage = new Page(elementBlock); + + // serialize page + PagesSerdeFactory factory = new PagesSerdeFactory(METADATA.getFunctionAndTypeManager().getBlockEncodingSerde(), + false); + PagesSerde pagesSerde = factory.createPagesSerde(); + SerializedPage serializedPage = pagesSerde.serialize(singleMapPage); + + // deserialize page + OpenLooKengDeserializer deserializer = new OpenLooKengDeserializer(); + + // generate exception + deserializer.deserialize(serializedPage); + } + + /** + * Test DictionaryType deserializer + */ + @Test + public void testDictionaryTypeDeserializer() + { + int[] ids = new int[100]; + Arrays.setAll(ids, index -> index % 10); + Block dictionary = createLongSequenceBlock(0, 10); + DictionaryBlock dictionaryBlock = new DictionaryBlock<>(dictionary, ids); + Page dictionaryPage = new Page(dictionaryBlock); + + // serialize page + PagesSerdeFactory factory = new PagesSerdeFactory(new OmniDataBlockEncodingSerde(), false); + PagesSerde pagesSerde = factory.createPagesSerde(); + SerializedPage serializedPage = pagesSerde.serialize(dictionaryPage); + + // deserialize page + OpenLooKengDeserializer deserializer = new OpenLooKengDeserializer(); + + // generate exception + Page page = deserializer.deserialize(serializedPage); + + assertEquals(490, page.getSizeInBytes()); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/omnidata/decode/type/TestOmniDataDecodeTypes.java b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/omnidata/decode/type/TestOmniDataDecodeTypes.java new file mode 100644 index 0000000000000000000000000000000000000000..937bdf55640bf08b5ec4ddb45a839332a9160754 --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/connector/src/test/java/io/prestosql/plugin/hive/omnidata/decode/type/TestOmniDataDecodeTypes.java @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2018-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.plugin.hive.omnidata.decode.type; + +import io.prestosql.spi.type.RowType; +import org.testng.annotations.Test; + +import java.util.Optional; + +import static org.testng.Assert.assertEquals; + +/** + * Test All OmniData DecodeTypes + * + * @since 2022-07-18 + */ +public class TestOmniDataDecodeTypes +{ + @Test + public void testOmnidataDecodeTypes() + { + // Test DecimalDecodeType + DecodeType decimalDecodeType = new DecimalDecodeType(10, 5); + assertEquals(decimalDecodeType.getJavaType().get(), DecimalDecodeType.class); + + // Test TimestampDecodeType + DecodeType timestampDecodeType = new TimestampDecodeType(); + assertEquals(timestampDecodeType.getJavaType().get(), TimestampDecodeType.class); + + // Test VarcharDecodeType + DecodeType varcharDecodeType = new VarcharDecodeType(); + assertEquals(varcharDecodeType.getJavaType().get(), String.class); + + // Test ShortDecodeType + DecodeType shortDecodeType = new ShortDecodeType(); + assertEquals(shortDecodeType.getJavaType().get(), short.class); + + // Test RowDecodeType + DecodeType rowDecodeType = new RowDecodeType(); + assertEquals(rowDecodeType.getJavaType().get(), RowType.class); + + // Test MapDecodeType + DecodeType mapDecodeType = new MapDecodeType(new ShortDecodeType(), new ShortDecodeType()); + assertEquals(mapDecodeType.getJavaType(), Optional.empty()); + + // Test IntDecodeType + DecodeType intDecodeType = new IntDecodeType(); + assertEquals(intDecodeType.getJavaType().get(), int.class); + + // Test FloatDecodeType + DecodeType floatDecodeType = new FloatDecodeType(); + assertEquals(floatDecodeType.getJavaType().get(), float.class); + + // Test DoubleDecodeType + DecodeType doubleDecodeType = new DoubleDecodeType(); + assertEquals(doubleDecodeType.getJavaType().get(), double.class); + + // Test ByteDecodeType + DecodeType byteDecodeType = new ByteDecodeType(); + assertEquals(byteDecodeType.getJavaType().get(), byte.class); + + // Test BooleanDecodeType + DecodeType booleanDecodeType = new BooleanDecodeType(); + assertEquals(booleanDecodeType.getJavaType().get(), boolean.class); + } +} diff --git a/omnidata/omnidata-openlookeng-connector/stub/server/pom.xml b/omnidata/omnidata-openlookeng-connector/stub/server/pom.xml index 11fde2862f4568188c78e3b0e85751c7343ae8cf..e4b16fac7880bdf8d2aed7fe897a94eb6b40a45b 100644 --- a/omnidata/omnidata-openlookeng-connector/stub/server/pom.xml +++ b/omnidata/omnidata-openlookeng-connector/stub/server/pom.xml @@ -23,5 +23,11 @@ commons-io ${dep.commons.io.version} + + io.hetu.core + hetu-transport + 1.6.1 + compile + \ No newline at end of file diff --git a/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/decode/Deserializer.java b/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/decode/Deserializer.java index ea5c2957b0d1ea76e9d72340eb9d0d6f2cdf6c98..e53a6fc6386bec1a6af167795a6d7517801b10ca 100644 --- a/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/decode/Deserializer.java +++ b/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/decode/Deserializer.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (C) 2018-2022. Huawei Technologies Co., Ltd. All rights reserved. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -14,5 +14,8 @@ */ package com.huawei.boostkit.omnidata.decode; +import io.hetu.core.transport.execution.buffer.SerializedPage; + public interface Deserializer { + T deserialize(SerializedPage page); } diff --git a/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/exception/OmniDataException.java b/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/exception/OmniDataException.java index 98ca5c4495ba1850e6f0d583868926a94c0ab433..9dd26007a8c97aacecfb27780ceba6ad0dae8699 100644 --- a/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/exception/OmniDataException.java +++ b/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/exception/OmniDataException.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved. + * Copyright (C) 2018-2022. Huawei Technologies Co., Ltd. All rights reserved. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -16,8 +16,32 @@ package com.huawei.boostkit.omnidata.exception; import static com.huawei.boostkit.omnidata.exception.OmniErrorCode.OMNIDATA_GENERIC_ERROR; -public class OmniDataException { +/** + * OmniDataException + * + * @since 2022-07-18 + */ +public class OmniDataException extends RuntimeException { + private static final long serialVersionUID = -9034897193745766939L; + + private final OmniErrorCode errorCode; + + public OmniDataException(String message) { + super(message); + errorCode = OMNIDATA_GENERIC_ERROR; + } + + public OmniDataException(String message, Throwable throwable) { + super(message, throwable); + errorCode = OMNIDATA_GENERIC_ERROR; + } + + public OmniDataException(OmniErrorCode omniErrorCode, String message) { + super(message); + errorCode = omniErrorCode; + } + public OmniErrorCode getErrorCode() { - return OMNIDATA_GENERIC_ERROR; + return errorCode; } } diff --git a/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/serialize/OmniDataBlockEncodingSerde.java b/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/serialize/OmniDataBlockEncodingSerde.java new file mode 100644 index 0000000000000000000000000000000000000000..7ebc828d2b80453fd99af2f565bd4aedf53a6e6a --- /dev/null +++ b/omnidata/omnidata-openlookeng-connector/stub/server/src/main/java/com/huawei/boostkit/omnidata/serialize/OmniDataBlockEncodingSerde.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2022-2022. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.huawei.boostkit.omnidata.serialize; + +import com.google.common.collect.ImmutableMap; +import io.airlift.slice.SliceInput; +import io.airlift.slice.SliceOutput; +import io.prestosql.spi.block.ArrayBlockEncoding; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.BlockEncoding; +import io.prestosql.spi.block.BlockEncodingSerde; +import io.prestosql.spi.block.ByteArrayBlockEncoding; +import io.prestosql.spi.block.DictionaryBlockEncoding; +import io.prestosql.spi.block.Int128ArrayBlockEncoding; +import io.prestosql.spi.block.IntArrayBlockEncoding; +import io.prestosql.spi.block.LazyBlockEncoding; +import io.prestosql.spi.block.LongArrayBlockEncoding; +import io.prestosql.spi.block.RowBlockEncoding; +import io.prestosql.spi.block.RunLengthBlockEncoding; +import io.prestosql.spi.block.ShortArrayBlockEncoding; +import io.prestosql.spi.block.SingleRowBlockEncoding; +import io.prestosql.spi.block.VariableWidthBlockEncoding; + +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Optional; + +public class OmniDataBlockEncodingSerde implements BlockEncodingSerde { + private final Map blockEncodings; + public OmniDataBlockEncodingSerde() { + blockEncodings = + ImmutableMap.builder() + .put(VariableWidthBlockEncoding.NAME, new VariableWidthBlockEncoding()) + .put(ByteArrayBlockEncoding.NAME, new ByteArrayBlockEncoding()) + .put(ShortArrayBlockEncoding.NAME, new ShortArrayBlockEncoding()) + .put(IntArrayBlockEncoding.NAME, new IntArrayBlockEncoding()) + .put(LongArrayBlockEncoding.NAME, new LongArrayBlockEncoding()) + .put(Int128ArrayBlockEncoding.NAME, new Int128ArrayBlockEncoding()) + .put(DictionaryBlockEncoding.NAME, new DictionaryBlockEncoding()) + .put(ArrayBlockEncoding.NAME, new ArrayBlockEncoding()) + .put(RowBlockEncoding.NAME, new RowBlockEncoding()) + .put(SingleRowBlockEncoding.NAME, new SingleRowBlockEncoding()) + .put(RunLengthBlockEncoding.NAME, new RunLengthBlockEncoding()) + .put(LazyBlockEncoding.NAME, new LazyBlockEncoding()) + .build(); + } + + private static String readLengthPrefixedString(SliceInput sliceInput) + { + int length = sliceInput.readInt(); + byte[] bytes = new byte[length]; + sliceInput.readBytes(bytes); + + return new String(bytes, StandardCharsets.UTF_8); + } + + private static void writeLengthPrefixedString(SliceOutput sliceOutput, String value) + { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + sliceOutput.writeInt(bytes.length); + sliceOutput.writeBytes(bytes); + } + + @Override + public Block readBlock(SliceInput input) + { + return blockEncodings.get(readLengthPrefixedString(input)).readBlock(this, input); + } + + @Override + public void writeBlock(SliceOutput output, Block block) + { + Block readBlock = block; + while (true) { + String encodingName = readBlock.getEncodingName(); + + BlockEncoding blockEncoding = blockEncodings.get(encodingName); + + Optional replacementBlock = blockEncoding.replacementBlockForWrite(readBlock); + if (replacementBlock.isPresent()) { + readBlock = replacementBlock.get(); + continue; + } + + writeLengthPrefixedString(output, encodingName); + + blockEncoding.writeBlock(this, output, readBlock); + + break; + } + } +}