diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/reader/OmniOrcRecordReader.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/reader/OmniOrcRecordReader.java index d0fbf3f0b5602585546f4e6e535151d0d35228bb..bb63e7581c1c34bcae600d3bf921656c8317530d 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/reader/OmniOrcRecordReader.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/reader/OmniOrcRecordReader.java @@ -95,11 +95,8 @@ public class OmniOrcRecordReader implements RecordReader { if (op.getType().equals(OperatorType.TABLESCAN)) { @@ -136,23 +132,6 @@ public class OmniOrcRecordReader implements RecordReader requiredIds = new HashSet<>(included); TypeDescription result = TypeDescription.createStruct(); diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/reader/OrcColumnarBatchScanReader.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/reader/OrcColumnarBatchScanReader.java index 89c40ed4cbd101a80b7cb7df668521b3eb98cd03..3d5e6851ccf3eef8604608c82a79b8ba87d50e74 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/reader/OrcColumnarBatchScanReader.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/reader/OrcColumnarBatchScanReader.java @@ -56,6 +56,8 @@ public class OrcColumnarBatchScanReader { public OrcColumnarBatchJniReader jniReader; + private TypeDescription requiredSchema; + public OrcColumnarBatchScanReader() { jniReader = new OrcColumnarBatchJniReader(); } @@ -149,13 +151,9 @@ public class OrcColumnarBatchScanReader { * @param options split file options * @return long */ - public long initializeReaderJava(URI uri, ReaderOptions options) { + public long initializeReaderJava(URI uri) { JSONObject job = new JSONObject(); - if (options.getOrcTail() == null) { - job.put("serializedTail", ""); - } else { - job.put("serializedTail", options.getOrcTail().getSerializedTail().toString()); - } + job.put("serializedTail", ""); job.put("tailLocation", 9223372036854775807L); job.put("scheme", uri.getScheme() == null ? "" : uri.getScheme()); job.put("host", uri.getHost() == null ? "" : uri.getHost()); @@ -174,31 +172,15 @@ public class OrcColumnarBatchScanReader { * @param options split file options * @return long */ - public long initializeRecordReaderJava(Options options) { + public long initializeRecordReaderJava(long offset, long length, TypeDescription requiredSchema) { + this.requiredSchema = requiredSchema; JSONObject job = new JSONObject(); - if (options.getInclude() == null) { - job.put("include", ""); - } else { - job.put("include", options.getInclude().toString()); - } - job.put("offset", options.getOffset()); - job.put("length", options.getLength()); - if (options.getSearchArgument() != null) { - LOGGER.debug("SearchArgument: {}", options.getSearchArgument().toString()); - JSONObject jsonexpressionTree = getSubJson(options.getSearchArgument().getExpression()); - job.put("expressionTree", jsonexpressionTree); - JSONObject jsonleaves = getLeavesJson(options.getSearchArgument().getLeaves(), options.getSchema()); - job.put("leaves", jsonleaves); - } + job.put("offset", offset); + job.put("length", length); - List allCols; - if (options.getColumnNames() == null) { - allCols = Arrays.asList(jniReader.getAllColumnNames(reader)); - } else { - allCols = Arrays.asList(options.getColumnNames()); - } + List allCols = Arrays.asList(jniReader.getAllColumnNames(reader)); ArrayList colToInclu = new ArrayList(); - List optionField = options.getSchema().getFieldNames(); + List optionField = requiredSchema.getFieldNames(); colsToGet = new int[optionField.size()]; realColsCnt = 0; for (int i = 0; i < optionField.size(); i++) { @@ -251,6 +233,7 @@ public class OrcColumnarBatchScanReader { int rowCount = 0; for (int i = 0; i < vectorCnt; i++) { if (colsToGet[i] != 0) { + rowCount = (int) rtn; vecList[i] = getNullVec(DataType.DataTypeId.values()[typeIds[nativeGetId]], rowCount); nativeGetId++; continue; @@ -292,7 +275,6 @@ public class OrcColumnarBatchScanReader { } } nativeGetId++; - rowCount = vecList[i].getSize(); } return (int) rtn; }