From 502813c130074d0aa92dae0d0150a3c0c18bf20a Mon Sep 17 00:00:00 2001 From: chen-guang-wang <18767185082@163.com> Date: Tue, 20 Sep 2022 17:53:40 +0800 Subject: [PATCH 1/3] fixbug for tablescan decimal --- .../huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java index 555c1db50..7b2b74c25 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java @@ -84,9 +84,9 @@ public class OrcColumnarBatchJniReader { String[] spiltValues = pl.getLiteral().toString().split("\\."); String strToAdd = ""; if (spiltValues.length == 2) { - strToAdd = String.format("%1$" + decimalS + "s", spiltValues[1]).replace(' ', '0'); + strToAdd = String.format("%1$-" + decimalS + "s", spiltValues[1]).replace(' ', '0'); } else { - strToAdd = String.format("%1$" + decimalS + "s", "").replace(' ', '0'); + strToAdd = String.format("%1$-" + decimalS + "s", "").replace(' ', '0'); } jsonObject.put("literal", spiltValues[0] + "." + strToAdd + " " + decimalP + " " + decimalS); } else { -- Gitee From abe1fc6056da7e592656cedba04b2e49a7b6a6a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=A8=E5=85=89?= <18767185082@163.com> Date: Tue, 20 Sep 2022 12:09:31 +0000 Subject: [PATCH 2/3] tableScan decimal decimals pad 0 fixup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 王晨光 <18767185082@163.com> --- .../spark/jni/OrcColumnarBatchJniReader.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java index 7b2b74c25..8f1bd0211 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java @@ -82,12 +82,12 @@ public class OrcColumnarBatchJniReader { int decimalP = schema.findSubtype(pl.getColumnName()).getPrecision(); int decimalS = schema.findSubtype(pl.getColumnName()).getScale(); String[] spiltValues = pl.getLiteral().toString().split("\\."); - String strToAdd = ""; - if (spiltValues.length == 2) { - strToAdd = String.format("%1$-" + decimalS + "s", spiltValues[1]).replace(' ', '0'); - } else { - strToAdd = String.format("%1$-" + decimalS + "s", "").replace(' ', '0'); + String decimalVal = ""; // Integer without decimals, eg: 12345 + if (spiltValues.length == 2) { // Integer with decimals, eg: 12345.6 + decimalVal = spiltValues[1]; } + // If the length of the formatted number string is insufficient, pad '0's. + String strToAdd = String.format("%1$-" + decimalS + "s", decimalVal).replace(' ', '0'); jsonObject.put("literal", spiltValues[0] + "." + strToAdd + " " + decimalP + " " + decimalS); } else { jsonObject.put("literal", pl.getLiteral().toString()); @@ -102,12 +102,12 @@ public class OrcColumnarBatchJniReader { int decimalP = schema.findSubtype(pl.getColumnName()).getPrecision(); int decimalS = schema.findSubtype(pl.getColumnName()).getScale(); String[] spiltValues = ob.toString().split("\\."); - String strToAdd = ""; - if (spiltValues.length == 2) { - strToAdd = String.format("%1$" + decimalS + "s", spiltValues[1]).replace(' ', '0'); - } else { - strToAdd = String.format("%1$" + decimalS + "s", "").replace(' ', '0'); + String decimalVal = ""; // Integer without decimals, eg: 12345 + if (spiltValues.length == 2) { // Integer with decimals, eg: 12345.6 + decimalVal = spiltValues[1]; } + // If the length of the formatted number string is insufficient, pad '0's. + String strToAdd = String.format("%1$-" + decimalS + "s", decimalVal).replace(' ', '0'); lst.add(spiltValues[0] + "." + strToAdd + " " + decimalP + " " + decimalS); } else if (pl.getType() == PredicateLeaf.Type.DATE) { lst.add(((int)Math.ceil(((Date)pl.getLiteral()).getTime()* 1.0/3600/24/1000)) + ""); -- Gitee From 9794262ee9e8bbe64923111d1e761277bebc66d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=A8=E5=85=89?= <18767185082@163.com> Date: Tue, 20 Sep 2022 12:26:19 +0000 Subject: [PATCH 3/3] refactor for tablescan json phrase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 王晨光 <18767185082@163.com> --- .../spark/jni/OrcColumnarBatchJniReader.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java index 8f1bd0211..0a0d2208b 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java +++ b/omnioperator/omniop-spark-extension/java/src/main/java/com/huawei/boostkit/spark/jni/OrcColumnarBatchJniReader.java @@ -67,6 +67,15 @@ public class OrcColumnarBatchJniReader { return jsonObject; } + public String PadZeroForDecimals(String [] decimalStrArray, int decimalScale) { + String decimalVal = ""; // Integer without decimals, eg: 12345 + if (decimalStrArray.length == 2) { // Integer with decimals, eg: 12345.6 + decimalVal = decimalStrArray[1]; + } + // If the length of the formatted number string is insufficient, pad '0's. + return String.format("%1$-" + decimalScale + "s", decimalVal).replace(' ', '0'); + } + public JSONObject getLeavesJson(List leaves, TypeDescription schema) { JSONObject jsonObjectList = new JSONObject(); for (int i = 0; i < leaves.size(); i++) { @@ -82,13 +91,8 @@ public class OrcColumnarBatchJniReader { int decimalP = schema.findSubtype(pl.getColumnName()).getPrecision(); int decimalS = schema.findSubtype(pl.getColumnName()).getScale(); String[] spiltValues = pl.getLiteral().toString().split("\\."); - String decimalVal = ""; // Integer without decimals, eg: 12345 - if (spiltValues.length == 2) { // Integer with decimals, eg: 12345.6 - decimalVal = spiltValues[1]; - } - // If the length of the formatted number string is insufficient, pad '0's. - String strToAdd = String.format("%1$-" + decimalS + "s", decimalVal).replace(' ', '0'); - jsonObject.put("literal", spiltValues[0] + "." + strToAdd + " " + decimalP + " " + decimalS); + String scalePadZeroStr = PadZeroForDecimals(spiltValues, decimalS); + jsonObject.put("literal", spiltValues[0] + "." + scalePadZeroStr + " " + decimalP + " " + decimalS); } else { jsonObject.put("literal", pl.getLiteral().toString()); } @@ -102,13 +106,8 @@ public class OrcColumnarBatchJniReader { int decimalP = schema.findSubtype(pl.getColumnName()).getPrecision(); int decimalS = schema.findSubtype(pl.getColumnName()).getScale(); String[] spiltValues = ob.toString().split("\\."); - String decimalVal = ""; // Integer without decimals, eg: 12345 - if (spiltValues.length == 2) { // Integer with decimals, eg: 12345.6 - decimalVal = spiltValues[1]; - } - // If the length of the formatted number string is insufficient, pad '0's. - String strToAdd = String.format("%1$-" + decimalS + "s", decimalVal).replace(' ', '0'); - lst.add(spiltValues[0] + "." + strToAdd + " " + decimalP + " " + decimalS); + String scalePadZeroStr = PadZeroForDecimals(spiltValues, decimalS); + lst.add(spiltValues[0] + "." + scalePadZeroStr + " " + decimalP + " " + decimalS); } else if (pl.getType() == PredicateLeaf.Type.DATE) { lst.add(((int)Math.ceil(((Date)pl.getLiteral()).getTime()* 1.0/3600/24/1000)) + ""); } else { -- Gitee