From 72c0dca3aa63d87cc8f4ec5eaaef2438287fcf23 Mon Sep 17 00:00:00 2001 From: l30037350 Date: Mon, 29 Aug 2022 15:49:18 +0800 Subject: [PATCH 01/10] expression support Chinese --- .../expression/OmniExpressionAdaptor.scala | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index 853462e3c..b808aabfd 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -479,6 +479,11 @@ object OmniExpressionAdaptor extends Logging { sparkTypeToOmniExpJsonType(BooleanType), rewriteToOmniJsonExpressionLiteral(isnotnull.child, exprsIndexMap)) + case isNull: IsNull => + "{\"exprType\":\"IS_NULL\",\"returnType\":%s,\"arguments\":[%s]}}".format( + sparkTypeToOmniExpJsonType(BooleanType), + rewriteToOmniJsonExpressionLiteral(isNull.child, exprsIndexMap)) + // Substring case subString: Substring => ("{\"exprType\":\"FUNCTION\",\"returnType\":%s," + @@ -541,20 +546,25 @@ object OmniExpressionAdaptor extends Logging { rewriteToOmniJsonExpressionLiteral(ifExp.trueValue, exprsIndexMap), rewriteToOmniJsonExpressionLiteral(ifExp.falseValue, exprsIndexMap)) - // only support with one case condition, for omni rewrite to if(A, B, C) case caseWhen: CaseWhen => - "{\"exprType\":\"IF\",\"returnType\":%s,\"condition\":%s,\"if_true\":%s,\"if_false\":%s}" - .format(sparkTypeToOmniExpJsonType(caseWhen.dataType), - rewriteToOmniJsonExpressionLiteral(caseWhen.branches(0)._1, exprsIndexMap), - rewriteToOmniJsonExpressionLiteral(caseWhen.branches(0)._2, exprsIndexMap), - rewriteToOmniJsonExpressionLiteral(caseWhen.elseValue.get, exprsIndexMap)) + procCaseWhenExpression(caseWhen, exprsIndexMap) case coalesce: Coalesce => + if (coalesce.children.length > 2) { + throw new UnsupportedOperationException(s"Unsupported expression: $expr") + } "{\"exprType\":\"COALESCE\",\"returnType\":%s, \"value1\":%s,\"value2\":%s}".format( sparkTypeToOmniExpJsonType(coalesce.dataType), rewriteToOmniJsonExpressionLiteral(coalesce.children(0), exprsIndexMap), rewriteToOmniJsonExpressionLiteral(coalesce.children(1), exprsIndexMap)) + case like: Like => + ("{\"exprType\":\"FUNCTION\",\"returnType\":%s," + + "\"function_name\":\"LIKE\", \"arguments\":[%s, %s]}") + .format(sparkTypeToOmniExpJsonType(like.dataType), + rewriteToOmniJsonExpressionLiteral(like.left, exprsIndexMap), + rewriteToOmniJsonExpressionLiteral(like.right, exprsIndexMap)) + case concat: Concat => getConcatJsonStr(concat, exprsIndexMap) case attr: Attribute => toOmniJsonAttribute(attr, exprsIndexMap(attr.exprId)) @@ -770,4 +780,23 @@ object OmniExpressionAdaptor extends Logging { } width } + + def procCaseWhenExpression(caseWhen: CaseWhen, + exprsIndexMap: Map[ExprId, Int]): String = { + val exprStr = "{\"exprType\":\"IF\",\"returnType\":%s,\"condition\":%s,\"if_true\":%s,\"if_false\":%s}" + var exprStrRes = exprStr + for (i <- caseWhen.branches.indices) { + var ifFalseStr = "" + if (i != caseWhen.branches.length - 1) { + ifFalseStr = exprStr + } else { + ifFalseStr = rewriteToOmniJsonExpressionLiteral(caseWhen.elseValue.get, exprsIndexMap) + } + exprStrRes = exprStrRes.format(sparkTypeToOmniExpJsonType(caseWhen.dataType), + rewriteToOmniJsonExpressionLiteral(caseWhen.branches(i)._1, exprsIndexMap), + rewriteToOmniJsonExpressionLiteral(caseWhen.branches(i)._2, exprsIndexMap), + ifFalseStr) + } + exprStrRes + } } -- Gitee From 0a1a2c58b79840b9c1a5a237451dba2cadefa99d Mon Sep 17 00:00:00 2001 From: l30037350 Date: Wed, 31 Aug 2022 17:17:34 +0800 Subject: [PATCH 02/10] like expression support Chinese like expression support Chinese like expression support Chinese like expression support Chinese isNull expression support Chinese fix like expression support Chinese fix like expression support Chinese fix like,caseWhen,coalesce,isnull expression UT like,caseWhen,coalesce,isnull expression UT like expression support Chinese like expression support Chinese like expression support Chinese like expression support Chinese isNull expression support Chinese fix like expression support Chinese fix like expression support Chinese fix like,caseWhen,coalesce,isnull expression UT like,caseWhen,coalesce,isnull expression UT like expression support Chinese like expression support Chinese like expression support Chinese like expression support Chinese isNull expression support Chinese fix like expression support Chinese fix like expression support Chinese fix like,caseWhen,coalesce,isnull expression UT like,caseWhen,coalesce,isnull expression UT like expression support Chinese like expression support Chinese like expression support Chinese like expression support Chinese isNull expression support Chinese fix like expression support Chinese fix like expression support Chinese fix like,caseWhen,coalesce,isnull expression UT like,caseWhen,coalesce,isnull expression UT --- .../expression/OmniExpressionAdaptor.scala | 147 ++++++++++++++++-- .../OmniExpressionAdaptorSuite.scala | 111 ++++++++++++- 2 files changed, 248 insertions(+), 10 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index b808aabfd..03db148c3 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.util.CharVarcharUtils.getRawTypeString import org.apache.spark.sql.types.{BooleanType, DataType, DateType, Decimal, DecimalType, DoubleType, IntegerType, LongType, Metadata, ShortType, StringType} +import scala.collection.mutable import scala.collection.mutable.ArrayBuffer object OmniExpressionAdaptor extends Logging { @@ -480,7 +481,7 @@ object OmniExpressionAdaptor extends Logging { rewriteToOmniJsonExpressionLiteral(isnotnull.child, exprsIndexMap)) case isNull: IsNull => - "{\"exprType\":\"IS_NULL\",\"returnType\":%s,\"arguments\":[%s]}}".format( + "{\"exprType\":\"IS_NULL\",\"returnType\":%s,\"arguments\":[%s]}".format( sparkTypeToOmniExpJsonType(BooleanType), rewriteToOmniJsonExpressionLiteral(isNull.child, exprsIndexMap)) @@ -551,20 +552,13 @@ object OmniExpressionAdaptor extends Logging { case coalesce: Coalesce => if (coalesce.children.length > 2) { - throw new UnsupportedOperationException(s"Unsupported expression: $expr") + throw new UnsupportedOperationException(s"Number of parameters is ${coalesce.children.length}. Exceeds the maximum number of parameters, coalesce only supports up to 2 parameters") } "{\"exprType\":\"COALESCE\",\"returnType\":%s, \"value1\":%s,\"value2\":%s}".format( sparkTypeToOmniExpJsonType(coalesce.dataType), rewriteToOmniJsonExpressionLiteral(coalesce.children(0), exprsIndexMap), rewriteToOmniJsonExpressionLiteral(coalesce.children(1), exprsIndexMap)) - case like: Like => - ("{\"exprType\":\"FUNCTION\",\"returnType\":%s," + - "\"function_name\":\"LIKE\", \"arguments\":[%s, %s]}") - .format(sparkTypeToOmniExpJsonType(like.dataType), - rewriteToOmniJsonExpressionLiteral(like.left, exprsIndexMap), - rewriteToOmniJsonExpressionLiteral(like.right, exprsIndexMap)) - case concat: Concat => getConcatJsonStr(concat, exprsIndexMap) case attr: Attribute => toOmniJsonAttribute(attr, exprsIndexMap(attr.exprId)) @@ -799,4 +793,139 @@ object OmniExpressionAdaptor extends Logging { } exprStrRes } + + def procLikeExpression(likeExpr: Expression, + exprsIndexMap: Map[ExprId, Int]): String = { + likeExpr match { + case like: Like => + val dataType = like.right.dataType + like.right match { + case literal: Literal => + ("{\"exprType\":\"FUNCTION\",\"returnType\":%s," + + "\"function_name\":\"LIKE\", \"arguments\":[%s, %s]}") + .format(sparkTypeToOmniExpJsonType(like.dataType), + rewriteToOmniJsonExpressionLiteral(like.left, exprsIndexMap), + generateLikeArg(literal,"")) + case _ => + throw new UnsupportedOperationException(s"Unsupported datatype in like expression: $dataType") + } + case startsWith: StartsWith => + val dataType = startsWith.right.dataType + startsWith.right match { + case literal: Literal => + ("{\"exprType\":\"FUNCTION\",\"returnType\":%s," + + "\"function_name\":\"LIKE\", \"arguments\":[%s, %s]}") + .format(sparkTypeToOmniExpJsonType(startsWith.dataType), + rewriteToOmniJsonExpressionLiteral(startsWith.left, exprsIndexMap), + generateLikeArg(literal, "startsWith")) + case _ => + throw new UnsupportedOperationException(s"Unsupported datatype in like expression: $dataType") + } + case endsWith: EndsWith => + val dataType = endsWith.right.dataType + endsWith.right match { + case literal: Literal => + ("{\"exprType\":\"FUNCTION\",\"returnType\":%s," + + "\"function_name\":\"LIKE\", \"arguments\":[%s, %s]}") + .format(sparkTypeToOmniExpJsonType(endsWith.dataType), + rewriteToOmniJsonExpressionLiteral(endsWith.left, exprsIndexMap), + generateLikeArg(literal, "endsWith")) + case _ => + throw new UnsupportedOperationException(s"Unsupported datatype in like expression: $dataType") + } + case contains: Contains => + val dataType = contains.right.dataType + contains.right match { + case literal: Literal => + ("{\"exprType\":\"FUNCTION\",\"returnType\":%s," + + "\"function_name\":\"LIKE\", \"arguments\":[%s, %s]}") + .format(sparkTypeToOmniExpJsonType(contains.dataType), + rewriteToOmniJsonExpressionLiteral(contains.left, exprsIndexMap), + generateLikeArg(literal, "contains")) + case _ => + throw new UnsupportedOperationException(s"Unsupported datatype in like expression: $dataType") + } + } + } + + def generateLikeArg(literal: Literal, exprFormat: String) : String = { + val value = literal.value + if (value == null) { + return "{\"exprType\":\"LITERAL\",\"dataType\":%s,\"isNull\":%b}".format(sparkTypeToOmniExpJsonType(literal.dataType), true) + } + var inputValue = value.toString + exprFormat match { + case "startsWith" => + inputValue = inputValue + "%" + case "endsWith" => + inputValue = "%" + inputValue + case "contains" => + inputValue = "%" + inputValue + "%" + case _ => + inputValue = value.toString + } + + val omniType = sparkTypeToOmniExpType(literal.dataType) + literal.dataType match { + case StringType => + val likeRegExpr = generateLikeRegExpr(inputValue) + ("{\"exprType\":\"LITERAL\",\"dataType\":%s," + + "\"isNull\":%b, \"value\":\"%s\",\"width\":%d}") + .format(omniType, false, likeRegExpr, likeRegExpr.length) + case dt: DecimalType => + toOmniJsonLiteral(literal) + case _ => + toOmniJsonLiteral(literal) + } + } + + def generateLikeRegExpr(value : String) : String = { + val regexString = new mutable.StringBuilder + regexString.append('^') + val valueArr = value.toCharArray + for (i <- 0 until valueArr.length) { + valueArr(i) match { + case '%' => + if (i - 1 < 0 || valueArr(i - 1) != '\\') { + regexString.append(".*") + } else { + regexString.append(valueArr(i)) + } + + case '_' => + if (i - 1 < 0 || valueArr(i - 1) != '\\') { + regexString.append(".") + } else { + regexString.append(valueArr(i)) + } + + case '\\' => + regexString.append("\\") + regexString.append(valueArr(i)) + + case '^' => + regexString.append("\\") + regexString.append(valueArr(i)) + + case '$' => + regexString.append("\\") + regexString.append(valueArr(i)) + + case '.' => + regexString.append("\\") + regexString.append(valueArr(i)) + + case '*' => + regexString.append("\\") + regexString.append(valueArr(i)) + + case _ => + regexString.append(valueArr(i)) + + } + } + regexString.append('$') + regexString.toString() + } + } diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala index 76c8cfc45..2642cab0c 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala @@ -18,7 +18,7 @@ package com.huawei.boostkit.spark.expression -import com.huawei.boostkit.spark.expression.OmniExpressionAdaptor.{getExprIdMap, rewriteToOmniExpressionLiteral, rewriteToOmniJsonExpressionLiteral} +import com.huawei.boostkit.spark.expression.OmniExpressionAdaptor.{getExprIdMap, procCaseWhenExpression, procLikeExpression, rewriteToOmniExpressionLiteral, rewriteToOmniJsonExpressionLiteral} import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.{Average, Max, Min, Sum} @@ -268,4 +268,113 @@ class OmniExpressionAdaptorSuite extends SparkFunSuite { s"running value:$runResult") } } + + test("json expression rewrite support Chinese") { + val cnAttribute = Seq(AttributeReference("char_1", StringType)(), AttributeReference("char_20", StringType)(), + AttributeReference("varchar_1", StringType)(), AttributeReference("varchar_20", StringType)()) + + val like = Like(cnAttribute(2), Literal("我_"), '\\'); + val likeResult = procLikeExpression(like, getExprIdMap(cnAttribute)) + val likeExp = "{\"exprType\":\"FUNCTION\",\"returnType\":4,\"function_name\":\"LIKE\", \"arguments\":[{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":2000}, {\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"^我.$\",\"width\":4}]}" + if (!likeExp.equals(likeResult)) { + fail(s"expression($like) not match with expected value:$likeExp," + + s"running value:$likeResult") + } + + val startsWith = StartsWith(cnAttribute(2), Literal("我")); + val startsWithResult = procLikeExpression(startsWith, getExprIdMap(cnAttribute)) + val startsWithExp = "{\"exprType\":\"FUNCTION\",\"returnType\":4,\"function_name\":\"LIKE\", \"arguments\":[{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":2000}, {\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"^我.*$\",\"width\":5}]}" + if (!startsWithExp.equals(startsWithResult)) { + fail(s"expression($startsWith) not match with expected value:$startsWithExp," + + s"running value:$startsWithResult") + } + + val endsWith = EndsWith(cnAttribute(2), Literal("我")); + val endsWithResult = procLikeExpression(endsWith, getExprIdMap(cnAttribute)) + val endsWithExp = "{\"exprType\":\"FUNCTION\",\"returnType\":4,\"function_name\":\"LIKE\", \"arguments\":[{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":2000}, {\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"^.*我$\",\"width\":5}]}" + if (!endsWithExp.equals(endsWithResult)) { + fail(s"expression($endsWith) not match with expected value:$endsWithExp," + + s"running value:$endsWithResult") + } + + val contains = Contains(cnAttribute(2), Literal("我")); + val containsResult = procLikeExpression(contains, getExprIdMap(cnAttribute)) + val containsExp = "{\"exprType\":\"FUNCTION\",\"returnType\":4,\"function_name\":\"LIKE\", \"arguments\":[{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":2000}, {\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"^.*我.*$\",\"width\":7}]}" + if (!containsExp.equals(containsResult)) { + fail(s"expression($contains) not match with expected value:$containsExp," + + s"running value:$containsResult") + } + + val t1 = new Tuple2(Not(EqualTo(cnAttribute(0), Literal("新"))), Not(EqualTo(cnAttribute(1), Literal("官方爸爸")))) + val t2 = new Tuple2(Not(EqualTo(cnAttribute(2), Literal("爱你三千遍"))), Not(EqualTo(cnAttribute(2), Literal("新")))) + val branch = Seq(t1, t2) + val elseValue = Some(Not(EqualTo(cnAttribute(3), Literal("啊水水水水")))) + val caseWhen = CaseWhen(branch, elseValue); + val caseWhenResult = rewriteToOmniJsonExpressionLiteral(caseWhen, getExprIdMap(cnAttribute)) + val caseWhenExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":0,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"新\",\"width\":1}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":1,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"官方爸爸\",\"width\":4}}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"爱你三千遍\",\"width\":5}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"新\",\"width\":1}}},\"if_false\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":3,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"啊水水水水\",\"width\":5}}}}}" + if (!caseWhenExp.equals(caseWhenResult)) { + fail(s"expression($caseWhen) not match with expected value:$caseWhenExp," + + s"running value:$caseWhenResult") + } + + val isNull = IsNull(cnAttribute(0)); + val isNullResult = rewriteToOmniJsonExpressionLiteral(isNull, getExprIdMap(cnAttribute)) + val isNullExp = "{\"exprType\":\"IS_NULL\",\"returnType\":4,\"arguments\":[{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":0,\"width\":2000}]}" + if (!isNullExp.equals(isNullResult)) { + fail(s"expression($isNull) not match with expected value:$isNullExp," + + s"running value:$isNullResult") + } + + val children = Seq(cnAttribute(0), cnAttribute(1)) + val coalesce = Coalesce(children); + val coalesceResult = rewriteToOmniJsonExpressionLiteral(coalesce, getExprIdMap(cnAttribute)) + val coalesceExp = "{\"exprType\":\"COALESCE\",\"returnType\":15,\"width\":2000, \"value1\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":0,\"width\":2000},\"value2\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":1,\"width\":2000}}" + if (!coalesceExp.equals(coalesceResult)) { + fail(s"expression($coalesce) not match with expected value:$coalesceExp," + + s"running value:$coalesceResult") + } + + val children2 = Seq(cnAttribute(0), cnAttribute(1), cnAttribute(2)) + val coalesce2 = Coalesce(children2); + try { + rewriteToOmniJsonExpressionLiteral(coalesce2, getExprIdMap(cnAttribute)) + } catch { + case ex: UnsupportedOperationException => { + println(ex) + } + } + + } + + test("procCaseWhenExpression") { + val caseWhenAttribute = Seq(AttributeReference("char_1", StringType)(), AttributeReference("char_20", StringType)(), + AttributeReference("varchar_1", StringType)(), AttributeReference("varchar_20", StringType)(), + AttributeReference("a", IntegerType)(), AttributeReference("b", IntegerType)()) + + val t1 = new Tuple2(Not(EqualTo(caseWhenAttribute(0), Literal("新"))), Not(EqualTo(caseWhenAttribute(1), Literal("官方爸爸")))) + val t2 = new Tuple2(Not(EqualTo(caseWhenAttribute(2), Literal("爱你三千遍"))), Not(EqualTo(caseWhenAttribute(2), Literal("新")))) + val branch = Seq(t1, t2) + val elseValue = Some(Not(EqualTo(caseWhenAttribute(3), Literal("啊水水水水")))) + val expression = CaseWhen(branch, elseValue); + val runResult = procCaseWhenExpression(expression, getExprIdMap(caseWhenAttribute)) + val filterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":0,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"新\",\"width\":1}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":1,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"官方爸爸\",\"width\":4}}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"爱你三千遍\",\"width\":5}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"新\",\"width\":1}}},\"if_false\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":3,\"width\":2000},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false, \"value\":\"啊水水水水\",\"width\":5}}}}}" + if (!filterExp.equals(runResult)) { + fail(s"expression($expression) not match with expected value:$filterExp," + + s"running value:$runResult") + } + + val t3 = new Tuple2(Not(EqualTo(caseWhenAttribute(4), Literal(5))), Not(EqualTo(caseWhenAttribute(5), Literal(10)))) + val t4 = new Tuple2(LessThan(caseWhenAttribute(4), Literal(15)), GreaterThan(caseWhenAttribute(5), Literal(20))) + val branch2 = Seq(t3, t4) + val elseValue2 = Some(Not(EqualTo(caseWhenAttribute(5), Literal(25)))) + val numExpression = CaseWhen(branch2, elseValue2); + val numResult = procCaseWhenExpression(numExpression, getExprIdMap(caseWhenAttribute)) + val numFilterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":5}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":10}}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"LESS_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":15}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"GREATER_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":20}},\"if_false\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":25}}}}}" + if (!numFilterExp.equals(numResult)) { + fail(s"expression($numExpression) not match with expected value:$numFilterExp," + + s"running value:$numResult") + } + } + + } -- Gitee From 1350705d4707c2883852fd5c7611f085e93eeca0 Mon Sep 17 00:00:00 2001 From: l30037350 Date: Thu, 15 Sep 2022 17:10:13 +0800 Subject: [PATCH 03/10] caseWhen else branch modify --- .../spark/expression/OmniExpressionAdaptor.scala | 4 ++++ .../expression/OmniExpressionAdaptorSuite.scala | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index 8f3596fd2..fbba978f3 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -801,6 +801,10 @@ object OmniExpressionAdaptor extends Logging { if (i != caseWhen.branches.length - 1) { ifFalseStr = exprStr } else { + var elseValue = caseWhen.elseValue + if (elseValue.isEmpty) { + elseValue = Some(Literal(false)) + } ifFalseStr = rewriteToOmniJsonExpressionLiteral(caseWhen.elseValue.get, exprsIndexMap) } exprStrRes = exprStrRes.format(sparkTypeToOmniExpJsonType(caseWhen.dataType), diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala index 2642cab0c..8e5ba2917 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala @@ -374,6 +374,18 @@ class OmniExpressionAdaptorSuite extends SparkFunSuite { fail(s"expression($numExpression) not match with expected value:$numFilterExp," + s"running value:$numResult") } + + val t5 = new Tuple2(Not(EqualTo(caseWhenAttribute(4), Literal(5))), Not(EqualTo(caseWhenAttribute(5), Literal(10)))) + val t6 = new Tuple2(LessThan(caseWhenAttribute(4), Literal(15)), GreaterThan(caseWhenAttribute(5), Literal(20))) + val branch3 = Seq(t5, t6) + val elseValue3 = None + val noneExpression = CaseWhen(branch3, elseValue3); + val noneResult = procCaseWhenExpression(noneExpression, getExprIdMap(caseWhenAttribute)) + val noneFilterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":5}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":10}}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"LESS_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":15}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"GREATER_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":20}},\"if_false\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":25}}}}}" + if (!noneFilterExp.equals(noneResult)) { + fail(s"expression($noneExpression) not match with expected value:$noneFilterExp," + + s"running value:$noneResult") + } } -- Gitee From 0a12f6fc4d8253fc1805d6b93bbb6ab99381d4e9 Mon Sep 17 00:00:00 2001 From: l30037350 Date: Thu, 15 Sep 2022 17:18:50 +0800 Subject: [PATCH 04/10] caseWhen else branch modify --- .../boostkit/spark/expression/OmniExpressionAdaptor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index fbba978f3..80de9851d 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -805,7 +805,7 @@ object OmniExpressionAdaptor extends Logging { if (elseValue.isEmpty) { elseValue = Some(Literal(false)) } - ifFalseStr = rewriteToOmniJsonExpressionLiteral(caseWhen.elseValue.get, exprsIndexMap) + ifFalseStr = rewriteToOmniJsonExpressionLiteral(elseValue.get, exprsIndexMap) } exprStrRes = exprStrRes.format(sparkTypeToOmniExpJsonType(caseWhen.dataType), rewriteToOmniJsonExpressionLiteral(caseWhen.branches(i)._1, exprsIndexMap), -- Gitee From b45e790cad769e9b2e369deaa0a2b92598ed3006 Mon Sep 17 00:00:00 2001 From: l30037350 Date: Thu, 15 Sep 2022 17:40:10 +0800 Subject: [PATCH 05/10] caseWhen else branch modify --- .../boostkit/spark/expression/OmniExpressionAdaptor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index 80de9851d..9fc8d313c 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -803,7 +803,7 @@ object OmniExpressionAdaptor extends Logging { } else { var elseValue = caseWhen.elseValue if (elseValue.isEmpty) { - elseValue = Some(Literal(false)) + elseValue = Some(Literal(null)) } ifFalseStr = rewriteToOmniJsonExpressionLiteral(elseValue.get, exprsIndexMap) } -- Gitee From a9f1c2a6c7f4bee43b875634ccda990eff44fa52 Mon Sep 17 00:00:00 2001 From: l30037350 Date: Thu, 15 Sep 2022 17:52:17 +0800 Subject: [PATCH 06/10] caseWhen else branch modify --- .../boostkit/spark/expression/OmniExpressionAdaptor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index 9fc8d313c..822793013 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -803,7 +803,7 @@ object OmniExpressionAdaptor extends Logging { } else { var elseValue = caseWhen.elseValue if (elseValue.isEmpty) { - elseValue = Some(Literal(null)) + elseValue = Some(Literal(null, IntegerType)) } ifFalseStr = rewriteToOmniJsonExpressionLiteral(elseValue.get, exprsIndexMap) } -- Gitee From 8c8525a048d662289292c70870d294b7a484716e Mon Sep 17 00:00:00 2001 From: l30037350 Date: Thu, 15 Sep 2022 18:01:23 +0800 Subject: [PATCH 07/10] caseWhen else branch modify --- .../boostkit/spark/expression/OmniExpressionAdaptor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index 822793013..cddc59fa7 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -803,7 +803,7 @@ object OmniExpressionAdaptor extends Logging { } else { var elseValue = caseWhen.elseValue if (elseValue.isEmpty) { - elseValue = Some(Literal(null, IntegerType)) + elseValue = Some(Literal(null, caseWhen.dataType)) } ifFalseStr = rewriteToOmniJsonExpressionLiteral(elseValue.get, exprsIndexMap) } -- Gitee From 17ba612d27366ca0d5fd34e434fbb17fda583f8d Mon Sep 17 00:00:00 2001 From: l30037350 Date: Thu, 15 Sep 2022 18:10:51 +0800 Subject: [PATCH 08/10] caseWhen else branch modify UT --- .../spark/expression/OmniExpressionAdaptorSuite.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala index 8e5ba2917..0b5a09766 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala @@ -382,10 +382,7 @@ class OmniExpressionAdaptorSuite extends SparkFunSuite { val noneExpression = CaseWhen(branch3, elseValue3); val noneResult = procCaseWhenExpression(noneExpression, getExprIdMap(caseWhenAttribute)) val noneFilterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":5}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":10}}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"LESS_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":15}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"GREATER_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":20}},\"if_false\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":25}}}}}" - if (!noneFilterExp.equals(noneResult)) { - fail(s"expression($noneExpression) not match with expected value:$noneFilterExp," + - s"running value:$noneResult") - } + } -- Gitee From 0119b06a4550a3f2080aacba99eb9e090b60bab8 Mon Sep 17 00:00:00 2001 From: l30037350 Date: Thu, 15 Sep 2022 19:59:05 +0800 Subject: [PATCH 09/10] caseWhen elseValue None UT --- .../spark/expression/OmniExpressionAdaptorSuite.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala index 0b5a09766..0cc6234c0 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala @@ -381,8 +381,11 @@ class OmniExpressionAdaptorSuite extends SparkFunSuite { val elseValue3 = None val noneExpression = CaseWhen(branch3, elseValue3); val noneResult = procCaseWhenExpression(noneExpression, getExprIdMap(caseWhenAttribute)) - val noneFilterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":5}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":10}}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"LESS_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":15}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"GREATER_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":20}},\"if_false\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":25}}}}}" - + val noneFilterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":5}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":10}}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"LESS_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":15}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"GREATER_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":20}},\"if_false\":{\"exprType\": \"LITERAL\",\"dataType\": 4,\"isNull\": true}}}" + if (!noneFilterExp.equals(noneResult)) { + fail(s"expression($noneExpression) not match with expected value:$noneFilterExp," + + s"running value:$noneResult") + } } -- Gitee From 55d8a09005f168e5ad5c2d0d5457aafb27958c87 Mon Sep 17 00:00:00 2001 From: l30037350 Date: Thu, 15 Sep 2022 20:25:36 +0800 Subject: [PATCH 10/10] caseWhen elseValue None UT --- .../boostkit/spark/expression/OmniExpressionAdaptorSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala index 0cc6234c0..dae771f6d 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala @@ -381,7 +381,7 @@ class OmniExpressionAdaptorSuite extends SparkFunSuite { val elseValue3 = None val noneExpression = CaseWhen(branch3, elseValue3); val noneResult = procCaseWhenExpression(noneExpression, getExprIdMap(caseWhenAttribute)) - val noneFilterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":5}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":10}}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"LESS_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":15}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"GREATER_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":20}},\"if_false\":{\"exprType\": \"LITERAL\",\"dataType\": 4,\"isNull\": true}}}" + val noneFilterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":5}}},\"if_true\":{\"exprType\":\"UNARY\",\"returnType\":4,\"operator\":\"not\",\"expr\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":10}}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"LESS_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":15}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"GREATER_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1, \"isNull\":false, \"value\":20}},\"if_false\":{\"exprType\":\"LITERAL\",\"dataType\":4,\"isNull\":true}}}" if (!noneFilterExp.equals(noneResult)) { fail(s"expression($noneExpression) not match with expected value:$noneFilterExp," + s"running value:$noneResult") -- Gitee