diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala index 3221f3ec5af8788d1a275e7df2204b727e7d796c..4653fc55078260cb77beaae72610f57f775c763c 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptor.scala @@ -104,26 +104,22 @@ object OmniExpressionAdaptor extends Logging { } } - - def rewriteToOmniJsonExpressionLiteral(expr: Expression, - exprsIndexMap: Map[ExprId, Int]): String = { - rewriteToOmniJsonExpressionLiteral(expr, exprsIndexMap, expr.dataType) - } - def rewriteToOmniJsonExpressionLiteral(expr: Expression, exprsIndexMap: Map[ExprId, Int], - returnDatatype: DataType): String = { - rewriteToOmniJsonExpressionLiteralJsonObject(expr, exprsIndexMap, returnDatatype).toString + buildCheck: Boolean = false): String = { + rewriteToOmniJsonExpressionLiteralJsonObject(expr, exprsIndexMap, expr.dataType, buildCheck).toString } private def rewriteToOmniJsonExpressionLiteralJsonObject(expr: Expression, - exprsIndexMap: Map[ExprId, Int]): JSONObject = { - rewriteToOmniJsonExpressionLiteralJsonObject(expr, exprsIndexMap, expr.dataType) + exprsIndexMap: Map[ExprId, Int], + buildCheck: Boolean): JSONObject = { + rewriteToOmniJsonExpressionLiteralJsonObject(expr, exprsIndexMap, expr.dataType, buildCheck) } private def rewriteToOmniJsonExpressionLiteralJsonObject(expr: Expression, exprsIndexMap: Map[ExprId, Int], - returnDatatype: DataType): JSONObject = { + returnDatatype: DataType, + buildCheck: Boolean): JSONObject = { expr match { case subquery: execution.ScalarSubquery => var result: Any = null @@ -145,9 +141,9 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", unscaledValue.dataType) .put("function_name", "UnscaledValue") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(unscaledValue.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(unscaledValue.child, exprsIndexMap, buildCheck))) case checkOverflow: CheckOverflow => - rewriteToOmniJsonExpressionLiteralJsonObject(checkOverflow.child, exprsIndexMap, returnDatatype) + rewriteToOmniJsonExpressionLiteralJsonObject(checkOverflow.child, exprsIndexMap, returnDatatype, buildCheck) case makeDecimal: MakeDecimal => makeDecimal.child.dataType match { @@ -155,104 +151,104 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", makeDecimal.dataType) .put("function_name", "MakeDecimal") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(makeDecimal.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(makeDecimal.child, exprsIndexMap, buildCheck))) case longChild: LongType => new JSONObject().put("exprType", "FUNCTION") .put("function_name", "MakeDecimal") .addOmniExpJsonType("returnType", makeDecimal.dataType) - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(makeDecimal.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(makeDecimal.child, exprsIndexMap, buildCheck))) case _ => throw new UnsupportedOperationException(s"Unsupported datatype for MakeDecimal: ${makeDecimal.child.dataType}") } case promotePrecision: PromotePrecision => - rewriteToOmniJsonExpressionLiteralJsonObject(promotePrecision.child, exprsIndexMap) + rewriteToOmniJsonExpressionLiteralJsonObject(promotePrecision.child, exprsIndexMap, buildCheck) case sub: Subtract => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", returnDatatype) .put("operator", "SUBTRACT") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(sub.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(sub.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(sub.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(sub.right, exprsIndexMap, buildCheck)) case add: Add => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", returnDatatype) .put("operator", "ADD") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(add.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(add.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(add.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(add.right, exprsIndexMap, buildCheck)) case mult: Multiply => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", returnDatatype) .put("operator", "MULTIPLY") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(mult.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(mult.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(mult.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(mult.right, exprsIndexMap, buildCheck)) case divide: Divide => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", returnDatatype) .put("operator", "DIVIDE") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(divide.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(divide.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(divide.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(divide.right, exprsIndexMap, buildCheck)) case mod: Remainder => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", returnDatatype) .put("operator", "MODULUS") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(mod.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(mod.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(mod.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(mod.right, exprsIndexMap, buildCheck)) case greaterThan: GreaterThan => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", greaterThan.dataType) .put("operator", "GREATER_THAN") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(greaterThan.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(greaterThan.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(greaterThan.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(greaterThan.right, exprsIndexMap, buildCheck)) case greaterThanOrEq: GreaterThanOrEqual => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", greaterThanOrEq.dataType) .put("operator", "GREATER_THAN_OR_EQUAL") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(greaterThanOrEq.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(greaterThanOrEq.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(greaterThanOrEq.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(greaterThanOrEq.right, exprsIndexMap, buildCheck)) case lessThan: LessThan => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", lessThan.dataType) .put("operator", "LESS_THAN") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(lessThan.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(lessThan.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(lessThan.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(lessThan.right, exprsIndexMap, buildCheck)) case lessThanOrEq: LessThanOrEqual => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", lessThanOrEq.dataType) .put("operator", "LESS_THAN_OR_EQUAL") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(lessThanOrEq.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(lessThanOrEq.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(lessThanOrEq.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(lessThanOrEq.right, exprsIndexMap, buildCheck)) case equal: EqualTo => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", equal.dataType) .put("operator", "EQUAL") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(equal.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(equal.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(equal.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(equal.right, exprsIndexMap, buildCheck)) case or: Or => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", or.dataType) .put("operator", "OR") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(or.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(or.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(or.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(or.right, exprsIndexMap, buildCheck)) case and: And => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", and.dataType) .put("operator", "AND") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(and.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(and.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(and.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(and.right, exprsIndexMap, buildCheck)) - case alias: Alias => rewriteToOmniJsonExpressionLiteralJsonObject(alias.child, exprsIndexMap) + case alias: Alias => rewriteToOmniJsonExpressionLiteralJsonObject(alias.child, exprsIndexMap, buildCheck) case literal: Literal => toOmniJsonLiteral(literal) case not: Not => not.child match { @@ -260,20 +256,20 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "UNARY") .addOmniExpJsonType("returnType", BooleanType) .put("operator", "not") - .put("expr", rewriteToOmniJsonExpressionLiteralJsonObject(isnull, exprsIndexMap)) + .put("expr", rewriteToOmniJsonExpressionLiteralJsonObject(isnull, exprsIndexMap, buildCheck)) case equal: EqualTo => new JSONObject().put("exprType", "BINARY") .addOmniExpJsonType("returnType", equal.dataType) .put("operator", "NOT_EQUAL") - .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(equal.left, exprsIndexMap)) - .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(equal.right, exprsIndexMap)) + .put("left", rewriteToOmniJsonExpressionLiteralJsonObject(equal.left, exprsIndexMap, buildCheck)) + .put("right", rewriteToOmniJsonExpressionLiteralJsonObject(equal.right, exprsIndexMap, buildCheck)) case _ => new JSONObject().put("exprType", "UNARY") .addOmniExpJsonType("returnType", BooleanType) .put("operator", "not") - .put("expr", rewriteToOmniJsonExpressionLiteralJsonObject(not.child, exprsIndexMap)) + .put("expr", rewriteToOmniJsonExpressionLiteralJsonObject(not.child, exprsIndexMap, buildCheck)) } case isnotnull: IsNotNull => @@ -283,11 +279,11 @@ object OmniExpressionAdaptor extends Logging { .put("expr", new JSONObject() .put("exprType", "IS_NULL") .addOmniExpJsonType("returnType", BooleanType) - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(isnotnull.child, exprsIndexMap)))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(isnotnull.child, exprsIndexMap, buildCheck)))) case isNull: IsNull => new JSONObject().put("exprType", "IS_NULL") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(isNull.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(isNull.child, exprsIndexMap, buildCheck))) .addOmniExpJsonType("returnType", BooleanType) // Substring @@ -295,9 +291,9 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", subString.dataType) .put("function_name", "substr") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(subString.str, exprsIndexMap)). - put(rewriteToOmniJsonExpressionLiteralJsonObject(subString.pos, exprsIndexMap)) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(subString.len, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(subString.str, exprsIndexMap, buildCheck)). + put(rewriteToOmniJsonExpressionLiteralJsonObject(subString.pos, exprsIndexMap, buildCheck)) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(subString.len, exprsIndexMap, buildCheck))) // Cast case cast: CastBase => @@ -308,13 +304,13 @@ object OmniExpressionAdaptor extends Logging { .addOmniExpJsonType("returnType", cast.dataType) .put("width", 50) .put("function_name", "CAST") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(cast.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(cast.child, exprsIndexMap, buildCheck))) case _ => new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", cast.dataType) .put("function_name", "CAST") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(cast.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(cast.child, exprsIndexMap, buildCheck))) } // Abs @@ -322,45 +318,45 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", abs.dataType) .put("function_name", "abs") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(abs.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(abs.child, exprsIndexMap, buildCheck))) case lower: Lower => new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", lower.dataType) .put("function_name", "lower") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(lower.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(lower.child, exprsIndexMap, buildCheck))) case upper: Upper => new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", upper.dataType) .put("function_name", "upper") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(upper.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(upper.child, exprsIndexMap, buildCheck))) case length: Length => new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", length.dataType) .put("function_name", "length") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(length.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(length.child, exprsIndexMap, buildCheck))) case replace: StringReplace => new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", replace.dataType) .put("function_name", "replace") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(replace.srcExpr, exprsIndexMap)) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(replace.searchExpr, exprsIndexMap)) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(replace.replaceExpr, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(replace.srcExpr, exprsIndexMap, buildCheck)) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(replace.searchExpr, exprsIndexMap, buildCheck)) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(replace.replaceExpr, exprsIndexMap, buildCheck))) // In case in: In => new JSONObject().put("exprType", "IN") .addOmniExpJsonType("returnType", in.dataType) - .put("arguments", new JSONArray(in.children.map(child => rewriteToOmniJsonExpressionLiteralJsonObject(child, exprsIndexMap)).toArray)) + .put("arguments", new JSONArray(in.children.map(child => rewriteToOmniJsonExpressionLiteralJsonObject(child, exprsIndexMap, buildCheck)).toArray)) // coming from In expression with optimizerInSetConversionThreshold case inSet: InSet => val jSONObject = new JSONObject().put("exprType", "IN") .addOmniExpJsonType("returnType", inSet.dataType) - val jsonArray = new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(inSet.child, exprsIndexMap)) + val jsonArray = new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(inSet.child, exprsIndexMap, buildCheck)) inSet.set.foreach(child => jsonArray.put(toOmniJsonLiteral(Literal(child, inSet.child.dataType)))) jSONObject.put("arguments", jsonArray) jSONObject @@ -368,12 +364,12 @@ object OmniExpressionAdaptor extends Logging { case ifExp: If => new JSONObject().put("exprType", "IF") .addOmniExpJsonType("returnType", ifExp.dataType) - .put("condition", rewriteToOmniJsonExpressionLiteralJsonObject(ifExp.predicate, exprsIndexMap)) - .put("if_true", rewriteToOmniJsonExpressionLiteralJsonObject(ifExp.trueValue, exprsIndexMap)) - .put("if_false", rewriteToOmniJsonExpressionLiteralJsonObject(ifExp.falseValue, exprsIndexMap)) + .put("condition", rewriteToOmniJsonExpressionLiteralJsonObject(ifExp.predicate, exprsIndexMap, buildCheck)) + .put("if_true", rewriteToOmniJsonExpressionLiteralJsonObject(ifExp.trueValue, exprsIndexMap, buildCheck)) + .put("if_false", rewriteToOmniJsonExpressionLiteralJsonObject(ifExp.falseValue, exprsIndexMap, buildCheck)) case caseWhen: CaseWhen => - procCaseWhenExpression(caseWhen, exprsIndexMap) + procCaseWhenExpression(caseWhen, exprsIndexMap, buildCheck) case coalesce: Coalesce => if (coalesce.children.length > 2) { @@ -381,17 +377,17 @@ object OmniExpressionAdaptor extends Logging { } new JSONObject().put("exprType", "COALESCE") .addOmniExpJsonType("returnType", coalesce.dataType) - .put("value1", rewriteToOmniJsonExpressionLiteralJsonObject(coalesce.children.head, exprsIndexMap)) - .put("value2", rewriteToOmniJsonExpressionLiteralJsonObject(coalesce.children(1), exprsIndexMap)) + .put("value1", rewriteToOmniJsonExpressionLiteralJsonObject(coalesce.children.head, exprsIndexMap, buildCheck)) + .put("value2", rewriteToOmniJsonExpressionLiteralJsonObject(coalesce.children(1), exprsIndexMap, buildCheck)) case concat: Concat => - getConcatJsonStr(concat, exprsIndexMap) + getConcatJsonStr(concat, exprsIndexMap, buildCheck) case round: Round => new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", round.dataType) .put("function_name", "round") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(round.child, exprsIndexMap)) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(round.scale, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(round.child, exprsIndexMap, buildCheck)) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(round.scale, exprsIndexMap, buildCheck))) case attr: Attribute => toOmniJsonAttribute(attr, exprsIndexMap(attr.exprId)) @@ -399,18 +395,18 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", inStr.dataType) .put("function_name", "instr") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(inStr.str, exprsIndexMap)) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(inStr.substr, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(inStr.str, exprsIndexMap, buildCheck)) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(inStr.substr, exprsIndexMap, buildCheck))) // for floating numbers normalize case normalizeNaNAndZero: NormalizeNaNAndZero => new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", normalizeNaNAndZero.dataType) .put("function_name", "NormalizeNaNAndZero") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(normalizeNaNAndZero.child, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(normalizeNaNAndZero.child, exprsIndexMap, buildCheck))) case knownFloatingPointNormalized: KnownFloatingPointNormalized => - rewriteToOmniJsonExpressionLiteralJsonObject(knownFloatingPointNormalized.child, exprsIndexMap) + rewriteToOmniJsonExpressionLiteralJsonObject(knownFloatingPointNormalized.child, exprsIndexMap, buildCheck) // for like case startsWith: StartsWith => @@ -419,8 +415,8 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", startsWith.dataType) .put("function_name", "StartsWith") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(startsWith.left, exprsIndexMap)) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(startsWith.right, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(startsWith.left, exprsIndexMap, buildCheck)) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(startsWith.right, exprsIndexMap, buildCheck))) case _ => throw new UnsupportedOperationException(s"Unsupported right expression in like expression: $startsWith") @@ -431,8 +427,8 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", endsWith.dataType) .put("function_name", "EndsWith") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(endsWith.left, exprsIndexMap)) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(endsWith.right, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(endsWith.left, exprsIndexMap, buildCheck)) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(endsWith.right, exprsIndexMap, buildCheck))) case _ => throw new UnsupportedOperationException(s"Unsupported right expression in like expression: $endsWith") @@ -442,8 +438,8 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", truncDate.dataType) .put("function_name", "trunc_date") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(truncDate.left, exprsIndexMap)) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(truncDate.right, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(truncDate.left, exprsIndexMap, buildCheck)) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(truncDate.right, exprsIndexMap, buildCheck))) case md5: Md5 => md5.child match { @@ -453,7 +449,7 @@ object OmniExpressionAdaptor extends Logging { new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", md5.dataType) .put("function_name", "Md5") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(inputExpression, exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(inputExpression, exprsIndexMap, buildCheck))) } } @@ -465,21 +461,22 @@ object OmniExpressionAdaptor extends Logging { return new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", hiveUdf.dataType) .put("function_name", udfName) - .put("arguments", getJsonExprArgumentsByChildren(hiveUdf.children, exprsIndexMap)) + .put("arguments", getJsonExprArgumentsByChildren(hiveUdf.children, exprsIndexMap, buildCheck)) } throw new UnsupportedOperationException(s"Unsupported expression: $expr") } } private def getJsonExprArgumentsByChildren(children: Seq[Expression], - exprsIndexMap: Map[ExprId, Int]): JSONArray = { + exprsIndexMap: Map[ExprId, Int], + buildCheck: Boolean): JSONArray = { val size = children.size val jsonArray = new JSONArray() if (size == 0) { return jsonArray } for (i <- 0 until size) { - jsonArray.put(rewriteToOmniJsonExpressionLiteralJsonObject(children(i), exprsIndexMap)) + jsonArray.put(rewriteToOmniJsonExpressionLiteralJsonObject(children(i), exprsIndexMap, buildCheck)) } jsonArray } @@ -493,22 +490,22 @@ object OmniExpressionAdaptor extends Logging { } } - private def getConcatJsonStr(concat: Concat, exprsIndexMap: Map[ExprId, Int]): JSONObject = { + private def getConcatJsonStr(concat: Concat, exprsIndexMap: Map[ExprId, Int], buildCheck: Boolean): JSONObject = { val children: Seq[Expression] = concat.children checkInputDataTypes(children) if (children.length == 1) { - return rewriteToOmniJsonExpressionLiteralJsonObject(children.head, exprsIndexMap) + return rewriteToOmniJsonExpressionLiteralJsonObject(children.head, exprsIndexMap, buildCheck) } val res = new JSONObject().put("exprType", "FUNCTION") .addOmniExpJsonType("returnType", concat.dataType) .put("function_name", "concat") - .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(children.head, exprsIndexMap)) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(children(1), exprsIndexMap))) + .put("arguments", new JSONArray().put(rewriteToOmniJsonExpressionLiteralJsonObject(children.head, exprsIndexMap, buildCheck)) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(children(1), exprsIndexMap, buildCheck))) for (i <- 2 until children.length) { val preResJson = new JSONObject(res, JSONObject.getNames(res)) res.put("arguments", new JSONArray().put(preResJson) - .put(rewriteToOmniJsonExpressionLiteralJsonObject(children(i), exprsIndexMap))) + .put(rewriteToOmniJsonExpressionLiteralJsonObject(children(i), exprsIndexMap, buildCheck))) } res } @@ -756,13 +753,13 @@ object OmniExpressionAdaptor extends Logging { } def procCaseWhenExpression(caseWhen: CaseWhen, - exprsIndexMap: Map[ExprId, Int]): JSONObject = { + exprsIndexMap: Map[ExprId, Int], buildCheck: Boolean): JSONObject = { var jsonObject = new JSONObject() for (i <- caseWhen.branches.indices.reverse) { val outerJson = new JSONObject().put("exprType", "IF") .addOmniExpJsonType("returnType", caseWhen.dataType) - .put("condition", rewriteToOmniJsonExpressionLiteralJsonObject(caseWhen.branches(i)._1, exprsIndexMap)) - .put("if_true", rewriteToOmniJsonExpressionLiteralJsonObject(caseWhen.branches(i)._2, exprsIndexMap)) + .put("condition", rewriteToOmniJsonExpressionLiteralJsonObject(caseWhen.branches(i)._1, exprsIndexMap, buildCheck)) + .put("if_true", rewriteToOmniJsonExpressionLiteralJsonObject(caseWhen.branches(i)._2, exprsIndexMap, buildCheck)) if (i != caseWhen.branches.length - 1) { val innerJson = new JSONObject(jsonObject, JSONObject.getNames(jsonObject)) @@ -772,7 +769,7 @@ object OmniExpressionAdaptor extends Logging { if (elseValue.isEmpty) { elseValue = Some(Literal(null, caseWhen.dataType)) } - outerJson.put("if_false", rewriteToOmniJsonExpressionLiteralJsonObject(elseValue.get, exprsIndexMap)) + outerJson.put("if_false", rewriteToOmniJsonExpressionLiteralJsonObject(elseValue.get, exprsIndexMap, buildCheck)) } jsonObject = outerJson } diff --git a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/ColumnarBasicPhysicalOperators.scala b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/ColumnarBasicPhysicalOperators.scala index 0553edf0ba0222731e3fdd2add2969f4ba08f8e6..5a34a6b37ca4fb72f5b4eedcbaf67d81ec2a1e9b 100644 --- a/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/ColumnarBasicPhysicalOperators.scala +++ b/omnioperator/omniop-spark-extension/java/src/main/scala/org/apache/spark/sql/execution/ColumnarBasicPhysicalOperators.scala @@ -63,7 +63,7 @@ case class ColumnarProjectExec(projectList: Seq[NamedExpression], child: SparkPl child.output.map( exp => sparkTypeToOmniType(exp.dataType, exp.metadata)).toArray val omniExpressions: Array[AnyRef] = projectList.map( - exp => rewriteToOmniJsonExpressionLiteral(exp, omniAttrExpsIdMap)).toArray + exp => rewriteToOmniJsonExpressionLiteral(exp, omniAttrExpsIdMap, true)).toArray checkOmniJsonWhiteList("", omniExpressions) } diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala index ded676538759023db54d956f4c4449ae427e76b6..2ccec63d9852165e4914e441d3863354359bc478 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/com/huawei/boostkit/spark/expression/OmniExpressionAdaptorSuite.scala @@ -248,7 +248,7 @@ class OmniExpressionAdaptorSuite extends SparkFunSuite { val branch = Seq(t1, t2) val elseValue = Some(Not(EqualTo(caseWhenAttribute(3), Literal("啊水水水水")))) val expression = CaseWhen(branch, elseValue); - val runResult = procCaseWhenExpression(expression, getExprIdMap(caseWhenAttribute)).toString() + val runResult = procCaseWhenExpression(expression, getExprIdMap(caseWhenAttribute), false).toString() val filterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":0,\"width\":50},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false,\"value\":\"新\",\"width\":1}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":1,\"width\":50},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false,\"value\":\"官方爸爸\",\"width\":4}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":50},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false,\"value\":\"爱你三千遍\",\"width\":5}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":2,\"width\":50},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false,\"value\":\"新\",\"width\":1}},\"if_false\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":15,\"colVal\":3,\"width\":50},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":15,\"isNull\":false,\"value\":\"啊水水水水\",\"width\":5}}}}" checkJsonKeyValueIgnoreKeySequence(filterExp, runResult, expression) @@ -258,7 +258,7 @@ class OmniExpressionAdaptorSuite extends SparkFunSuite { val branch2 = Seq(t3, t4) val elseValue2 = Some(Not(EqualTo(caseWhenAttribute(5), Literal(25)))) val numExpression = CaseWhen(branch2, elseValue2); - val numResult = procCaseWhenExpression(numExpression, getExprIdMap(caseWhenAttribute)).toString() + val numResult = procCaseWhenExpression(numExpression, getExprIdMap(caseWhenAttribute),false).toString() val numFilterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1,\"isNull\":false,\"value\":5}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1,\"isNull\":false,\"value\":10}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"LESS_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1,\"isNull\":false,\"value\":15}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"GREATER_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1,\"isNull\":false,\"value\":20}},\"if_false\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1,\"isNull\":false,\"value\":25}}}}" checkJsonKeyValueIgnoreKeySequence(numFilterExp, numResult, numExpression) @@ -268,7 +268,7 @@ class OmniExpressionAdaptorSuite extends SparkFunSuite { val branch3 = Seq(t5, t6) val elseValue3 = None val noneExpression = CaseWhen(branch3, elseValue3); - val noneResult = procCaseWhenExpression(noneExpression, getExprIdMap(caseWhenAttribute)).toString() + val noneResult = procCaseWhenExpression(noneExpression, getExprIdMap(caseWhenAttribute), false).toString() val noneFilterExp = "{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1,\"isNull\":false,\"value\":5}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"NOT_EQUAL\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1,\"isNull\":false,\"value\":10}},\"if_false\":{\"exprType\":\"IF\",\"returnType\":4,\"condition\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"LESS_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":4},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1,\"isNull\":false,\"value\":15}},\"if_true\":{\"exprType\":\"BINARY\",\"returnType\":4,\"operator\":\"GREATER_THAN\",\"left\":{\"exprType\":\"FIELD_REFERENCE\",\"dataType\":1,\"colVal\":5},\"right\":{\"exprType\":\"LITERAL\",\"dataType\":1,\"isNull\":false,\"value\":20}},\"if_false\":{\"exprType\":\"LITERAL\",\"dataType\":4,\"isNull\":true}}}" checkJsonKeyValueIgnoreKeySequence(noneFilterExp, noneResult, noneExpression) @@ -278,7 +278,7 @@ class OmniExpressionAdaptorSuite extends SparkFunSuite { val branch4 = Seq(t7, t8) val elseValue4 = Some(Not(EqualTo(caseWhenAttribute(3), Literal("\"\\\\t/\\b\\f\\n\\r\\t456")))) val specialCharacterExpression = CaseWhen(branch4, elseValue4); - val specialCharacterRunResult = procCaseWhenExpression(specialCharacterExpression, getExprIdMap(caseWhenAttribute)).toString() + val specialCharacterRunResult = procCaseWhenExpression(specialCharacterExpression, getExprIdMap(caseWhenAttribute), false).toString() val specialCharacterFilterExp = "{\"condition\":{\"exprType\":\"BINARY\",\"left\":{\"colVal\":0,\"dataType\":15,\"exprType\":\"FIELD_REFERENCE\",\"width\":50},\"operator\":\"NOT_EQUAL\",\"returnType\":4,\"right\":{\"dataType\":15,\"exprType\":\"LITERAL\",\"isNull\":false,\"value\":\"\\\"\\\\\\\\t/\\\\b\\\\f\\\\n\\\\r\\\\t123\",\"width\":18}},\"exprType\":\"IF\",\"if_false\":{\"condition\":{\"exprType\":\"BINARY\",\"left\":{\"colVal\":2,\"dataType\":15,\"exprType\":\"FIELD_REFERENCE\",\"width\":50},\"operator\":\"NOT_EQUAL\",\"returnType\":4,\"right\":{\"dataType\":15,\"exprType\":\"LITERAL\",\"isNull\":false,\"value\":\"\\\"\\\\\\\\t/\\\\b\\\\f\\\\n\\\\r\\\\t345\",\"width\":18}},\"exprType\":\"IF\",\"if_false\":{\"exprType\":\"BINARY\",\"left\":{\"colVal\":3,\"dataType\":15,\"exprType\":\"FIELD_REFERENCE\",\"width\":50},\"operator\":\"NOT_EQUAL\",\"returnType\":4,\"right\":{\"dataType\":15,\"exprType\":\"LITERAL\",\"isNull\":false,\"value\":\"\\\"\\\\\\\\t/\\\\b\\\\f\\\\n\\\\r\\\\t456\",\"width\":18}},\"if_true\":{\"exprType\":\"BINARY\",\"left\":{\"colVal\":2,\"dataType\":15,\"exprType\":\"FIELD_REFERENCE\",\"width\":50},\"operator\":\"NOT_EQUAL\",\"returnType\":4,\"right\":{\"dataType\":15,\"exprType\":\"LITERAL\",\"isNull\":false,\"value\":\"\\\"\\\\\\\\t/\\\\b\\\\f\\\\n\\\\r\\\\t123\",\"width\":18}},\"returnType\":4},\"if_true\":{\"exprType\":\"BINARY\",\"left\":{\"colVal\":1,\"dataType\":15,\"exprType\":\"FIELD_REFERENCE\",\"width\":50},\"operator\":\"NOT_EQUAL\",\"returnType\":4,\"right\":{\"dataType\":15,\"exprType\":\"LITERAL\",\"isNull\":false,\"value\":\"\\\"\\\\\\\\t/\\\\b\\\\f\\\\n\\\\r\\\\t234\",\"width\":18}},\"returnType\":4} " checkJsonKeyValueIgnoreKeySequence(specialCharacterFilterExp, specialCharacterRunResult, specialCharacterExpression)