From dd152a24aa7a7711441c1c144c53e9f10051cd0a Mon Sep 17 00:00:00 2001 From: panmingyi Date: Mon, 26 Aug 2024 14:18:49 +0800 Subject: [PATCH 1/3] hive support power/pow --- .../hive/expression/ExpressionUtils.java | 3 + .../processor/PowerExpressionProcessor.java | 60 +++++++++++++++++++ .../hive/processor/PowerExpressionTest.java | 53 ++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/processor/PowerExpressionProcessor.java create mode 100644 omnioperator/omniop-hive-extension/src/test/java/com/huawei/boostkit/hive/processor/PowerExpressionTest.java diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/expression/ExpressionUtils.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/expression/ExpressionUtils.java index 0889b3ca1..5e2088f16 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/expression/ExpressionUtils.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/expression/ExpressionUtils.java @@ -40,6 +40,7 @@ import com.huawei.boostkit.hive.processor.UpperExpressionProcessor; import com.huawei.boostkit.hive.processor.LowerExpressionProcessor; import com.huawei.boostkit.hive.processor.LengthExpressionProcessor; import com.huawei.boostkit.hive.processor.InstrExpressionProcessor; +import com.huawei.boostkit.hive.processor.PowerExpressionProcessor; import nova.hetu.omniruntime.type.Decimal128DataType; import nova.hetu.omniruntime.type.Decimal64DataType; @@ -83,6 +84,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLower; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLength; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInstr; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFPower; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; @@ -133,6 +135,7 @@ public class ExpressionUtils { put(GenericUDFLength.class, new LengthExpressionProcessor()); put(GenericUDFLower.class, new LowerExpressionProcessor()); put(GenericUDFInstr.class, new InstrExpressionProcessor()); + put(GenericUDFPower.class, new PowerExpressionProcessor()); } }; diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/processor/PowerExpressionProcessor.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/processor/PowerExpressionProcessor.java new file mode 100644 index 000000000..5ac085484 --- /dev/null +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/processor/PowerExpressionProcessor.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.huawei.boostkit.hive.processor; + +import com.huawei.boostkit.hive.expression.BaseExpression; +import com.huawei.boostkit.hive.expression.ExpressionUtils; +import com.huawei.boostkit.hive.expression.FunctionExpression; +import com.huawei.boostkit.hive.expression.TypeUtils; + +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; + +import java.util.ArrayList; +import java.util.List; + +public class PowerExpressionProcessor implements ExpressionProcessor { + @Override + public BaseExpression process(ExprNodeGenericFuncDesc node, String operator, ObjectInspector inspector) { + List children = node.getChildren(); + FunctionExpression functionExpression = new FunctionExpression( + TypeUtils.convertHiveTypeToOmniType(node.getTypeInfo()), "power", children.size(), null); + for (ExprNodeDesc child : children) { + if (child.getTypeInfo().getTypeName().equals("double")) { + if (child instanceof ExprNodeGenericFuncDesc) { + functionExpression.add(ExpressionUtils.build((ExprNodeGenericFuncDesc) child, inspector)); + } else { + functionExpression.add(ExpressionUtils.createNode(child, inspector)); + } + } else { + PrimitiveTypeInfo typeInfo = new PrimitiveTypeInfo(); + typeInfo.setTypeName("double"); + GenericUDFBridge genericUDFBridge = new GenericUDFBridge("UDFToDouble", false, "org.apache.hadoop.hive.ql.udf.UDFToDouble"); + List exprNodeDescList = new ArrayList<>(); + exprNodeDescList.add(child); + ExprNodeGenericFuncDesc child2 = new ExprNodeGenericFuncDesc(typeInfo, genericUDFBridge, exprNodeDescList); + functionExpression.add(ExpressionUtils.build(child2, inspector)); + } + } + return functionExpression; + } +} diff --git a/omnioperator/omniop-hive-extension/src/test/java/com/huawei/boostkit/hive/processor/PowerExpressionTest.java b/omnioperator/omniop-hive-extension/src/test/java/com/huawei/boostkit/hive/processor/PowerExpressionTest.java new file mode 100644 index 000000000..a713d91c4 --- /dev/null +++ b/omnioperator/omniop-hive-extension/src/test/java/com/huawei/boostkit/hive/processor/PowerExpressionTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.huawei.boostkit.hive.processor; + +import com.huawei.boostkit.hive.CommonTest; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class PowerExpressionTest extends CommonTest { + @Test + public void testPowerExpression() throws IOException { + utRunSqlOnDriver("use ut_database"); + + utRunSqlOnDriver("drop table if exists power_expression"); + utRunSqlOnDriver("create table if not exists power_expression (id int, base double, exponent double) " + + "stored as orc"); + utRunSqlOnDriver("insert into power_expression values(1, 2, 1)"); + utRunSqlOnDriver("insert into power_expression values(2, 2, 2)"); + utRunSqlOnDriver("insert into power_expression values(3, 2, 3)"); + utRunSqlOnDriver("insert into power_expression values(4, 2, 4)"); + utRunSqlOnDriver("select power(base, exponent) from power_expression order by id"); + List rs = new ArrayList<>(); + driver.getResults(rs); + + utRunSqlOnDriver("drop table if exists power_expression"); + Assert.assertEquals(4, rs.size()); + Assert.assertEquals("2.0", rs.get(0)); + Assert.assertEquals("4.0", rs.get(1)); + Assert.assertEquals("8.0", rs.get(2)); + Assert.assertEquals("16.0", rs.get(3)); + } +} -- Gitee From 68a061d16cf32ada98d53357b2caa3dfc828d56d Mon Sep 17 00:00:00 2001 From: panmingyi Date: Mon, 26 Aug 2024 19:30:19 +0800 Subject: [PATCH 2/3] fix power/pow --- .../processor/PowerExpressionProcessor.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/processor/PowerExpressionProcessor.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/processor/PowerExpressionProcessor.java index 5ac085484..ffcf16f5a 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/processor/PowerExpressionProcessor.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/processor/PowerExpressionProcessor.java @@ -23,6 +23,8 @@ import com.huawei.boostkit.hive.expression.ExpressionUtils; import com.huawei.boostkit.hive.expression.FunctionExpression; import com.huawei.boostkit.hive.expression.TypeUtils; +import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; @@ -48,11 +50,18 @@ public class PowerExpressionProcessor implements ExpressionProcessor { } else { PrimitiveTypeInfo typeInfo = new PrimitiveTypeInfo(); typeInfo.setTypeName("double"); - GenericUDFBridge genericUDFBridge = new GenericUDFBridge("UDFToDouble", false, "org.apache.hadoop.hive.ql.udf.UDFToDouble"); - List exprNodeDescList = new ArrayList<>(); - exprNodeDescList.add(child); - ExprNodeGenericFuncDesc child2 = new ExprNodeGenericFuncDesc(typeInfo, genericUDFBridge, exprNodeDescList); - functionExpression.add(ExpressionUtils.build(child2, inspector)); + if (child instanceof ExprNodeConstantDesc && ((ExprNodeConstantDesc) child).getValue() instanceof HiveDecimal) { + HiveDecimal val = (HiveDecimal) ((ExprNodeConstantDesc) child).getValue(); + ((ExprNodeConstantDesc) child).setValue(val.doubleValue()); + child.setTypeInfo(typeInfo); + functionExpression.add(ExpressionUtils.createNode(child, inspector)); + } else { + GenericUDFBridge genericUDFBridge = new GenericUDFBridge("UDFToDouble", false, "org.apache.hadoop.hive.ql.udf.UDFToDouble"); + List exprNodeDescList = new ArrayList<>(); + exprNodeDescList.add(child); + ExprNodeGenericFuncDesc child2 = new ExprNodeGenericFuncDesc(typeInfo, genericUDFBridge, exprNodeDescList); + functionExpression.add(ExpressionUtils.build(child2, inspector)); + } } } return functionExpression; -- Gitee From b3971961508dea05a749af68497b57990c76c2fb Mon Sep 17 00:00:00 2001 From: panmingyi Date: Tue, 24 Sep 2024 10:38:12 +0800 Subject: [PATCH 3/3] =?UTF-8?q?Const=20NaN=E3=80=81=20Infinity=20rollback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index 8631aa772..3f92539d0 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -1021,6 +1021,11 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { } else if ((current instanceof ExprNodeColumnDesc || current instanceof ExprNodeConstantDesc) && !SUPPORTED_TYPE.contains(((PrimitiveTypeInfo) current.getTypeInfo()).getPrimitiveCategory())) { return false; + } else if (current instanceof ExprNodeConstantDesc && + ((ExprNodeConstantDesc) current).getValue() instanceof Double && + (((Double) ((ExprNodeConstantDesc) current).getValue()).isNaN() || + ((Double) ((ExprNodeConstantDesc) current).getValue()).isInfinite())) { + return false; } } return true; -- Gitee