From a35e7152941e507a5947f98fee5bb98dfa92dbe8 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Mon, 3 Jun 2024 20:26:37 +0800 Subject: [PATCH 1/7] [hive-extension] test --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index cb39f034c..eed014159 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -342,7 +342,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { // is VectorOperator reducer = (Operator) reducer.getChildOperators().get(0); } - if (isReplaceable(reducer, true) && reducer.getType().equals(OperatorType.GROUPBY)) { + if (isReplaceable(reducer, true) && (reducer.getType().equals(OperatorType.GROUPBY) || && reducer.getType().equals(OperatorType.SELECT))) { tezWork.getEdgeProperty(work, child).setEdgeType(TezEdgeProperty.EdgeType.CUSTOM_SIMPLE_EDGE); return true; } -- Gitee From fd931b3ea13d4ab21553316760028f1380331888 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Mon, 3 Jun 2024 20:32:42 +0800 Subject: [PATCH 2/7] [hive-extension] test --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index eed014159..f7200b8cb 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -342,7 +342,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { // is VectorOperator reducer = (Operator) reducer.getChildOperators().get(0); } - if (isReplaceable(reducer, true) && (reducer.getType().equals(OperatorType.GROUPBY) || && reducer.getType().equals(OperatorType.SELECT))) { + if (isReplaceable(reducer, true) && (reducer.getType().equals(OperatorType.GROUPBY) || reducer.getType().equals(OperatorType.SELECT))) { tezWork.getEdgeProperty(work, child).setEdgeType(TezEdgeProperty.EdgeType.CUSTOM_SIMPLE_EDGE); return true; } -- Gitee From b39dd062bf2b1015207ed1c4c8dacb671caf9736 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Tue, 4 Jun 2024 11:37:23 +0800 Subject: [PATCH 3/7] [hive-extension] test --- .../boostkit/hive/OmniExecuteWithHookContext.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index f7200b8cb..1eb11ddaf 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -157,6 +157,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { private Set reduceSinkReplaceableWorkName = new HashSet<>(); private final Set inputUnReplaceableWork = new HashSet<>(); + private final List reducesinkNeedSort = new ArrayList<>(); private TezWork tezWork; @@ -176,6 +177,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { reduceSinkReplaceableWorkName.clear(); inputUnReplaceableWork.clear(); needSortSerdeReduceSinkName.clear(); + reducesinkNeedSort.clear(); tezWork = null; } @@ -186,6 +188,15 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { return queryPlan.getRootTasks().get(0) instanceof ExplainTask; } + private static T getLastElement(final iterable elements) { + T lastElement = null; + for (T element: elements) { + lastElement = element; + } + + return lastElement; + } + public void run(HookContext hookContext) throws Exception { this.hookContext = hookContext; omniHiveConf = new OmniHiveConf(hookContext.getConf()); @@ -344,6 +355,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { } if (isReplaceable(reducer, true) && (reducer.getType().equals(OperatorType.GROUPBY) || reducer.getType().equals(OperatorType.SELECT))) { tezWork.getEdgeProperty(work, child).setEdgeType(TezEdgeProperty.EdgeType.CUSTOM_SIMPLE_EDGE); + reducesinkNeedSort.add(getLastElement(work.getAllOperators())); return true; } return false; @@ -1243,7 +1255,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { case REDUCESINK: OmniReduceSinkOperator omniReduceSinkOperator = new OmniReduceSinkOperator(current.getCompilationOpContext(), (ReduceSinkDesc) current.getConf(), reduceSinkCanReplace); ReduceSinkDesc conf = (ReduceSinkDesc) current.getConf(); - if (conf.getTopN() == -1) { + if (conf.getTopN() == -1 && !reducesinkNeedSort.contain(current)) { omniReduceSinkOperator.getConf().getKeySerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); omniReduceSinkOperator.getConf().getValueSerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); } else { -- Gitee From cb36f84cc57f08c83c9f07cee3d990cac7433f31 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Tue, 4 Jun 2024 11:43:19 +0800 Subject: [PATCH 4/7] [hive-extension] test --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index 1eb11ddaf..f513f6302 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -188,7 +188,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { return queryPlan.getRootTasks().get(0) instanceof ExplainTask; } - private static T getLastElement(final iterable elements) { + private static T getLastElement(final Iterable elements) { T lastElement = null; for (T element: elements) { lastElement = element; -- Gitee From 8cc160708373a74d3e2fe680fb60053055e8e40b Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Tue, 4 Jun 2024 11:46:22 +0800 Subject: [PATCH 5/7] [hive-extension] test --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index f513f6302..3dbcefa8d 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -1255,7 +1255,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { case REDUCESINK: OmniReduceSinkOperator omniReduceSinkOperator = new OmniReduceSinkOperator(current.getCompilationOpContext(), (ReduceSinkDesc) current.getConf(), reduceSinkCanReplace); ReduceSinkDesc conf = (ReduceSinkDesc) current.getConf(); - if (conf.getTopN() == -1 && !reducesinkNeedSort.contain(current)) { + if (conf.getTopN() == -1 && !reducesinkNeedSort.contains(current)) { omniReduceSinkOperator.getConf().getKeySerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); omniReduceSinkOperator.getConf().getValueSerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); } else { -- Gitee From e09f82f0ce56f38d40869df8450bfb70b3028f58 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Tue, 4 Jun 2024 19:49:46 +0800 Subject: [PATCH 6/7] [hive-extension] test --- .../huawei/boostkit/hive/OmniExecuteWithHookContext.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index 3dbcefa8d..963dae458 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -353,11 +353,17 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { // is VectorOperator reducer = (Operator) reducer.getChildOperators().get(0); } - if (isReplaceable(reducer, true) && (reducer.getType().equals(OperatorType.GROUPBY) || reducer.getType().equals(OperatorType.SELECT))) { + if (isReplaceable(reducer, true) && reducer.getType().equals(OperatorType.GROUPBY)) { tezWork.getEdgeProperty(work, child).setEdgeType(TezEdgeProperty.EdgeType.CUSTOM_SIMPLE_EDGE); reducesinkNeedSort.add(getLastElement(work.getAllOperators())); return true; } + + if (isReplaceable(reducer, true) && reducer.getType().equals(OperatorType.SELECT)) { + tezWork.getEdgeProperty(work, child).setEdgeType(TezEdgeProperty.EdgeType.SIMPLE_EDGE); + reducesinkNeedSort.add(getLastElement(work.getAllOperators())); + return true; + } return false; } -- Gitee From 4f3e1f7d713959fc3629c5577ac91ba021f3a5db Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Tue, 4 Jun 2024 19:59:18 +0800 Subject: [PATCH 7/7] [hive-extension] test --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 1 - 1 file changed, 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index 963dae458..e873c7e8e 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -355,7 +355,6 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { } if (isReplaceable(reducer, true) && reducer.getType().equals(OperatorType.GROUPBY)) { tezWork.getEdgeProperty(work, child).setEdgeType(TezEdgeProperty.EdgeType.CUSTOM_SIMPLE_EDGE); - reducesinkNeedSort.add(getLastElement(work.getAllOperators())); return true; } -- Gitee