From 7fef8ea744ebab346e25e709da1e64fd87dd0292 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Mon, 24 Jun 2024 16:44:33 +0800 Subject: [PATCH 1/9] [hive-extension] test for custom edge no sort --- .../hive/OmniExecuteWithHookContext.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index e3cfdcf52..656ca8f11 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -347,13 +347,31 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { return reduceSinkReplaceable; } + private boolean selectOrPtfEdgeReplaceable(Set> operators) { + final boolean[] canBeReplaced = {true}; + if (operators.size() > 4) { + return false; + } + + operators.forEach(item -> { + if (!(item.getOperatorId().contains("SEL") || item.getOperatorId().contains("PTF") || item.getOperatorId().contains("RS"))) { + canBeReplaced[0] = false; + } + }); + return canBeReplaced[0]; + } + private boolean groupByEdgeReplaceable(BaseWork work, BaseWork child) { Operator reducer = ((ReduceWork) child).getReducer(); if (reducer.getType() == null) { // is VectorOperator reducer = (Operator) reducer.getChildOperators().get(0); } - if (isReplaceable(reducer, true) && reducer.getType().equals(OperatorType.GROUPBY)) { + + if (isReplaceable(reducer, true)) { + return false; + } + if (reducer.getType().equals(OperatorType.GROUPBY) || (reducer.getType().equals(OperatorType.SELECT) && selectOrPtfEdgeReplaceable(child.getAllOperators()))) { tezWork.getEdgeProperty(work, child).setEdgeType(TezEdgeProperty.EdgeType.CUSTOM_SIMPLE_EDGE); return true; } -- Gitee From 8c799a6e5cb0f61eecdc1e1b7220aae77d4c4ce5 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Mon, 24 Jun 2024 16:59:31 +0800 Subject: [PATCH 2/9] [hive-extension] test for custom edge no sort --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index 656ca8f11..973adf41a 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -368,7 +368,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { reducer = (Operator) reducer.getChildOperators().get(0); } - if (isReplaceable(reducer, true)) { + if (!isReplaceable(reducer, true)) { return false; } if (reducer.getType().equals(OperatorType.GROUPBY) || (reducer.getType().equals(OperatorType.SELECT) && selectOrPtfEdgeReplaceable(child.getAllOperators()))) { -- Gitee From 50cd3c4e20d5c1ad8aee64735b87c8476469af8d Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Mon, 24 Jun 2024 19:12:06 +0800 Subject: [PATCH 3/9] [hive-extension] test for custom edge no sort --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index 973adf41a..15d9d023b 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -354,7 +354,10 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { } operators.forEach(item -> { - if (!(item.getOperatorId().contains("SEL") || item.getOperatorId().contains("PTF") || item.getOperatorId().contains("RS"))) { + if (item.getType().equals(OperatorType.PTF) && !PTFReplaceable(item)) { + canBeReplaced[0] = false; + } + if (!(item.getOperatorId().contains("SEL") || item.getOperatorId().contains("PTF") || item.getOperatorId().contains("RS") || item.getOperatorId().contains("LIMIT"))) { canBeReplaced[0] = false; } }); -- Gitee From 0ed74c21375000e59621d789f9860e7ae9b38981 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Mon, 24 Jun 2024 19:19:17 +0800 Subject: [PATCH 4/9] [hive-extension] test for custom edge no sort --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index 15d9d023b..d9ec8ece9 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -357,7 +357,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { if (item.getType().equals(OperatorType.PTF) && !PTFReplaceable(item)) { canBeReplaced[0] = false; } - if (!(item.getOperatorId().contains("SEL") || item.getOperatorId().contains("PTF") || item.getOperatorId().contains("RS") || item.getOperatorId().contains("LIMIT"))) { + if (!(item.getOperatorId().contains("SEL") || item.getOperatorId().contains("PTF") || item.getOperatorId().contains("RS") || item.getOperatorId().contains("LIM") || item.getOperatorId().contains("FS"))) { canBeReplaced[0] = false; } }); -- Gitee From 3e26f21ad42aaf96b8efaab26d8af40d5c7e3430 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Tue, 25 Jun 2024 11:03:17 +0800 Subject: [PATCH 5/9] [hive-extension] test for custom edge no sort --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index d9ec8ece9..b8ab67391 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -1275,7 +1275,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { case REDUCESINK: OmniReduceSinkOperator omniReduceSinkOperator = new OmniReduceSinkOperator(current.getCompilationOpContext(), (ReduceSinkDesc) current.getConf(), reduceSinkCanReplace); ReduceSinkDesc conf = (ReduceSinkDesc) current.getConf(); - if (conf.getTopN() == -1) { + if (conf.getTopN() == -1 && !conf.hasOrderBy()) { omniReduceSinkOperator.getConf().getKeySerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); omniReduceSinkOperator.getConf().getValueSerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); } else { -- Gitee From 30546f2f96f6306655931ab4137f16c9903541a5 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Wed, 26 Jun 2024 14:13:50 +0800 Subject: [PATCH 6/9] [hive-extension] test for custom edge no sort --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index b8ab67391..a636c897b 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -1275,7 +1275,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { case REDUCESINK: OmniReduceSinkOperator omniReduceSinkOperator = new OmniReduceSinkOperator(current.getCompilationOpContext(), (ReduceSinkDesc) current.getConf(), reduceSinkCanReplace); ReduceSinkDesc conf = (ReduceSinkDesc) current.getConf(); - if (conf.getTopN() == -1 && !conf.hasOrderBy()) { + if (conf.getTopN() == -1 && !conf.hasOrderBy() && !conf.isOrdering()) { omniReduceSinkOperator.getConf().getKeySerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); omniReduceSinkOperator.getConf().getValueSerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); } else { -- Gitee From 17db15da14b0faf82b7c8da7dc5d75df0240e1da Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Wed, 26 Jun 2024 14:55:19 +0800 Subject: [PATCH 7/9] [hive-extension] test for custom edge no sort --- .../huawei/boostkit/hive/OmniExecuteWithHookContext.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index a636c897b..a446277c2 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -374,10 +374,14 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { if (!isReplaceable(reducer, true)) { return false; } - if (reducer.getType().equals(OperatorType.GROUPBY) || (reducer.getType().equals(OperatorType.SELECT) && selectOrPtfEdgeReplaceable(child.getAllOperators()))) { + if (reducer.getType().equals(OperatorType.GROUPBY) || reducer.getType().equals(OperatorType.SELECT)) { tezWork.getEdgeProperty(work, child).setEdgeType(TezEdgeProperty.EdgeType.CUSTOM_SIMPLE_EDGE); return true; } + + if (selectOrPtfEdgeReplaceable(child.getAllOperators())) { + return true; + } return false; } @@ -1275,7 +1279,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { case REDUCESINK: OmniReduceSinkOperator omniReduceSinkOperator = new OmniReduceSinkOperator(current.getCompilationOpContext(), (ReduceSinkDesc) current.getConf(), reduceSinkCanReplace); ReduceSinkDesc conf = (ReduceSinkDesc) current.getConf(); - if (conf.getTopN() == -1 && !conf.hasOrderBy() && !conf.isOrdering()) { + if (conf.getTopN() == -1 && !conf.hasOrderBy()) { omniReduceSinkOperator.getConf().getKeySerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); omniReduceSinkOperator.getConf().getValueSerializeInfo().getProperties().setProperty(SERIALIZATION_LIB, OmniVecBatchSerDe.class.getName()); } else { -- Gitee From 689f58328f727a22338352f4f38f3c9462bd88ac Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Wed, 26 Jun 2024 14:57:35 +0800 Subject: [PATCH 8/9] [hive-extension] test for custom edge no sort --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index a446277c2..1e2837fe2 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -374,12 +374,12 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { if (!isReplaceable(reducer, true)) { return false; } - if (reducer.getType().equals(OperatorType.GROUPBY) || reducer.getType().equals(OperatorType.SELECT)) { + if (reducer.getType().equals(OperatorType.GROUPBY)) { tezWork.getEdgeProperty(work, child).setEdgeType(TezEdgeProperty.EdgeType.CUSTOM_SIMPLE_EDGE); return true; } - if (selectOrPtfEdgeReplaceable(child.getAllOperators())) { + if (reducer.getType().equals(OperatorType.SELECT) && selectOrPtfEdgeReplaceable(child.getAllOperators())) { return true; } return false; -- Gitee From 9eca7c99e0c0f46aa1fcbc486219cefabd89f7c0 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Wed, 26 Jun 2024 16:29:21 +0800 Subject: [PATCH 9/9] [hive-extension] test for custom edge no sort --- .../com/huawei/boostkit/hive/OmniExecuteWithHookContext.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java index 1e2837fe2..805934cd1 100644 --- a/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java +++ b/omnioperator/omniop-hive-extension/src/main/java/com/huawei/boostkit/hive/OmniExecuteWithHookContext.java @@ -333,7 +333,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { reduceSinkReplaceable = true; } if (edgeType == TezEdgeProperty.EdgeType.SIMPLE_EDGE && child instanceof ReduceWork) { - reduceSinkReplaceable = groupByEdgeReplaceable(work, child); + reduceSinkReplaceable = simpleEdgeForChildReplaceable(work, child); } if (child instanceof MergeJoinWork) { Operator reducer = ((MergeJoinWork) child).getMainWork().getAnyRootOperator(); @@ -364,7 +364,7 @@ public class OmniExecuteWithHookContext implements ExecuteWithHookContext { return canBeReplaced[0]; } - private boolean groupByEdgeReplaceable(BaseWork work, BaseWork child) { + private boolean simpleEdgeForChildReplaceable(BaseWork work, BaseWork child) { Operator reducer = ((ReduceWork) child).getReducer(); if (reducer.getType() == null) { // is VectorOperator -- Gitee