From becb23f34f0ac2ebc555f26729e44c5418966df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B8=85=E6=B5=A9=E7=BF=94?= <1549665469@qq.com> Date: Wed, 18 Dec 2024 09:48:21 +0800 Subject: [PATCH] =?UTF-8?q?=E5=9C=A83.2,3.3.1=EF=BC=8C3.4=EF=BC=8C3.5?= =?UTF-8?q?=E7=9A=84UT=E4=B8=AD=E5=8E=BB=E9=99=A4=E6=8E=89=E5=8E=9F?= =?UTF-8?q?=E6=9C=89=E5=85=B3=E4=BA=8E=E2=80=9D=E4=B8=8D=E6=94=AF=E6=8C=81?= =?UTF-8?q?bnlj"=E7=AE=97=E5=AD=90=E7=9A=84UT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../spark/FallbackStrategiesSuite.scala | 17 +---------------- .../spark/FallbackStrategiesSuite.scala | 18 +----------------- .../spark/FallbackStrategiesSuite.scala | 18 +----------------- .../spark/FallbackStrategiesSuite.scala | 18 +----------------- 4 files changed, 4 insertions(+), 67 deletions(-) diff --git a/omnioperator/omniop-spark-extension/spark-extension-ut/spark32-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala b/omnioperator/omniop-spark-extension/spark-extension-ut/spark32-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala index 9b97295fc..c86375ae0 100644 --- a/omnioperator/omniop-spark-extension/spark-extension-ut/spark32-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala +++ b/omnioperator/omniop-spark-extension/spark-extension-ut/spark32-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala @@ -143,22 +143,7 @@ class FallbackStrategiesSuite extends QueryTest with SharedSparkSession { } - test("Fall back the last stage contains unsupported bnlj if meeting the configured threshold") { - withSQLConf(("spark.omni.sql.columnar.wholeStage.fallback.threshold", "2"), - (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")) { - val df = spark.sql("select age + salary from (select age, salary from (select age from employees_for_fallback_ut_test order by age limit 1) s1," + - " (select salary from employees_for_fallback_ut_test order by salary limit 1) s2)") - QueryTest.checkAnswer(df, Seq(Row(10023))) - val plans = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan.collect({ - case plan: ProjectExec => plan - case plan: TakeOrderedAndProjectExec => plan - }) - assert(plans.count(_.isInstanceOf[ProjectExec]) == 1, "the last stage containing projectExec should fallback") - assert(plans.count(_.isInstanceOf[TakeOrderedAndProjectExec]) == 1, "the last stage containing projectExec should fallback") - } - } - - test("Don't Fall back the last stage contains unsupported bnlj if NOT meeting the configured threshold") { + test("Don't Fall back the last stage contains supported bnlj if NOT meeting the configured threshold") { withSQLConf(("spark.omni.sql.columnar.wholeStage.fallback.threshold", "3"), (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")) { val df = spark.sql("select age + salary from (select age, salary from (select age from employees_for_fallback_ut_test order by age limit 1) s1," + diff --git a/omnioperator/omniop-spark-extension/spark-extension-ut/spark33-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala b/omnioperator/omniop-spark-extension/spark-extension-ut/spark33-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala index 9b97295fc..2aa07b2e3 100644 --- a/omnioperator/omniop-spark-extension/spark-extension-ut/spark33-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala +++ b/omnioperator/omniop-spark-extension/spark-extension-ut/spark33-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala @@ -142,23 +142,7 @@ class FallbackStrategiesSuite extends QueryTest with SharedSparkSession { } } - - test("Fall back the last stage contains unsupported bnlj if meeting the configured threshold") { - withSQLConf(("spark.omni.sql.columnar.wholeStage.fallback.threshold", "2"), - (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")) { - val df = spark.sql("select age + salary from (select age, salary from (select age from employees_for_fallback_ut_test order by age limit 1) s1," + - " (select salary from employees_for_fallback_ut_test order by salary limit 1) s2)") - QueryTest.checkAnswer(df, Seq(Row(10023))) - val plans = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan.collect({ - case plan: ProjectExec => plan - case plan: TakeOrderedAndProjectExec => plan - }) - assert(plans.count(_.isInstanceOf[ProjectExec]) == 1, "the last stage containing projectExec should fallback") - assert(plans.count(_.isInstanceOf[TakeOrderedAndProjectExec]) == 1, "the last stage containing projectExec should fallback") - } - } - - test("Don't Fall back the last stage contains unsupported bnlj if NOT meeting the configured threshold") { + test("Don't Fall back the last stage contains supported bnlj if NOT meeting the configured threshold") { withSQLConf(("spark.omni.sql.columnar.wholeStage.fallback.threshold", "3"), (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")) { val df = spark.sql("select age + salary from (select age, salary from (select age from employees_for_fallback_ut_test order by age limit 1) s1," + diff --git a/omnioperator/omniop-spark-extension/spark-extension-ut/spark34-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala b/omnioperator/omniop-spark-extension/spark-extension-ut/spark34-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala index 9b97295fc..2aa07b2e3 100644 --- a/omnioperator/omniop-spark-extension/spark-extension-ut/spark34-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala +++ b/omnioperator/omniop-spark-extension/spark-extension-ut/spark34-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala @@ -142,23 +142,7 @@ class FallbackStrategiesSuite extends QueryTest with SharedSparkSession { } } - - test("Fall back the last stage contains unsupported bnlj if meeting the configured threshold") { - withSQLConf(("spark.omni.sql.columnar.wholeStage.fallback.threshold", "2"), - (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")) { - val df = spark.sql("select age + salary from (select age, salary from (select age from employees_for_fallback_ut_test order by age limit 1) s1," + - " (select salary from employees_for_fallback_ut_test order by salary limit 1) s2)") - QueryTest.checkAnswer(df, Seq(Row(10023))) - val plans = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan.collect({ - case plan: ProjectExec => plan - case plan: TakeOrderedAndProjectExec => plan - }) - assert(plans.count(_.isInstanceOf[ProjectExec]) == 1, "the last stage containing projectExec should fallback") - assert(plans.count(_.isInstanceOf[TakeOrderedAndProjectExec]) == 1, "the last stage containing projectExec should fallback") - } - } - - test("Don't Fall back the last stage contains unsupported bnlj if NOT meeting the configured threshold") { + test("Don't Fall back the last stage contains supported bnlj if NOT meeting the configured threshold") { withSQLConf(("spark.omni.sql.columnar.wholeStage.fallback.threshold", "3"), (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")) { val df = spark.sql("select age + salary from (select age, salary from (select age from employees_for_fallback_ut_test order by age limit 1) s1," + diff --git a/omnioperator/omniop-spark-extension/spark-extension-ut/spark35-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala b/omnioperator/omniop-spark-extension/spark-extension-ut/spark35-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala index 3847b29de..8bafbb1ba 100644 --- a/omnioperator/omniop-spark-extension/spark-extension-ut/spark35-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala +++ b/omnioperator/omniop-spark-extension/spark-extension-ut/spark35-ut/src/test/scala/com/huawei/boostkit/spark/FallbackStrategiesSuite.scala @@ -142,23 +142,7 @@ class FallbackStrategiesSuite extends QueryTest with SharedSparkSession { } } - - test("Fall back the last stage contains unsupported bnlj if meeting the configured threshold") { - withSQLConf(("spark.omni.sql.columnar.wholeStage.fallback.threshold", "2"), - (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")) { - val df = spark.sql("select age + salary from (select age, salary from (select age from employees_for_fallback_ut_test order by age limit 1) s1," + - " (select salary from employees_for_fallback_ut_test order by salary limit 1) s2)") - QueryTest.checkAnswer(df, Seq(Row(10023))) - val plans = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec].executedPlan.collect({ - case plan: ProjectExec => plan - case plan: TakeOrderedAndProjectExec => plan - }) - assert(plans.count(_.isInstanceOf[ProjectExec]) == 1, "the last stage containing projectExec should fallback") - assert(plans.count(_.isInstanceOf[TakeOrderedAndProjectExec]) == 1, "the last stage containing projectExec should fallback") - } - } - - test("Don't Fall back the last stage contains unsupported bnlj if NOT meeting the configured threshold") { + test("Don't Fall back the last stage contains supported bnlj if NOT meeting the configured threshold") { withSQLConf(("spark.omni.sql.columnar.wholeStage.fallback.threshold", "3"), (SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true")) { val df = spark.sql("select age + salary from (select age, salary from (select age from employees_for_fallback_ut_test order by age limit 1) s1," + -- Gitee