From 9a15237b7b2324268284671290ea32060abf2e63 Mon Sep 17 00:00:00 2001 From: huanglong Date: Tue, 15 Oct 2024 15:57:05 +0800 Subject: [PATCH 1/4] fix resin UT --- .../catalyst/optimizer/CombineJoinedAggregatesSuite.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineJoinedAggregatesSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineJoinedAggregatesSuite.scala index a55fcd437..d2fdee36b 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineJoinedAggregatesSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineJoinedAggregatesSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst.optimizer +import org.apache.spark.SparkConf import org.apache.spark.sql.DataFrame import org.apache.spark.sql.execution.{ColumnarSparkPlanTest, SparkPlan} @@ -27,10 +28,13 @@ class CombineJoinedAggregatesSuite extends ColumnarSparkPlanTest { private var dfForResin: DataFrame = _ + override def sparkConf: SparkConf = super.sparkConf + + .set("spark.sql.execution.combineJoinedAggregates.enabled", "true") + override def beforeAll(): Unit = { super.beforeAll() - spark.conf.set("spark.sql.execution.combineJoinedAggregates.enabled", true) dfForResin = Seq[(Boolean, Short, Int, Long, Float, Double, String)]( (true, 10.toShort, 100, 1000L, 1.5f, 10.5, "aaa"), (false, 20.toShort, 200, 2000L, 2.5f, 20.5, "bbb"), -- Gitee From 66b468993c11de230158abaf19a0bb153d9b5421 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Tue, 15 Oct 2024 16:05:39 +0800 Subject: [PATCH 2/4] [spark-extension] bug:fix ci error --- .../sql/execution/PushOrderedLimitThroughAggSuite.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/execution/PushOrderedLimitThroughAggSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/execution/PushOrderedLimitThroughAggSuite.scala index 8e6cad49a..7e6a8a999 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/execution/PushOrderedLimitThroughAggSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/execution/PushOrderedLimitThroughAggSuite.scala @@ -21,15 +21,16 @@ package org.apache.spark.sql.execution import org.apache.spark.sql.functions.{avg, count, first, max, min, sum} import org.apache.spark.sql.types._ import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.SparkConf class PushOrderedLimitThroughAggSuite extends ColumnarSparkPlanTest { private var df: DataFrame = _ + override def sparkConf: SparkConf = super.sparkConf.set("spark.omni.sql.columnar.topNSort", "true") + override def sparkConf: SparkConf = super.sparkConf.set("spark.omni.sql.columnar.pushOrderedLimitThroughAggEnable.enabled", "true") + override def sparkConf: SparkConf = super.sparkConf.set("spark.sql.adaptive.enabled", "true") protected override def beforeAll(): Unit = { super.beforeAll() - spark.conf.set("spark.omni.sql.columnar.topNSort", true) - spark.conf.set("spark.omni.sql.columnar.pushOrderedLimitThroughAggEnable.enabled", true) - spark.conf.set("spark.sql.adaptive.enabled", true) df = spark.createDataFrame( sparkContext.parallelize(Seq( Row(1, 2.0, 1L, "a"), -- Gitee From 80030e8cdfe9d7449b1c87bf2513da14f33d1c50 Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Tue, 15 Oct 2024 16:15:18 +0800 Subject: [PATCH 3/4] [spark-extension] bug:fix ci error for combineJoinedAggregates --- .../sql/catalyst/optimizer/CombineJoinedAggregatesSuite.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineJoinedAggregatesSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineJoinedAggregatesSuite.scala index d2fdee36b..955ec8ec2 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineJoinedAggregatesSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombineJoinedAggregatesSuite.scala @@ -28,9 +28,7 @@ class CombineJoinedAggregatesSuite extends ColumnarSparkPlanTest { private var dfForResin: DataFrame = _ - override def sparkConf: SparkConf = super.sparkConf - - .set("spark.sql.execution.combineJoinedAggregates.enabled", "true") + override def sparkConf: SparkConf = super.sparkConf.set("spark.sql.execution.combineJoinedAggregates.enabled", "true") override def beforeAll(): Unit = { super.beforeAll() -- Gitee From 3e3770a7a625a33cbc6a471d3d10610ec19ca9cb Mon Sep 17 00:00:00 2001 From: ycsongcs Date: Tue, 15 Oct 2024 16:36:10 +0800 Subject: [PATCH 4/4] [spark-extension] bug:fix ci error --- .../sql/execution/PushOrderedLimitThroughAggSuite.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/execution/PushOrderedLimitThroughAggSuite.scala b/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/execution/PushOrderedLimitThroughAggSuite.scala index 7e6a8a999..273b1f6cb 100644 --- a/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/execution/PushOrderedLimitThroughAggSuite.scala +++ b/omnioperator/omniop-spark-extension/java/src/test/scala/org/apache/spark/sql/execution/PushOrderedLimitThroughAggSuite.scala @@ -25,9 +25,12 @@ import org.apache.spark.SparkConf class PushOrderedLimitThroughAggSuite extends ColumnarSparkPlanTest { private var df: DataFrame = _ - override def sparkConf: SparkConf = super.sparkConf.set("spark.omni.sql.columnar.topNSort", "true") - override def sparkConf: SparkConf = super.sparkConf.set("spark.omni.sql.columnar.pushOrderedLimitThroughAggEnable.enabled", "true") - override def sparkConf: SparkConf = super.sparkConf.set("spark.sql.adaptive.enabled", "true") + override def sparkConf: SparkConf = { + super.sparkConf + .set("spark.omni.sql.columnar.topNSort", "true") + .set("spark.omni.sql.columnar.pushOrderedLimitThroughAggEnable.enabled", "true") + .set("spark.sql.adaptive.enabled", "true") + } protected override def beforeAll(): Unit = { super.beforeAll() -- Gitee