From c493dad8fbf43e9c40139ff5bbba429efcfb71ae Mon Sep 17 00:00:00 2001 From: Chang Chen Date: Tue, 4 Feb 2025 00:44:10 +0800 Subject: [PATCH] Fix https://github.com/apache/incubator-gluten/pull/8656 for spark34 --- .../org/apache/spark/sql/GlutenSQLQueryTestSuite.scala | 4 +--- .../sql/execution/benchmarks/ParquetReadBenchmark.scala | 7 ++----- .../sql/execution/joins/GlutenBroadcastJoinSuite.scala | 4 +--- .../sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala | 4 ---- .../spark/sql/statistics/SparkFunctionStatistics.scala | 4 +--- 5 files changed, 5 insertions(+), 18 deletions(-) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala index 7c0e9801c2de..39a36898af6d 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException -import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils} import org.apache.spark.{SparkConf, SparkException, SparkThrowable} import org.apache.spark.ErrorMessageFormat.MINIMAL @@ -199,7 +198,6 @@ class GlutenSQLQueryTestSuite .set("spark.io.compression.codec", "LZ4") .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set("spark.sql.files.openCostInBytes", "134217728") .set("spark.unsafe.exceptionOnMemoryLeak", "true") } else { diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index d065495b35d6..62eb3ec8a36e 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -21,7 +21,7 @@ import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.jni.JniLibLoader -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.SparkConf import org.apache.spark.benchmark.Benchmark @@ -83,7 +83,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false") - .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog") @@ -227,9 +226,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { override def afterAll(): Unit = { if (BackendTestUtils.isCHBackendLoaded()) { - val libPath = - spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) - JniLibLoader.unloadFromPath(libPath) + JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key)) } super.afterAll() } diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala index e9d5f9bbdc37..6543160f1ac9 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala @@ -16,8 +16,7 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession} import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ConvertToLocalRelation, NullPropagation} @@ -64,7 +63,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite with GlutenTestsCommon .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .config("spark.gluten.sql.enable.native.validation", "false") .config("spark.sql.files.openCostInBytes", "134217728") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .getOrCreate() } else { diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala index 0b14467aba80..859e6df48459 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala @@ -16,9 +16,6 @@ */ package org.apache.spark.sql.hive.execution -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.SystemParameters - import org.apache.spark.{DebugFilesystem, SparkConf} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier @@ -28,7 +25,6 @@ class GlutenHiveSQLQueryCHSuite extends GlutenHiveSQLQuerySuiteBase { override def sparkConf: SparkConf = { defaultSparkConf .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.gluten.sql.native.writer.enabled", "true") .set("spark.sql.storeAssignmentPolicy", "legacy") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala index f483d714f12b..f45b5c659c5f 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql.statistics -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPlan -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{GlutenTestConstants, QueryTest, SparkSession} import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ConvertToLocalRelation, NullPropagation} @@ -63,7 +62,6 @@ class SparkFunctionStatistics extends QueryTest { .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .config("spark.gluten.sql.enable.native.validation", "false") .config("spark.sql.files.openCostInBytes", "134217728") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .getOrCreate() } else {