Fix #8656 for spark34

apache · Feb 3, 2025 · 575d52c · 575d52c
1 parent 986642f
commit 575d52c
Show file tree

Hide file tree

Showing 5 changed files with 5 additions and 18 deletions.
diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala
@@ -16,9 +16,8 @@
  */
 package org.apache.spark.sql
 
-import org.apache.gluten.config.GlutenConfig
 import org.apache.gluten.exception.GlutenException
-import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils}
 
 import org.apache.spark.{SparkConf, SparkException, SparkThrowable}
 import org.apache.spark.ErrorMessageFormat.MINIMAL
@@ -199,7 +198,6 @@ class GlutenSQLQueryTestSuite
         .set("spark.io.compression.codec", "LZ4")
         .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
         .set("spark.gluten.sql.enable.native.validation", "false")
-        .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath)
         .set("spark.sql.files.openCostInBytes", "134217728")
         .set("spark.unsafe.exceptionOnMemoryLeak", "true")
     } else {

diff --git a/...ark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/...ark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -21,7 +21,7 @@ import org.apache.gluten.config.GlutenConfig
 import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer}
 import org.apache.gluten.extension.columnar.transition.Transitions
 import org.apache.gluten.jni.JniLibLoader
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
 
 import org.apache.spark.SparkConf
 import org.apache.spark.benchmark.Benchmark
@@ -83,7 +83,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
         .set("spark.gluten.sql.enable.native.validation", "false")
         .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
         .set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false")
-        .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath)
         .set(
           "spark.sql.catalog.spark_catalog",
           "org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog")
@@ -227,9 +226,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
 
   override def afterAll(): Unit = {
     if (BackendTestUtils.isCHBackendLoaded()) {
-      val libPath =
-        spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath)
-      JniLibLoader.unloadFromPath(libPath)
+      JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
     }
     super.afterAll()
   }

diff --git a/...park34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/...park34/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala
@@ -16,8 +16,7 @@
  */
 package org.apache.spark.sql.execution.joins
 
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
 
 import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession}
 import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ConvertToLocalRelation, NullPropagation}
@@ -64,7 +63,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite with GlutenTestsCommon
         .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
         .config("spark.gluten.sql.enable.native.validation", "false")
         .config("spark.sql.files.openCostInBytes", "134217728")
-        .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath)
         .config("spark.unsafe.exceptionOnMemoryLeak", "true")
         .getOrCreate()
     } else {

diff --git a/...park34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/...park34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -16,9 +16,6 @@
  */
 package org.apache.spark.sql.hive.execution
 
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.utils.SystemParameters
-
 import org.apache.spark.{DebugFilesystem, SparkConf}
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.TableIdentifier
@@ -28,7 +25,6 @@ class GlutenHiveSQLQueryCHSuite extends GlutenHiveSQLQuerySuiteBase {
   override def sparkConf: SparkConf = {
     defaultSparkConf
       .set("spark.plugins", "org.apache.gluten.GlutenPlugin")
-      .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath)
       .set("spark.gluten.sql.enable.native.validation", "false")
       .set("spark.gluten.sql.native.writer.enabled", "true")
       .set("spark.sql.storeAssignmentPolicy", "legacy")

diff --git a/...n-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala b/...n-ut/spark34/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala
@@ -16,9 +16,8 @@
  */
 package org.apache.spark.sql.statistics
 
-import org.apache.gluten.config.GlutenConfig
 import org.apache.gluten.execution.GlutenPlan
-import org.apache.gluten.utils.{BackendTestUtils, SystemParameters}
+import org.apache.gluten.utils.BackendTestUtils
 
 import org.apache.spark.sql.{GlutenTestConstants, QueryTest, SparkSession}
 import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ConvertToLocalRelation, NullPropagation}
@@ -63,7 +62,6 @@ class SparkFunctionStatistics extends QueryTest {
           .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1")
           .config("spark.gluten.sql.enable.native.validation", "false")
           .config("spark.sql.files.openCostInBytes", "134217728")
-          .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath)
           .config("spark.unsafe.exceptionOnMemoryLeak", "true")
           .getOrCreate()
       } else {