From c827c08d4e3bda49fd69dcd9da80b85a1d20cfa4 Mon Sep 17 00:00:00 2001 From: Chang Chen Date: Mon, 3 Feb 2025 22:14:11 +0800 Subject: [PATCH] https://github.com/apache/incubator-gluten/pull/8656 --- backends-clickhouse/pom.xml | 2 +- .../execution/GlutenClickHouseJoinSuite.scala | 3 -- ...GlutenClickHouseNativeExceptionSuite.scala | 10 +---- .../GlutenClickHouseNativeLibSuite.scala | 2 - .../GlutenClickHouseSyntheticDataSuite.scala | 2 - .../GlutenClickHouseTPCDSAbstractSuite.scala | 1 - .../GlutenClickHouseTPCHAbstractSuite.scala | 2 - ...ClickHouseWholeStageTransformerSuite.scala | 2 - .../GlutenFunctionValidateSuite.scala | 4 -- .../GlutenClickhouseFunctionSuite.scala | 2 - .../hive/GlutenClickHouseHiveTableSuite.scala | 3 -- ...lutenClickHouseNativeWriteTableSuite.scala | 2 - .../gluten/utils/UTSystemParameters.scala | 9 ----- .../benchmarks/CHSqlBasedBenchmark.scala | 6 +-- docs/developers/clickhouse-backend-debug.md | 2 +- gluten-ut/common/pom.xml | 2 +- .../gluten/utils/SystemParameters.scala | 38 ------------------- .../spark/sql/DummyFilterColmnarHelper.scala | 4 +- .../spark/sql/GlutenSQLTestsBaseTrait.scala | 4 +- .../apache/spark/sql/GlutenTestsTrait.scala | 3 +- gluten-ut/spark32/pom.xml | 5 +-- .../spark/sql/GlutenSQLQueryTestSuite.scala | 4 +- .../benchmarks/ParquetReadBenchmark.scala | 7 +--- .../joins/GlutenBroadcastJoinSuite.scala | 4 +- .../execution/GlutenHiveSQLQueryCHSuite.scala | 4 -- .../statistics/SparkFunctionStatistics.scala | 4 +- gluten-ut/spark33/pom.xml | 5 +-- .../spark/sql/GlutenSQLQueryTestSuite.scala | 4 +- .../benchmarks/ParquetReadBenchmark.scala | 7 +--- .../joins/GlutenBroadcastJoinSuite.scala | 4 +- .../execution/GlutenHiveSQLQueryCHSuite.scala | 4 -- .../statistics/SparkFunctionStatistics.scala | 4 +- gluten-ut/spark34/pom.xml | 3 +- gluten-ut/spark35/pom.xml | 3 +- .../spark/sql/GlutenSQLQueryTestSuite.scala | 4 +- .../benchmarks/ParquetReadBenchmark.scala | 7 +--- .../joins/GlutenBroadcastJoinSuite.scala | 4 +- .../execution/GlutenHiveSQLQueryCHSuite.scala | 4 -- .../statistics/SparkFunctionStatistics.scala | 4 +- gluten-ut/test/pom.xml | 5 +-- ...MergeTwoPhasesHashBaseAggregateSuite.scala | 4 +- .../GlutenExpressionMappingSuite.scala | 3 +- .../GlutenExtensionRewriteRuleSuite.scala | 4 +- .../org/apache/gluten/sql/SQLQuerySuite.scala | 2 - .../GlutenExpressionDataTypesValidation.scala | 4 +- .../GlutenNoopWriterRuleSuite.scala | 9 +---- pom.xml | 2 - 47 files changed, 33 insertions(+), 188 deletions(-) delete mode 100644 gluten-ut/common/src/test/scala/org/apache/gluten/utils/SystemParameters.scala diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index 52c88354a195..07166dbeb61b 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -492,7 +492,7 @@ - ${clickhouse.lib.path} + ${spark.gluten.sql.columnar.libpath} ${tpcds.data.path} diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala index 28aad5537903..efde239f4357 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala @@ -17,8 +17,6 @@ package org.apache.gluten.execution import org.apache.gluten.backendsapi.clickhouse.CHConf -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd} @@ -43,7 +41,6 @@ class GlutenClickHouseJoinSuite extends GlutenClickHouseWholeStageTransformerSui .set("spark.sql.adaptive.enabled", "false") .set("spark.sql.files.minPartitionNum", "1") .set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .set("spark.gluten.sql.columnar.iterator", "true") .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala index 9b617b998c6c..394895fdc8aa 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeExceptionSuite.scala @@ -16,18 +16,10 @@ */ package org.apache.gluten.execution -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.{TestExceptionUtils, UTSystemParameters} - -import org.apache.spark.SparkConf +import org.apache.gluten.utils.TestExceptionUtils class GlutenClickHouseNativeExceptionSuite extends GlutenClickHouseWholeStageTransformerSuite { - override protected def sparkConf: SparkConf = { - super.sparkConf - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) - } - test("native exception caught by jvm") { try { TestExceptionUtils.generateNativeException() diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala index 98d8fb9ee06b..f91e84194259 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseNativeLibSuite.scala @@ -18,7 +18,6 @@ package org.apache.gluten.execution import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException -import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.sql.SparkSession @@ -65,7 +64,6 @@ class GlutenClickHouseNativeLibSuite extends PlanTest { .builder() .master("local[1]") .config(baseSparkConf) - .config(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .config(GlutenConfig.GLUTEN_EXECUTOR_LIB_PATH.key, "/path/not/exist/libch.so") .getOrCreate() spark.sql("select 1").show() diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala index 6fffe30f05d8..25244c58618e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala @@ -17,7 +17,6 @@ package org.apache.gluten.execution import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.internal.Logging @@ -52,7 +51,6 @@ class GlutenClickHouseSyntheticDataSuite .set("spark.databricks.delta.properties.defaults.checkpointInterval", "5") .set("spark.databricks.delta.stalenessLimit", "3600000") .set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .set("spark.gluten.sql.columnar.iterator", "true") .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala index f1f93692ecb0..d15ac1334cfb 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala @@ -132,7 +132,6 @@ abstract class GlutenClickHouseTPCDSAbstractSuite .set("spark.databricks.delta.properties.defaults.checkpointInterval", "5") .set("spark.databricks.delta.stalenessLimit", "3600000") .set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .set("spark.gluten.sql.columnar.iterator", "true") .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala index 4c748fcb1d06..6a7b35337ddc 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala @@ -17,7 +17,6 @@ package org.apache.gluten.execution import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.internal.Logging @@ -568,7 +567,6 @@ abstract class GlutenClickHouseTPCHAbstractSuite .set("spark.databricks.delta.stalenessLimit", "3600000") .set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") .set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .set("spark.gluten.sql.columnar.iterator", "true") .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala index db5fede0d5ae..24efab534319 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala @@ -17,7 +17,6 @@ package org.apache.gluten.execution import org.apache.gluten.backendsapi.clickhouse.RuntimeConfig -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf} @@ -79,7 +78,6 @@ class GlutenClickHouseWholeStageTransformerSuite extends WholeStageTransformerSu import org.apache.gluten.backendsapi.clickhouse.CHConf._ val conf = super.sparkConf - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.sql.warehouse.dir", warehouse) .setCHConfig("user_defined_path", "/tmp/user_defined") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala index 5923f1484ae5..e14666976e6e 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala @@ -16,9 +16,6 @@ */ package org.apache.gluten.execution -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.UTSystemParameters - import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, Row, TestUtils} import org.apache.spark.sql.catalyst.expressions.{Expression, GetJsonObject, Literal} @@ -57,7 +54,6 @@ class GlutenFunctionValidateSuite extends GlutenClickHouseWholeStageTransformerS .set("spark.databricks.delta.properties.defaults.checkpointInterval", "5") .set("spark.databricks.delta.stalenessLimit", "3600000") .set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .set("spark.gluten.sql.columnar.iterator", "true") .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala index eac4c54cfc7a..5ef00623c392 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala @@ -18,7 +18,6 @@ package org.apache.gluten.execution.compatibility import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{GlutenClickHouseTPCHAbstractSuite, ProjectExecTransformer} -import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig @@ -51,7 +50,6 @@ class GlutenClickhouseFunctionSuite extends GlutenClickHouseTPCHAbstractSuite { .set("spark.databricks.delta.properties.defaults.checkpointInterval", "5") .set("spark.databricks.delta.stalenessLimit", "3600000") .set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .set("spark.gluten.sql.columnar.iterator", "true") .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala index 917317bcc65d..dddeb43d6a38 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala @@ -16,10 +16,8 @@ */ package org.apache.gluten.execution.hive -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, GlutenClickHouseWholeStageTransformerSuite, ProjectExecTransformer, TransformSupport} import org.apache.gluten.test.AllDataTypesWithComplexType -import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, SaveMode} @@ -56,7 +54,6 @@ class GlutenClickHouseHiveTableSuite .set("spark.sql.adaptive.enabled", "false") .set("spark.sql.files.minPartitionNum", "1") .set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .set("spark.gluten.sql.columnar.iterator", "true") .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala index b6067f138d07..1ee0b18b1194 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala @@ -20,7 +20,6 @@ import org.apache.gluten.backendsapi.clickhouse.RuntimeConfig import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenClickHouseWholeStageTransformerSuite import org.apache.gluten.test.AllDataTypesWithComplexType.genTestData -import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.gluten.NativeWriteChecker @@ -57,7 +56,6 @@ class GlutenClickHouseNativeWriteTableSuite .set("spark.databricks.delta.properties.defaults.checkpointInterval", "5") .set("spark.databricks.delta.stalenessLimit", "3600000") .set(ClickHouseConfig.CLICKHOUSE_WORKER_ID, "1") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .set("spark.gluten.sql.columnar.iterator", "true") .set("spark.gluten.sql.columnar.hashagg.enablefinal", "true") .set("spark.gluten.sql.enable.native.validation", "false") diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/utils/UTSystemParameters.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/utils/UTSystemParameters.scala index 334f9fb4bec2..f6a31f175d09 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/utils/UTSystemParameters.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/utils/UTSystemParameters.scala @@ -18,15 +18,6 @@ package org.apache.gluten.utils object UTSystemParameters { - private val CLICKHOUSE_LIB_PATH_KEY = "clickhouse.lib.path" - private val CLICKHOUSE_LIB_PATH_DEFAULT_VALUE = "/usr/local/clickhouse/lib/libch.so" - - def clickHouseLibPath: String = { - System.getProperty( - UTSystemParameters.CLICKHOUSE_LIB_PATH_KEY, - UTSystemParameters.CLICKHOUSE_LIB_PATH_DEFAULT_VALUE) - } - private val TEST_DATA_PATH_KEY = "gluten.test.data.path" private val TEST_DATA_PATH_DEFAULT_VALUE = "/data" diff --git a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala index 7b067643f68d..2a9a2b233a1f 100644 --- a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala +++ b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.execution.benchmarks import org.apache.gluten.config.GlutenConfig import org.apache.gluten.jni.JniLibLoader -import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.sql.delta.DeltaLog @@ -32,7 +31,6 @@ trait CHSqlBasedBenchmark extends SqlBasedBenchmark { def getSparkConf: SparkConf = { val conf = new SparkConf() .setAppName(appName) - .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) .setIfMissing("spark.master", s"local[$thrdNum]") .set("spark.plugins", "org.apache.gluten.GlutenPlugin") .set( @@ -57,9 +55,7 @@ trait CHSqlBasedBenchmark extends SqlBasedBenchmark { override def afterAll(): Unit = { DeltaLog.clearCache() - val libPath = - spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key, UTSystemParameters.clickHouseLibPath) - JniLibLoader.unloadFromPath(libPath) + JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key)) // Wait for Ctrl+C, convenient for seeing Spark UI // Thread.sleep(600000) super.afterAll() diff --git a/docs/developers/clickhouse-backend-debug.md b/docs/developers/clickhouse-backend-debug.md index 6f27b25a09d6..1d7249a1c224 100644 --- a/docs/developers/clickhouse-backend-debug.md +++ b/docs/developers/clickhouse-backend-debug.md @@ -31,7 +31,7 @@ parent: /developer-overview/ ![gluten-debug-idea-config.png](../image/ClickHouse/gluten-debug-idea-config.png) VM Options: - `-Dtpcds.data.path=/data/tpcds-data-sf1 -Dclickhouse.lib.path=/path/to/gluten/cpp-ch/build/utils/extern-local-engine/libch.so -Dspark.test.home=/path/to/spark33` + `-Dtpcds.data.path=/data/tpcds-data-sf1 -Dspark.gluten.sql.columnar.libpath=/path/to/gluten/cpp-ch/build/utils/extern-local-engine/libch.so -Dspark.test.home=/path/to/spark33` > Download tpcds-data in https://gluten-nginx.kyligence.com/dataset/ > Download spark33 using `git clone --depth 1 --branch v3.3.1 https://github.com/apache/spark.git /tmp/spark33` diff --git a/gluten-ut/common/pom.xml b/gluten-ut/common/pom.xml index 11caaa4dd4ff..7d8d6684921a 100644 --- a/gluten-ut/common/pom.xml +++ b/gluten-ut/common/pom.xml @@ -71,7 +71,7 @@ - ${clickhouse.lib.path} + ${spark.gluten.sql.columnar.libpath} ${tpcds.data.path} diff --git a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/SystemParameters.scala b/gluten-ut/common/src/test/scala/org/apache/gluten/utils/SystemParameters.scala deleted file mode 100644 index 71fc26227baa..000000000000 --- a/gluten-ut/common/src/test/scala/org/apache/gluten/utils/SystemParameters.scala +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.utils - -object SystemParameters { - - val CLICKHOUSE_LIB_PATH_KEY = "clickhouse.lib.path" - val CLICKHOUSE_LIB_PATH_DEFAULT_VALUE = "/usr/local/clickhouse/lib/libch.so" - - val TPCDS_DATA_PATH_KEY = "tpcds.data.path" - val TPCDS_DATA_PATH_DEFAULT_VALUE = "/data/tpcds-data-sf1" - - def getClickHouseLibPath: String = { - System.getProperty( - SystemParameters.CLICKHOUSE_LIB_PATH_KEY, - SystemParameters.CLICKHOUSE_LIB_PATH_DEFAULT_VALUE) - } - - def getTpcdsDataPath: String = { - System.getProperty( - SystemParameters.TPCDS_DATA_PATH_KEY, - SystemParameters.TPCDS_DATA_PATH_DEFAULT_VALUE) - } -} diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala index ab4c6de441db..41c652fc60d7 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/DummyFilterColmnarHelper.scala @@ -16,8 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow @@ -86,7 +85,6 @@ object DummyFilterColmnarHelper { .config("spark.memory.offHeap.size", "1024MB") .config("spark.plugins", "org.apache.gluten.GlutenPlugin") .config("spark.shuffle.manager", "org.apache.spark.shuffle.sort.ColumnarShuffleManager") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.io.compression.codec", "LZ4") .config("spark.gluten.sql.enable.native.validation", "false") } else { diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala index 128ce144f2ea..b6ece98e4af7 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsBaseTrait.scala @@ -16,8 +16,7 @@ */ package org.apache.spark.sql -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.SparkConf import org.apache.spark.sql.execution.SparkPlan @@ -105,7 +104,6 @@ object GlutenSQLTestsBaseTrait { .set("spark.io.compression.codec", "LZ4") .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set("spark.sql.files.openCostInBytes", "134217728") .set("spark.unsafe.exceptionOnMemoryLeak", "true") } else { diff --git a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala index 330bf3b7ac89..5e0686095c80 100644 --- a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala +++ b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala @@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.gluten.test.TestStats -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.GlutenQueryTestUtil.isNaNOrInf import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} @@ -113,7 +113,6 @@ trait GlutenTestsTrait extends GlutenTestsCommonTrait { .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .config("spark.gluten.sql.enable.native.validation", "false") .config("spark.sql.files.openCostInBytes", "134217728") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .config(GlutenConfig.UT_STATISTIC.key, "true") .getOrCreate() diff --git a/gluten-ut/spark32/pom.xml b/gluten-ut/spark32/pom.xml index 8d0d0f4fcae2..3ecdfa41603e 100644 --- a/gluten-ut/spark32/pom.xml +++ b/gluten-ut/spark32/pom.xml @@ -58,9 +58,6 @@ test - - - @@ -107,7 +104,7 @@ - ${clickhouse.lib.path} + ${spark.gluten.sql.columnar.libpath} ${tpcds.data.path} diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala index 739446111bd9..7a4b1c781674 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException -import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils} import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator @@ -195,7 +194,6 @@ class GlutenSQLQueryTestSuite .set("spark.io.compression.codec", "LZ4") .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set("spark.sql.files.openCostInBytes", "134217728") .set("spark.unsafe.exceptionOnMemoryLeak", "true") } else { diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index c55f4bf7f0da..aa561f17f0c1 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -20,7 +20,7 @@ import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.jni.JniLibLoader -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.SparkConf import org.apache.spark.benchmark.Benchmark @@ -82,7 +82,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false") - .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog") @@ -225,9 +224,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { override def afterAll(): Unit = { if (BackendTestUtils.isCHBackendLoaded()) { - val libPath = - spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) - JniLibLoader.unloadFromPath(libPath) + JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key)) } super.afterAll() } diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala index 838d30e7ca67..7d83405ebaf4 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{BroadcastHashJoinExecTransformerBase, BroadcastNestedLoopJoinExecTransformer, ColumnarToRowExecBase, WholeStageTransformer} -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession} import org.apache.spark.sql.catalyst.optimizer._ @@ -89,7 +88,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite with GlutenTestsCommon .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .config("spark.gluten.sql.enable.native.validation", "false") .config("spark.sql.files.openCostInBytes", "134217728") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .getOrCreate() } else { diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala index 0b14467aba80..859e6df48459 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala @@ -16,9 +16,6 @@ */ package org.apache.spark.sql.hive.execution -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.SystemParameters - import org.apache.spark.{DebugFilesystem, SparkConf} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier @@ -28,7 +25,6 @@ class GlutenHiveSQLQueryCHSuite extends GlutenHiveSQLQuerySuiteBase { override def sparkConf: SparkConf = { defaultSparkConf .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.gluten.sql.native.writer.enabled", "true") .set("spark.sql.storeAssignmentPolicy", "legacy") diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala index be9063c83966..925d60df8eab 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql.statistics -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPlan -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{GlutenTestConstants, QueryTest, SparkSession} import org.apache.spark.sql.catalyst.analysis.FunctionRegistry @@ -64,7 +63,6 @@ class SparkFunctionStatistics extends QueryTest { .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .config("spark.gluten.sql.enable.native.validation", "false") .config("spark.sql.files.openCostInBytes", "134217728") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .getOrCreate() } else { diff --git a/gluten-ut/spark33/pom.xml b/gluten-ut/spark33/pom.xml index f8377195eec9..a4abb9f36505 100644 --- a/gluten-ut/spark33/pom.xml +++ b/gluten-ut/spark33/pom.xml @@ -87,9 +87,6 @@ test - - - @@ -135,7 +132,7 @@ - ${clickhouse.lib.path} + ${spark.gluten.sql.columnar.libpath} ${tpcds.data.path} diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala index 84d677ef596f..e336ca88a788 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException -import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils} import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator @@ -195,7 +194,6 @@ class GlutenSQLQueryTestSuite .set("spark.io.compression.codec", "LZ4") .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set("spark.sql.files.openCostInBytes", "134217728") .set("spark.unsafe.exceptionOnMemoryLeak", "true") } else { diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index aa5ee0a881ec..517f17bba927 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -20,7 +20,7 @@ import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.jni.JniLibLoader -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.SparkConf import org.apache.spark.benchmark.Benchmark @@ -82,7 +82,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false") - .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog") @@ -225,9 +224,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { override def afterAll(): Unit = { if (BackendTestUtils.isCHBackendLoaded()) { - val libPath = - spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) - JniLibLoader.unloadFromPath(libPath) + JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key)) } super.afterAll() } diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala index f5fd73079619..c8b371f48fb8 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala @@ -16,8 +16,7 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession} import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ConvertToLocalRelation, NullPropagation} @@ -66,7 +65,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite with GlutenTestsCommon .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .config("spark.gluten.sql.enable.native.validation", "false") .config("spark.sql.files.openCostInBytes", "134217728") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .getOrCreate() } else { diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala index 0b14467aba80..859e6df48459 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala @@ -16,9 +16,6 @@ */ package org.apache.spark.sql.hive.execution -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.SystemParameters - import org.apache.spark.{DebugFilesystem, SparkConf} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier @@ -28,7 +25,6 @@ class GlutenHiveSQLQueryCHSuite extends GlutenHiveSQLQuerySuiteBase { override def sparkConf: SparkConf = { defaultSparkConf .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.gluten.sql.native.writer.enabled", "true") .set("spark.sql.storeAssignmentPolicy", "legacy") diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala index 69969e948e8c..c349d9ca9184 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql.statistics -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPlan -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{GlutenTestConstants, QueryTest, SparkSession} import org.apache.spark.sql.catalyst.analysis.FunctionRegistry @@ -64,7 +63,6 @@ class SparkFunctionStatistics extends QueryTest { .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .config("spark.gluten.sql.enable.native.validation", "false") .config("spark.sql.files.openCostInBytes", "134217728") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .getOrCreate() } else { diff --git a/gluten-ut/spark34/pom.xml b/gluten-ut/spark34/pom.xml index e61142b79df8..16f47667986b 100644 --- a/gluten-ut/spark34/pom.xml +++ b/gluten-ut/spark34/pom.xml @@ -87,7 +87,6 @@ 2.0.6 2.19.0 - @@ -134,7 +133,7 @@ - ${clickhouse.lib.path} + ${spark.gluten.sql.columnar.libpath} ${tpcds.data.path} diff --git a/gluten-ut/spark35/pom.xml b/gluten-ut/spark35/pom.xml index d95e4aeeee05..8510148b0e28 100644 --- a/gluten-ut/spark35/pom.xml +++ b/gluten-ut/spark35/pom.xml @@ -148,7 +148,6 @@ 2.0.6 2.19.0 - @@ -217,7 +216,7 @@ - ${clickhouse.lib.path} + ${spark.gluten.sql.columnar.libpath} ${tpcds.data.path} diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala index 3eebda83a0d1..01d36635f357 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenSQLQueryTestSuite.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.exception.GlutenException -import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.{BackendTestSettings, BackendTestUtils} import org.apache.spark.{SparkConf, SparkException, SparkThrowable} import org.apache.spark.ErrorMessageFormat.MINIMAL @@ -197,7 +196,6 @@ class GlutenSQLQueryTestSuite .set("spark.io.compression.codec", "LZ4") .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set("spark.sql.files.openCostInBytes", "134217728") .set("spark.unsafe.exceptionOnMemoryLeak", "true") } else { diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala index aa5ee0a881ec..517f17bba927 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala @@ -20,7 +20,7 @@ import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{FileSourceScanExecTransformer, WholeStageTransformer} import org.apache.gluten.extension.columnar.transition.Transitions import org.apache.gluten.jni.JniLibLoader -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.SparkConf import org.apache.spark.benchmark.Benchmark @@ -82,7 +82,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .set("spark.gluten.sql.columnar.separate.scan.rdd.for.ch", "false") - .setIfMissing(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set( "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseSparkCatalog") @@ -225,9 +224,7 @@ object ParquetReadBenchmark extends SqlBasedBenchmark { override def afterAll(): Unit = { if (BackendTestUtils.isCHBackendLoaded()) { - val libPath = - spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) - JniLibLoader.unloadFromPath(libPath) + JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key)) } super.afterAll() } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala index f5fd73079619..c8b371f48fb8 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/joins/GlutenBroadcastJoinSuite.scala @@ -16,8 +16,7 @@ */ package org.apache.spark.sql.execution.joins -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{GlutenTestsCommonTrait, SparkSession} import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ConvertToLocalRelation, NullPropagation} @@ -66,7 +65,6 @@ class GlutenBroadcastJoinSuite extends BroadcastJoinSuite with GlutenTestsCommon .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .config("spark.gluten.sql.enable.native.validation", "false") .config("spark.sql.files.openCostInBytes", "134217728") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .getOrCreate() } else { diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala index 0b14467aba80..859e6df48459 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala @@ -16,9 +16,6 @@ */ package org.apache.spark.sql.hive.execution -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.SystemParameters - import org.apache.spark.{DebugFilesystem, SparkConf} import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.TableIdentifier @@ -28,7 +25,6 @@ class GlutenHiveSQLQueryCHSuite extends GlutenHiveSQLQuerySuiteBase { override def sparkConf: SparkConf = { defaultSparkConf .set("spark.plugins", "org.apache.gluten.GlutenPlugin") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .set("spark.gluten.sql.enable.native.validation", "false") .set("spark.gluten.sql.native.writer.enabled", "true") .set("spark.sql.storeAssignmentPolicy", "legacy") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala index f483d714f12b..f45b5c659c5f 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/statistics/SparkFunctionStatistics.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql.statistics -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.GlutenPlan -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.sql.{GlutenTestConstants, QueryTest, SparkSession} import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ConvertToLocalRelation, NullPropagation} @@ -63,7 +62,6 @@ class SparkFunctionStatistics extends QueryTest { .config("spark.gluten.sql.columnar.backend.ch.worker.id", "1") .config("spark.gluten.sql.enable.native.validation", "false") .config("spark.sql.files.openCostInBytes", "134217728") - .config(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) .config("spark.unsafe.exceptionOnMemoryLeak", "true") .getOrCreate() } else { diff --git a/gluten-ut/test/pom.xml b/gluten-ut/test/pom.xml index a641a19b2faf..9ec4432469d2 100644 --- a/gluten-ut/test/pom.xml +++ b/gluten-ut/test/pom.xml @@ -97,9 +97,6 @@ test - - - @@ -145,7 +142,7 @@ - ${clickhouse.lib.path} + ${spark.gluten.sql.columnar.libpath} ${tpcds.data.path} diff --git a/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala b/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala index 32eb10a32fad..061f5660e375 100644 --- a/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala +++ b/gluten-ut/test/src/test/scala/org/apache/gluten/execution/MergeTwoPhasesHashBaseAggregateSuite.scala @@ -16,8 +16,7 @@ */ package org.apache.gluten.execution -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.SparkConf import org.apache.spark.sql.DataFrame @@ -57,7 +56,6 @@ abstract class BaseMergeTwoPhasesHashBaseAggregateSuite extends WholeStageTransf if (BackendTestUtils.isCHBackendLoaded()) { conf .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) } conf } diff --git a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala index 14a07aea5bf7..b2fbd11c0abc 100644 --- a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala +++ b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala @@ -19,7 +19,7 @@ package org.apache.gluten.expressions import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.ProjectExecTransformer import org.apache.gluten.expression.ExpressionMappings -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.SparkConf import org.apache.spark.sql.{GlutenQueryTest, Row} @@ -43,7 +43,6 @@ class GlutenExpressionMappingSuite if (BackendTestUtils.isCHBackendLoaded()) { conf .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) } conf } diff --git a/gluten-ut/test/src/test/scala/org/apache/gluten/extension/GlutenExtensionRewriteRuleSuite.scala b/gluten-ut/test/src/test/scala/org/apache/gluten/extension/GlutenExtensionRewriteRuleSuite.scala index 0068e83a6df3..a29550815041 100644 --- a/gluten-ut/test/src/test/scala/org/apache/gluten/extension/GlutenExtensionRewriteRuleSuite.scala +++ b/gluten-ut/test/src/test/scala/org/apache/gluten/extension/GlutenExtensionRewriteRuleSuite.scala @@ -16,9 +16,8 @@ */ package org.apache.gluten.extension -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{ProjectExecTransformer, WholeStageTransformerSuite} -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.SparkConf @@ -33,7 +32,6 @@ class GlutenExtensionRewriteRuleSuite extends WholeStageTransformerSuite { if (BackendTestUtils.isCHBackendLoaded()) { conf .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) } conf } diff --git a/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala b/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala index 74d0e44db72e..c584852b9110 100644 --- a/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala +++ b/gluten-ut/test/src/test/scala/org/apache/gluten/sql/SQLQuerySuite.scala @@ -19,7 +19,6 @@ package org.apache.gluten.sql import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.WholeStageTransformerSuite import org.apache.gluten.utils.BackendTestUtils -import org.apache.gluten.utils.SystemParameters import org.apache.spark.SparkConf import org.apache.spark.sql.Row @@ -41,7 +40,6 @@ class SQLQuerySuite extends WholeStageTransformerSuite { if (BackendTestUtils.isCHBackendLoaded()) { conf .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) } conf } diff --git a/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala index adf72a3f6331..9bea13703793 100644 --- a/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala +++ b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala @@ -16,9 +16,8 @@ */ package org.apache.spark.sql -import org.apache.gluten.config.GlutenConfig import org.apache.gluten.execution.{ProjectExecTransformer, TransformSupport, WholeStageTransformerSuite} -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} +import org.apache.gluten.utils.BackendTestUtils import org.apache.spark.SparkConf import org.apache.spark.rdd.RDD @@ -46,7 +45,6 @@ class GlutenExpressionDataTypesValidation extends WholeStageTransformerSuite { if (BackendTestUtils.isCHBackendLoaded()) { conf .set("spark.gluten.sql.enable.native.validation", "false") - .set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) } conf } diff --git a/gluten-ut/test/src/test/scala/org/apache/spark/sql/datasources/GlutenNoopWriterRuleSuite.scala b/gluten-ut/test/src/test/scala/org/apache/spark/sql/datasources/GlutenNoopWriterRuleSuite.scala index b01b543006e8..b2a3a2ba8ae6 100644 --- a/gluten-ut/test/src/test/scala/org/apache/spark/sql/datasources/GlutenNoopWriterRuleSuite.scala +++ b/gluten-ut/test/src/test/scala/org/apache/spark/sql/datasources/GlutenNoopWriterRuleSuite.scala @@ -16,9 +16,6 @@ */ package org.apache.spark.sql.datasources -import org.apache.gluten.config.GlutenConfig -import org.apache.gluten.utils.{BackendTestUtils, SystemParameters} - import org.apache.spark.SparkConf import org.apache.spark.sql.{GlutenQueryTest, SaveMode} import org.apache.spark.sql.execution.QueryExecution @@ -29,17 +26,13 @@ import org.apache.spark.sql.util.QueryExecutionListener class GlutenNoopWriterRuleSuite extends GlutenQueryTest with SharedSparkSession { override def sparkConf: SparkConf = { - val conf = super.sparkConf + super.sparkConf .set("spark.plugins", "org.apache.gluten.GlutenPlugin") .set("spark.default.parallelism", "1") .set("spark.memory.offHeap.enabled", "true") .set("spark.memory.offHeap.size", "1024MB") .set("spark.ui.enabled", "false") .set("spark.gluten.ui.enabled", "false") - if (BackendTestUtils.isCHBackendLoaded()) { - conf.set(GlutenConfig.GLUTEN_LIB_PATH.key, SystemParameters.getClickHouseLibPath) - } - conf } class WriterColumnarListener extends QueryExecutionListener { diff --git a/pom.xml b/pom.xml index 07cd0d5f3ad2..2ea50cfa8ea9 100644 --- a/pom.xml +++ b/pom.xml @@ -82,8 +82,6 @@ all - /usr/local/clickhouse/lib/libch.so - /data/tpcds-data-sf1 2.13.5 4.13.1