From e2b73c1230f5fc6a4d82f649ee45f6b553b7fece Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Wed, 15 Jan 2025 20:04:35 +0800 Subject: [PATCH] unified cache --- .../workflows/backend-integration-test-action.yml | 2 +- spark-connector/spark-common/build.gradle.kts | 13 +++++++++++++ .../connector/catalog/GravitinoCatalogManager.java | 11 +++++------ spark-connector/v3.3/spark/build.gradle.kts | 2 +- spark-connector/v3.4/spark/build.gradle.kts | 2 +- spark-connector/v3.5/spark/build.gradle.kts | 2 +- 6 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/backend-integration-test-action.yml b/.github/workflows/backend-integration-test-action.yml index b15c5d226ca..dc6594bf705 100644 --- a/.github/workflows/backend-integration-test-action.yml +++ b/.github/workflows/backend-integration-test-action.yml @@ -57,7 +57,7 @@ jobs: id: integrationTest run: > ./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false - -x :web:web:test -x :web:integration-test:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:spark-common:test + -x :web:web:test -x :web:integration-test:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:spark-common:test -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test diff --git a/spark-connector/spark-common/build.gradle.kts b/spark-connector/spark-common/build.gradle.kts index 06e0077d21e..3e12aaabb77 100644 --- a/spark-connector/spark-common/build.gradle.kts +++ b/spark-connector/spark-common/build.gradle.kts @@ -16,6 +16,8 @@ * specific language governing permissions and limitations * under the License. */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + plugins { `maven-publish` id("java") @@ -40,6 +42,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat dependencies { implementation(project(":catalogs:catalog-common")) implementation(libs.guava) + implementation(libs.caffeine) compileOnly(project(":clients:client-java-runtime", configuration = "shadow")) compileOnly("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion") @@ -137,6 +140,16 @@ dependencies { testRuntimeOnly(libs.junit.jupiter.engine) } +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + + // Relocate dependencies to avoid conflicts + relocate("com.google", "org.apache.gravitino.shaded.com.google") + relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine") +} + tasks.test { val skipITs = project.hasProperty("skipITs") if (skipITs) { diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java index f04193f33a3..a484972656a 100644 --- a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java @@ -18,13 +18,12 @@ */ package org.apache.gravitino.spark.connector.catalog; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import java.util.Arrays; import java.util.Map; -import java.util.concurrent.ExecutionException; import org.apache.gravitino.Catalog; import org.apache.gravitino.client.GravitinoClient; import org.slf4j.Logger; @@ -42,7 +41,7 @@ public class GravitinoCatalogManager { private GravitinoCatalogManager(Supplier clientBuilder) { this.gravitinoClient = clientBuilder.get(); // Will not evict catalog by default - this.gravitinoCatalogs = CacheBuilder.newBuilder().build(); + this.gravitinoCatalogs = Caffeine.newBuilder().build(); } public static GravitinoCatalogManager create(Supplier clientBuilder) { @@ -69,8 +68,8 @@ public void close() { public Catalog getGravitinoCatalogInfo(String name) { try { - return gravitinoCatalogs.get(name, () -> loadCatalog(name)); - } catch (ExecutionException e) { + return gravitinoCatalogs.get(name, catalogName -> loadCatalog(catalogName)); + } catch (Exception e) { LOG.error(String.format("Load catalog %s failed", name), e); throw new RuntimeException(e); } diff --git a/spark-connector/v3.3/spark/build.gradle.kts b/spark-connector/v3.3/spark/build.gradle.kts index 66c65f863b9..a1cae53e917 100644 --- a/spark-connector/v3.3/spark/build.gradle.kts +++ b/spark-connector/v3.3/spark/build.gradle.kts @@ -38,7 +38,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion" dependencies { - implementation(project(":spark-connector:spark-common")) + implementation(project(":spark-connector:spark-common", configuration = "shadow")) compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion") compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") { exclude("com.fasterxml.jackson") diff --git a/spark-connector/v3.4/spark/build.gradle.kts b/spark-connector/v3.4/spark/build.gradle.kts index aa4134a3c71..18c103619f1 100644 --- a/spark-connector/v3.4/spark/build.gradle.kts +++ b/spark-connector/v3.4/spark/build.gradle.kts @@ -38,7 +38,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion" dependencies { - implementation(project(":spark-connector:spark-common")) + implementation(project(":spark-connector:spark-common", configuration = "shadow")) compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion") compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") { exclude("com.fasterxml.jackson") diff --git a/spark-connector/v3.5/spark/build.gradle.kts b/spark-connector/v3.5/spark/build.gradle.kts index 15aa018081d..9e26dd21451 100644 --- a/spark-connector/v3.5/spark/build.gradle.kts +++ b/spark-connector/v3.5/spark/build.gradle.kts @@ -39,7 +39,7 @@ val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion dependencies { implementation(project(":spark-connector:spark-3.4")) - implementation(project(":spark-connector:spark-common")) + implementation(project(":spark-connector:spark-common", configuration = "shadow")) compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion") compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") { exclude("com.fasterxml.jackson")