unified cache

apache · Jan 17, 2025 · 8fab887 · 8fab887
1 parent 39ad18a
commit 8fab887
Show file tree

Hide file tree

Showing 8 changed files with 58 additions and 14 deletions.
diff --git a/.github/workflows/backend-integration-test-action.yml b/.github/workflows/backend-integration-test-action.yml
@@ -57,7 +57,7 @@ jobs:
         id: integrationTest
         run: >
           ./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false
-          -x :web:web:test -x :web:integration-test:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:spark-common:test 
+          -x :web:web:test -x :web:integration-test:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:spark-common:test -x :spark-connector:spark-common-runtime:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
           -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test

diff --git a/settings.gradle.kts b/settings.gradle.kts
@@ -59,7 +59,7 @@ include("iceberg:iceberg-common")
 include("iceberg:iceberg-rest-server")
 include("authorizations:authorization-ranger", "authorizations:authorization-common", "authorizations:authorization-chain")
 include("trino-connector:trino-connector", "trino-connector:integration-test")
-include("spark-connector:spark-common")
+include("spark-connector:spark-common", "spark-connector:spark-common-runtime")
 // kyuubi hive connector doesn't support 2.13 for Spark3.3
 if (scalaVersion == "2.12") {
   include("spark-connector:spark-3.3", "spark-connector:spark-runtime-3.3")

diff --git a/spark-connector/spark-common-runtime/build.gradle.kts b/spark-connector/spark-common-runtime/build.gradle.kts
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+  `maven-publish`
+  id("java")
+  alias(libs.plugins.shadow)
+}
+
+dependencies {
+  implementation(project(":spark-connector:spark-common"))
+}
+
+tasks.withType<ShadowJar>(ShadowJar::class.java) {
+  isZip64 = true
+  configurations = listOf(project.configurations.runtimeClasspath.get())
+  archiveClassifier.set("")
+
+  // Relocate dependencies to avoid conflicts
+  relocate("com.google", "org.apache.gravitino.shaded.com.google")
+  relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine")
+}
+
+tasks.jar {
+  dependsOn(tasks.named("shadowJar"))
+  archiveClassifier.set("empty")
+}
diff --git a/spark-connector/spark-common/build.gradle.kts b/spark-connector/spark-common/build.gradle.kts
@@ -40,6 +40,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
 dependencies {
   implementation(project(":catalogs:catalog-common"))
   implementation(libs.guava)
+  implementation(libs.caffeine)
 
   compileOnly(project(":clients:client-java-runtime", configuration = "shadow"))
   compileOnly("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")

diff --git a/...n/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java b/...n/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java
@@ -18,13 +18,12 @@
  */
 package org.apache.gravitino.spark.connector.catalog;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Supplier;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
 import java.util.Arrays;
 import java.util.Map;
-import java.util.concurrent.ExecutionException;
 import org.apache.gravitino.Catalog;
 import org.apache.gravitino.client.GravitinoClient;
 import org.slf4j.Logger;
@@ -42,7 +41,7 @@ public class GravitinoCatalogManager {
   private GravitinoCatalogManager(Supplier<GravitinoClient> clientBuilder) {
     this.gravitinoClient = clientBuilder.get();
     // Will not evict catalog by default
-    this.gravitinoCatalogs = CacheBuilder.newBuilder().build();
+    this.gravitinoCatalogs = Caffeine.newBuilder().build();
   }
 
   public static GravitinoCatalogManager create(Supplier<GravitinoClient> clientBuilder) {
@@ -69,8 +68,8 @@ public void close() {
 
   public Catalog getGravitinoCatalogInfo(String name) {
     try {
-      return gravitinoCatalogs.get(name, () -> loadCatalog(name));
-    } catch (ExecutionException e) {
+      return gravitinoCatalogs.get(name, catalogName -> loadCatalog(catalogName));
+    } catch (Exception e) {
       LOG.error(String.format("Load catalog %s failed", name), e);
       throw new RuntimeException(e);
     }

diff --git a/spark-connector/v3.3/spark/build.gradle.kts b/spark-connector/v3.3/spark/build.gradle.kts
@@ -38,7 +38,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
 val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion"
 
 dependencies {
-  implementation(project(":spark-connector:spark-common"))
+  implementation(project(":spark-connector:spark-common-runtime", configuration = "shadow"))
   compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
   compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
     exclude("com.fasterxml.jackson")
@@ -79,7 +79,7 @@ dependencies {
     exclude("org.apache.logging.log4j")
     exclude("org.slf4j")
   }
-  testImplementation(project(":spark-connector:spark-common", "testArtifacts")) {
+  testImplementation(project(":spark-connector:spark-common-runtime", "testArtifacts")) {
     exclude("com.fasterxml.jackson")
   }
 

diff --git a/spark-connector/v3.4/spark/build.gradle.kts b/spark-connector/v3.4/spark/build.gradle.kts
@@ -38,7 +38,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
 val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion"
 
 dependencies {
-  implementation(project(":spark-connector:spark-common"))
+  implementation(project(":spark-connector:spark-common-runtime", configuration = "shadow"))
   compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
   compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
     exclude("com.fasterxml.jackson")
@@ -80,7 +80,7 @@ dependencies {
     exclude("org.apache.logging.log4j")
     exclude("org.slf4j")
   }
-  testImplementation(project(":spark-connector:spark-common", "testArtifacts")) {
+  testImplementation(project(":spark-connector:spark-common-runtime", "testArtifacts")) {
     exclude("com.fasterxml.jackson")
   }
 

diff --git a/spark-connector/v3.5/spark/build.gradle.kts b/spark-connector/v3.5/spark/build.gradle.kts
@@ -39,7 +39,7 @@ val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion
 
 dependencies {
   implementation(project(":spark-connector:spark-3.4"))
-  implementation(project(":spark-connector:spark-common"))
+  implementation(project(":spark-connector:spark-common-runtime", configuration = "shadow"))
   compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
   compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
     exclude("com.fasterxml.jackson")
@@ -80,7 +80,7 @@ dependencies {
     exclude("org.apache.logging.log4j")
     exclude("org.slf4j")
   }
-  testImplementation(project(":spark-connector:spark-common", "testArtifacts")) {
+  testImplementation(project(":spark-connector:spark-common-runtime", "testArtifacts")) {
     exclude("com.fasterxml.jackson")
     exclude("org.apache.logging.log4j")
     exclude("org.slf4j")