From 8fab887452d370c7fc9e993651374cbc73be608f Mon Sep 17 00:00:00 2001 From: sunxiaojian Date: Wed, 15 Jan 2025 20:04:35 +0800 Subject: [PATCH] unified cache --- .../backend-integration-test-action.yml | 2 +- settings.gradle.kts | 2 +- .../spark-common-runtime/build.gradle.kts | 44 +++++++++++++++++++ spark-connector/spark-common/build.gradle.kts | 1 + .../catalog/GravitinoCatalogManager.java | 11 +++-- spark-connector/v3.3/spark/build.gradle.kts | 4 +- spark-connector/v3.4/spark/build.gradle.kts | 4 +- spark-connector/v3.5/spark/build.gradle.kts | 4 +- 8 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 spark-connector/spark-common-runtime/build.gradle.kts diff --git a/.github/workflows/backend-integration-test-action.yml b/.github/workflows/backend-integration-test-action.yml index b15c5d226ca..80eda3f1384 100644 --- a/.github/workflows/backend-integration-test-action.yml +++ b/.github/workflows/backend-integration-test-action.yml @@ -57,7 +57,7 @@ jobs: id: integrationTest run: > ./gradlew test -PskipTests -PtestMode=${{ inputs.test-mode }} -PjdkVersion=${{ inputs.java-version }} -PjdbcBackend=${{ inputs.backend }} -PskipDockerTests=false - -x :web:web:test -x :web:integration-test:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:spark-common:test + -x :web:web:test -x :web:integration-test:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:spark-common:test -x :spark-connector:spark-common-runtime:test -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test diff --git a/settings.gradle.kts b/settings.gradle.kts index c865e14e7a2..02dc61177ab 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -59,7 +59,7 @@ include("iceberg:iceberg-common") include("iceberg:iceberg-rest-server") include("authorizations:authorization-ranger", "authorizations:authorization-common", "authorizations:authorization-chain") include("trino-connector:trino-connector", "trino-connector:integration-test") -include("spark-connector:spark-common") +include("spark-connector:spark-common", "spark-connector:spark-common-runtime") // kyuubi hive connector doesn't support 2.13 for Spark3.3 if (scalaVersion == "2.12") { include("spark-connector:spark-3.3", "spark-connector:spark-runtime-3.3") diff --git a/spark-connector/spark-common-runtime/build.gradle.kts b/spark-connector/spark-common-runtime/build.gradle.kts new file mode 100644 index 00000000000..31abcb3e60a --- /dev/null +++ b/spark-connector/spark-common-runtime/build.gradle.kts @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + implementation(project(":spark-connector:spark-common")) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + + // Relocate dependencies to avoid conflicts + relocate("com.google", "org.apache.gravitino.shaded.com.google") + relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} diff --git a/spark-connector/spark-common/build.gradle.kts b/spark-connector/spark-common/build.gradle.kts index 06e0077d21e..dc6bbd258e7 100644 --- a/spark-connector/spark-common/build.gradle.kts +++ b/spark-connector/spark-common/build.gradle.kts @@ -40,6 +40,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat dependencies { implementation(project(":catalogs:catalog-common")) implementation(libs.guava) + implementation(libs.caffeine) compileOnly(project(":clients:client-java-runtime", configuration = "shadow")) compileOnly("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion") diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java index f04193f33a3..a484972656a 100644 --- a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java @@ -18,13 +18,12 @@ */ package org.apache.gravitino.spark.connector.catalog; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import java.util.Arrays; import java.util.Map; -import java.util.concurrent.ExecutionException; import org.apache.gravitino.Catalog; import org.apache.gravitino.client.GravitinoClient; import org.slf4j.Logger; @@ -42,7 +41,7 @@ public class GravitinoCatalogManager { private GravitinoCatalogManager(Supplier clientBuilder) { this.gravitinoClient = clientBuilder.get(); // Will not evict catalog by default - this.gravitinoCatalogs = CacheBuilder.newBuilder().build(); + this.gravitinoCatalogs = Caffeine.newBuilder().build(); } public static GravitinoCatalogManager create(Supplier clientBuilder) { @@ -69,8 +68,8 @@ public void close() { public Catalog getGravitinoCatalogInfo(String name) { try { - return gravitinoCatalogs.get(name, () -> loadCatalog(name)); - } catch (ExecutionException e) { + return gravitinoCatalogs.get(name, catalogName -> loadCatalog(catalogName)); + } catch (Exception e) { LOG.error(String.format("Load catalog %s failed", name), e); throw new RuntimeException(e); } diff --git a/spark-connector/v3.3/spark/build.gradle.kts b/spark-connector/v3.3/spark/build.gradle.kts index 66c65f863b9..1bce6533803 100644 --- a/spark-connector/v3.3/spark/build.gradle.kts +++ b/spark-connector/v3.3/spark/build.gradle.kts @@ -38,7 +38,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion" dependencies { - implementation(project(":spark-connector:spark-common")) + implementation(project(":spark-connector:spark-common-runtime", configuration = "shadow")) compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion") compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") { exclude("com.fasterxml.jackson") @@ -79,7 +79,7 @@ dependencies { exclude("org.apache.logging.log4j") exclude("org.slf4j") } - testImplementation(project(":spark-connector:spark-common", "testArtifacts")) { + testImplementation(project(":spark-connector:spark-common-runtime", "testArtifacts")) { exclude("com.fasterxml.jackson") } diff --git a/spark-connector/v3.4/spark/build.gradle.kts b/spark-connector/v3.4/spark/build.gradle.kts index aa4134a3c71..7a3ea29d2a0 100644 --- a/spark-connector/v3.4/spark/build.gradle.kts +++ b/spark-connector/v3.4/spark/build.gradle.kts @@ -38,7 +38,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion" dependencies { - implementation(project(":spark-connector:spark-common")) + implementation(project(":spark-connector:spark-common-runtime", configuration = "shadow")) compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion") compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") { exclude("com.fasterxml.jackson") @@ -80,7 +80,7 @@ dependencies { exclude("org.apache.logging.log4j") exclude("org.slf4j") } - testImplementation(project(":spark-connector:spark-common", "testArtifacts")) { + testImplementation(project(":spark-connector:spark-common-runtime", "testArtifacts")) { exclude("com.fasterxml.jackson") } diff --git a/spark-connector/v3.5/spark/build.gradle.kts b/spark-connector/v3.5/spark/build.gradle.kts index 15aa018081d..e8ceeba85f0 100644 --- a/spark-connector/v3.5/spark/build.gradle.kts +++ b/spark-connector/v3.5/spark/build.gradle.kts @@ -39,7 +39,7 @@ val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion dependencies { implementation(project(":spark-connector:spark-3.4")) - implementation(project(":spark-connector:spark-common")) + implementation(project(":spark-connector:spark-common-runtime", configuration = "shadow")) compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion") compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") { exclude("com.fasterxml.jackson") @@ -80,7 +80,7 @@ dependencies { exclude("org.apache.logging.log4j") exclude("org.slf4j") } - testImplementation(project(":spark-connector:spark-common", "testArtifacts")) { + testImplementation(project(":spark-connector:spark-common-runtime", "testArtifacts")) { exclude("com.fasterxml.jackson") exclude("org.apache.logging.log4j") exclude("org.slf4j")