Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#6270] (Improvement) unified cache framework #6271

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+26 −9
Open

[#6270] (Improvement) unified cache framework #6271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions spark-connector/spark-common/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
* specific language governing permissions and limitations
* under the License.
*/
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

plugins {
`maven-publish`
id("java")
Expand All @@ -40,6 +42,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
dependencies {
implementation(project(":catalogs:catalog-common"))
implementation(libs.guava)
implementation(libs.caffeine)

compileOnly(project(":clients:client-java-runtime", configuration = "shadow"))
compileOnly("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion")
Expand Down Expand Up @@ -171,3 +174,18 @@ configurations {
artifacts {
add("testArtifacts", testJar)
}

tasks.withType<ShadowJar>(ShadowJar::class.java) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we could shade the jar in spark-runtime module, like spark-connector/v3.5/spark-runtime/build.gradle.kts

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to add this to every spark-runtime version package? Should we extract the common packages or handle them directly in the common module? This way, we can avoid redundant processing.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would be inclined to use @FANNG1 's way to shade jars in spark-runtime, so we don't have to do the shade in this common component, and change all the implementation to shadow.

isZip64 = true
configurations = listOf(project.configurations.runtimeClasspath.get())
archiveClassifier.set("")

// Relocate dependencies to avoid conflicts
relocate("com.google", "org.apache.gravitino.shaded.com.google")
relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine")
}

tasks.jar {
dependsOn(tasks.named("shadowJar"))
archiveClassifier.set("empty")
}
11 changes: 5 additions & 6 deletions ...n/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@
*/
package org.apache.gravitino.spark.connector.catalog;

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.apache.gravitino.Catalog;
import org.apache.gravitino.client.GravitinoClient;
import org.slf4j.Logger;
Expand All @@ -42,7 +41,7 @@ public class GravitinoCatalogManager {
private GravitinoCatalogManager(Supplier<GravitinoClient> clientBuilder) {
this.gravitinoClient = clientBuilder.get();
// Will not evict catalog by default
this.gravitinoCatalogs = CacheBuilder.newBuilder().build();
this.gravitinoCatalogs = Caffeine.newBuilder().build();
}

public static GravitinoCatalogManager create(Supplier<GravitinoClient> clientBuilder) {
Expand All @@ -69,8 +68,8 @@ public void close() {

public Catalog getGravitinoCatalogInfo(String name) {
try {
return gravitinoCatalogs.get(name, () -> loadCatalog(name));
} catch (ExecutionException e) {
return gravitinoCatalogs.get(name, catalogName -> loadCatalog(catalogName));
} catch (Exception e) {
LOG.error(String.format("Load catalog %s failed", name), e);
throw new RuntimeException(e);
}
Expand Down
2 changes: 1 addition & 1 deletion spark-connector/v3.3/spark/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion"

dependencies {
implementation(project(":spark-connector:spark-common"))
implementation(project(":spark-connector:spark-common", configuration = "shadow"))
compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
exclude("com.fasterxml.jackson")
Expand Down
2 changes: 1 addition & 1 deletion spark-connector/v3.4/spark/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat
val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion"

dependencies {
implementation(project(":spark-connector:spark-common"))
implementation(project(":spark-connector:spark-common", configuration = "shadow"))
compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
exclude("com.fasterxml.jackson")
Expand Down
2 changes: 1 addition & 1 deletion spark-connector/v3.5/spark/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ val artifactName = "${rootProject.name}-spark-${sparkMajorVersion}_$scalaVersion

dependencies {
implementation(project(":spark-connector:spark-3.4"))
implementation(project(":spark-connector:spark-common"))
implementation(project(":spark-connector:spark-common", configuration = "shadow"))
compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
exclude("com.fasterxml.jackson")
Expand Down
Loading