[SPARK-50493][SPARK-37687][BUILD] Migrate kubernetes-client from `6.x…

…` to `7.x` ### What changes were proposed in this pull request? The pr aims to migrate kubernetes-client from `6.x` to `7.x`, includes: - upgrade `kubernetes-client` from `6.13.2` to `7.0.1`. - switch default `HttpClient` implementation from `OkHttp` to `Vert.x`. ### Why are the changes needed? - The full release notes: https://github.com/fabric8io/kubernetes-client/releases/tag/v7.0.1 https://github.com/fabric8io/kubernetes-client/releases/tag/v7.0.0 - [Vert.x as default HttpClient implementation](https://github.com/fabric8io/kubernetes-client/blob/v7.0.0/doc/MIGRATION-v7.md#vertx-as-default-httpclient-implementation-) ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass GA. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #49159 from panbingkun/k8s_client_7_vert_x. Authored-by: panbingkun <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
apache · Jan 3, 2025 · dccb129 · dccb129
1 parent a59941a
commit dccb129
Show file tree

Hide file tree

Showing 9 changed files with 55 additions and 61 deletions.
diff --git a/LICENSE-binary b/LICENSE-binary
@@ -286,6 +286,10 @@ io.netty:netty-transport-classes-kqueue
 io.netty:netty-transport-native-epoll
 io.netty:netty-transport-native-kqueue
 io.netty:netty-transport-native-unix-common
+io.vertx:vertx-auth-common
+io.vertx:vertx-core
+io.vertx:vertx-web-client
+io.vertx:vertx-web-common
 jakarta.inject:jakarta.inject-api
 jakarta.validation:jakarta.validation-api
 javax.jdo:jdo-api

diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -158,31 +158,31 @@ jsr305/3.0.0//jsr305-3.0.0.jar
 jta/1.1//jta-1.1.jar
 jul-to-slf4j/2.0.16//jul-to-slf4j-2.0.16.jar
 kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client-api/6.13.4//kubernetes-client-api-6.13.4.jar
-kubernetes-client/6.13.4//kubernetes-client-6.13.4.jar
-kubernetes-httpclient-okhttp/6.13.4//kubernetes-httpclient-okhttp-6.13.4.jar
-kubernetes-model-admissionregistration/6.13.4//kubernetes-model-admissionregistration-6.13.4.jar
-kubernetes-model-apiextensions/6.13.4//kubernetes-model-apiextensions-6.13.4.jar
-kubernetes-model-apps/6.13.4//kubernetes-model-apps-6.13.4.jar
-kubernetes-model-autoscaling/6.13.4//kubernetes-model-autoscaling-6.13.4.jar
-kubernetes-model-batch/6.13.4//kubernetes-model-batch-6.13.4.jar
-kubernetes-model-certificates/6.13.4//kubernetes-model-certificates-6.13.4.jar
-kubernetes-model-common/6.13.4//kubernetes-model-common-6.13.4.jar
-kubernetes-model-coordination/6.13.4//kubernetes-model-coordination-6.13.4.jar
-kubernetes-model-core/6.13.4//kubernetes-model-core-6.13.4.jar
-kubernetes-model-discovery/6.13.4//kubernetes-model-discovery-6.13.4.jar
-kubernetes-model-events/6.13.4//kubernetes-model-events-6.13.4.jar
-kubernetes-model-extensions/6.13.4//kubernetes-model-extensions-6.13.4.jar
-kubernetes-model-flowcontrol/6.13.4//kubernetes-model-flowcontrol-6.13.4.jar
-kubernetes-model-gatewayapi/6.13.4//kubernetes-model-gatewayapi-6.13.4.jar
-kubernetes-model-metrics/6.13.4//kubernetes-model-metrics-6.13.4.jar
-kubernetes-model-networking/6.13.4//kubernetes-model-networking-6.13.4.jar
-kubernetes-model-node/6.13.4//kubernetes-model-node-6.13.4.jar
-kubernetes-model-policy/6.13.4//kubernetes-model-policy-6.13.4.jar
-kubernetes-model-rbac/6.13.4//kubernetes-model-rbac-6.13.4.jar
-kubernetes-model-resource/6.13.4//kubernetes-model-resource-6.13.4.jar
-kubernetes-model-scheduling/6.13.4//kubernetes-model-scheduling-6.13.4.jar
-kubernetes-model-storageclass/6.13.4//kubernetes-model-storageclass-6.13.4.jar
+kubernetes-client-api/7.0.1//kubernetes-client-api-7.0.1.jar
+kubernetes-client/7.0.1//kubernetes-client-7.0.1.jar
+kubernetes-httpclient-vertx/7.0.1//kubernetes-httpclient-vertx-7.0.1.jar
+kubernetes-model-admissionregistration/7.0.1//kubernetes-model-admissionregistration-7.0.1.jar
+kubernetes-model-apiextensions/7.0.1//kubernetes-model-apiextensions-7.0.1.jar
+kubernetes-model-apps/7.0.1//kubernetes-model-apps-7.0.1.jar
+kubernetes-model-autoscaling/7.0.1//kubernetes-model-autoscaling-7.0.1.jar
+kubernetes-model-batch/7.0.1//kubernetes-model-batch-7.0.1.jar
+kubernetes-model-certificates/7.0.1//kubernetes-model-certificates-7.0.1.jar
+kubernetes-model-common/7.0.1//kubernetes-model-common-7.0.1.jar
+kubernetes-model-coordination/7.0.1//kubernetes-model-coordination-7.0.1.jar
+kubernetes-model-core/7.0.1//kubernetes-model-core-7.0.1.jar
+kubernetes-model-discovery/7.0.1//kubernetes-model-discovery-7.0.1.jar
+kubernetes-model-events/7.0.1//kubernetes-model-events-7.0.1.jar
+kubernetes-model-extensions/7.0.1//kubernetes-model-extensions-7.0.1.jar
+kubernetes-model-flowcontrol/7.0.1//kubernetes-model-flowcontrol-7.0.1.jar
+kubernetes-model-gatewayapi/7.0.1//kubernetes-model-gatewayapi-7.0.1.jar
+kubernetes-model-metrics/7.0.1//kubernetes-model-metrics-7.0.1.jar
+kubernetes-model-networking/7.0.1//kubernetes-model-networking-7.0.1.jar
+kubernetes-model-node/7.0.1//kubernetes-model-node-7.0.1.jar
+kubernetes-model-policy/7.0.1//kubernetes-model-policy-7.0.1.jar
+kubernetes-model-rbac/7.0.1//kubernetes-model-rbac-7.0.1.jar
+kubernetes-model-resource/7.0.1//kubernetes-model-resource-7.0.1.jar
+kubernetes-model-scheduling/7.0.1//kubernetes-model-scheduling-7.0.1.jar
+kubernetes-model-storageclass/7.0.1//kubernetes-model-storageclass-7.0.1.jar
 lapack/3.0.3//lapack-3.0.3.jar
 leveldbjni-all/1.8//leveldbjni-all-1.8.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
@@ -193,7 +193,6 @@ log4j-api/2.24.3//log4j-api-2.24.3.jar
 log4j-core/2.24.3//log4j-core-2.24.3.jar
 log4j-layout-template-json/2.24.3//log4j-layout-template-json-2.24.3.jar
 log4j-slf4j2-impl/2.24.3//log4j-slf4j2-impl-2.24.3.jar
-logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar
 lz4-java/1.8.0//lz4-java-1.8.0.jar
 metrics-core/4.2.29//metrics-core-4.2.29.jar
 metrics-graphite/4.2.29//metrics-graphite-4.2.29.jar
@@ -203,13 +202,15 @@ metrics-jvm/4.2.29//metrics-jvm-4.2.29.jar
 minlog/1.3.0//minlog-1.3.0.jar
 netty-all/4.1.115.Final//netty-all-4.1.115.Final.jar
 netty-buffer/4.1.115.Final//netty-buffer-4.1.115.Final.jar
+netty-codec-dns/4.1.115.Final//netty-codec-dns-4.1.115.Final.jar
 netty-codec-http/4.1.115.Final//netty-codec-http-4.1.115.Final.jar
 netty-codec-http2/4.1.115.Final//netty-codec-http2-4.1.115.Final.jar
 netty-codec-socks/4.1.115.Final//netty-codec-socks-4.1.115.Final.jar
 netty-codec/4.1.115.Final//netty-codec-4.1.115.Final.jar
 netty-common/4.1.115.Final//netty-common-4.1.115.Final.jar
 netty-handler-proxy/4.1.115.Final//netty-handler-proxy-4.1.115.Final.jar
 netty-handler/4.1.115.Final//netty-handler-4.1.115.Final.jar
+netty-resolver-dns/4.1.115.Final//netty-resolver-dns-4.1.115.Final.jar
 netty-resolver/4.1.115.Final//netty-resolver-4.1.115.Final.jar
 netty-tcnative-boringssl-static/2.0.69.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar
 netty-tcnative-boringssl-static/2.0.69.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar
@@ -258,7 +259,7 @@ scala-parser-combinators_2.13/2.4.0//scala-parser-combinators_2.13-2.4.0.jar
 scala-reflect/2.13.15//scala-reflect-2.13.15.jar
 scala-xml_2.13/2.3.0//scala-xml_2.13-2.3.0.jar
 slf4j-api/2.0.16//slf4j-api-2.0.16.jar
-snakeyaml-engine/2.7//snakeyaml-engine-2.7.jar
+snakeyaml-engine/2.8//snakeyaml-engine-2.8.jar
 snakeyaml/2.3//snakeyaml-2.3.jar
 snappy-java/1.1.10.7//snappy-java-1.1.10.7.jar
 spire-macros_2.13/0.18.0//spire-macros_2.13-0.18.0.jar
@@ -272,11 +273,15 @@ threeten-extra/1.7.1//threeten-extra-1.7.1.jar
 tink/1.16.0//tink-1.16.0.jar
 transaction-api/1.1//transaction-api-1.1.jar
 univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar
+vertx-auth-common/4.5.11//vertx-auth-common-4.5.11.jar
+vertx-core/4.5.11//vertx-core-4.5.11.jar
+vertx-web-client/4.5.11//vertx-web-client-4.5.11.jar
+vertx-web-common/4.5.11//vertx-web-common-4.5.11.jar
 wildfly-openssl/1.1.3.Final//wildfly-openssl-1.1.3.Final.jar
 xbean-asm9-shaded/4.26//xbean-asm9-shaded-4.26.jar
 xmlschema-core/2.3.1//xmlschema-core-2.3.1.jar
 xz/1.10//xz-1.10.jar
-zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
+zjsonpatch/7.0.1//zjsonpatch-7.0.1.jar
 zookeeper-jute/3.9.3//zookeeper-jute-3.9.3.jar
 zookeeper/3.9.3//zookeeper-3.9.3.jar
 zstd-jni/1.5.6-8//zstd-jni-1.5.6-8.jar
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
@@ -34,6 +34,8 @@
   </description>
   <properties>
     <sbt.project.name>hadoop-cloud</sbt.project.name>
+    <okhttp.version>3.12.12</okhttp.version>
+    <okio.version>1.17.6</okio.version>
   </properties>
 
   <dependencies>
@@ -191,6 +193,16 @@
       <version>${jetty.version}</version>
       <scope>${hadoop.deps.scope}</scope>
     </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp</artifactId>
+      <version>${okhttp.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okio</groupId>
+      <artifactId>okio</artifactId>
+      <version>${okio.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

diff --git a/pom.xml b/pom.xml
@@ -235,8 +235,7 @@
 
     <!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
     <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
-    <kubernetes-client.version>6.13.4</kubernetes-client.version>
-    <okio.version>1.17.6</okio.version>
+    <kubernetes-client.version>7.0.1</kubernetes-client.version>
 
     <test.java.home>${java.home}</test.java.home>
 
@@ -2594,11 +2593,6 @@
         <artifactId>javax.servlet-api</artifactId>
         <version>${javaxservlet.version}</version>
       </dependency>
-      <dependency>
-        <groupId>com.squareup.okio</groupId>
-        <artifactId>okio</artifactId>
-        <version>${okio.version}</version>
-      </dependency>
     </dependencies>
   </dependencyManagement>
 

diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml
@@ -37,7 +37,7 @@
       <dependencies>
         <dependency>
           <groupId>io.fabric8</groupId>
-          <artifactId>volcano-model-v1beta1</artifactId>
+          <artifactId>volcano-model</artifactId>
           <version>${kubernetes-client.version}</version>
         </dependency>
         <dependency>
@@ -105,11 +105,6 @@
       <scope>test</scope>
     </dependency>
 
-    <dependency>
-      <groupId>io.fabric8</groupId>
-      <artifactId>kubernetes-httpclient-okhttp</artifactId>
-      <version>${kubernetes-client.version}</version>
-    </dependency>
     <dependency>
       <groupId>io.fabric8</groupId>
       <artifactId>kubernetes-client</artifactId>

diff --git a/...rnetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala b/...rnetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
@@ -24,18 +24,14 @@ import com.google.common.io.Files
 import io.fabric8.kubernetes.client.{ConfigBuilder, KubernetesClient, KubernetesClientBuilder}
 import io.fabric8.kubernetes.client.Config.KUBERNETES_REQUEST_RETRY_BACKOFFLIMIT_SYSTEM_PROPERTY
 import io.fabric8.kubernetes.client.Config.autoConfigure
-import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory
 import io.fabric8.kubernetes.client.utils.Utils.getSystemPropertyOrEnvVar
-import okhttp3.Dispatcher
-import okhttp3.OkHttpClient
 
 import org.apache.spark.SparkConf
 import org.apache.spark.annotation.{DeveloperApi, Since, Stable}
 import org.apache.spark.deploy.k8s.Config._
 import org.apache.spark.internal.{Logging, MDC}
 import org.apache.spark.internal.LogKeys.K8S_CONTEXT
 import org.apache.spark.internal.config.ConfigEntry
-import org.apache.spark.util.ThreadUtils
 
 /**
  * :: DeveloperApi ::
@@ -78,10 +74,6 @@ object SparkKubernetesClientFactory extends Logging {
       .getOption(s"$kubernetesAuthConfPrefix.$CLIENT_KEY_FILE_CONF_SUFFIX")
     val clientCertFile = sparkConf
       .getOption(s"$kubernetesAuthConfPrefix.$CLIENT_CERT_FILE_CONF_SUFFIX")
-    // TODO(SPARK-37687): clean up direct usage of OkHttpClient, see also:
-    // https://github.com/fabric8io/kubernetes-client/issues/3547
-    val dispatcher = new Dispatcher(
-      ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher"))
 
     // Allow for specifying a context used to auto-configure from the users K8S config file
     val kubeContext = sparkConf.get(KUBERNETES_CONTEXT).filter(_.nonEmpty)
@@ -117,17 +109,9 @@ object SparkKubernetesClientFactory extends Logging {
       }.withOption(namespace) {
         (ns, configBuilder) => configBuilder.withNamespace(ns)
       }.build()
-    val factoryWithCustomDispatcher = new OkHttpClientFactory() {
-      override protected def additionalConfig(builder: OkHttpClient.Builder): Unit = {
-        builder.dispatcher(dispatcher)
-      }
-    }
     logDebug("Kubernetes client config: " +
       new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(config))
-    new KubernetesClientBuilder()
-      .withHttpClientFactory(factoryWithCustomDispatcher)
-      .withConfig(config)
-      .build()
+    new KubernetesClientBuilder().withConfig(config).build()
   }
 
   private implicit class OptionConfigurableConfigBuilder(val configBuilder: ConfigBuilder)

diff --git a/...core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala b/...core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
@@ -17,8 +17,8 @@
 package org.apache.spark.deploy.k8s.features
 
 import io.fabric8.kubernetes.api.model._
+import io.fabric8.volcano.api.model.scheduling.v1beta1.{PodGroup, PodGroupSpec}
 import io.fabric8.volcano.client.DefaultVolcanoClient
-import io.fabric8.volcano.scheduling.v1beta1.{PodGroup, PodGroupSpec}
 
 import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverConf, KubernetesExecutorConf, SparkPod}
 import org.apache.spark.internal.Logging

diff --git a/...volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala b/...volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.features
 
 import java.io.File
 
-import io.fabric8.volcano.scheduling.v1beta1.PodGroup
+import io.fabric8.volcano.api.model.scheduling.v1beta1.PodGroup
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.k8s._

diff --git a/...olcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala b/...olcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
@@ -28,8 +28,8 @@ import scala.concurrent.Future
 import scala.jdk.CollectionConverters._
 
 import io.fabric8.kubernetes.api.model.{HasMetadata, Pod, Quantity}
+import io.fabric8.volcano.api.model.scheduling.v1beta1.{Queue, QueueBuilder}
 import io.fabric8.volcano.client.VolcanoClient
-import io.fabric8.volcano.scheduling.v1beta1.{Queue, QueueBuilder}
 import org.scalatest.BeforeAndAfterEach
 import org.scalatest.concurrent.Eventually