[WIP] karmada operator support external etcd, part1

charts/karmada-operator/crds/operator.karmada.io_karmadas.yaml

@@ -55,14 +55,12 @@ spec:
                         properties:
                           caData:
                             description: CAData is an SSL Certificate Authority file
-                              used to secure etcd communication. Required if using
-                              a TLS connection.
+                              used to secure etcd communication.
                             format: byte
                             type: string
                           certData:
                             description: CertData is an SSL certification file used
-                              to secure etcd communication. Required if using a TLS
-                              connection.
+                              to secure etcd communication.
                             format: byte
                             type: string
                           endpoints:
@@ -72,14 +70,11 @@ spec:
                             type: array
                           keyData:
                             description: KeyData is an SSL key file used to secure
-                              etcd communication. Required if using a TLS connection.
+                              etcd communication.
                             format: byte
                             type: string
                         required:
-                        - caData
-                        - certData
                         - endpoints
-                        - keyData
                         type: object
                       local:
                         description: Local provides configuration knobs for configuring

operator/pkg/apis/operator/v1alpha1/type.go

@@ -197,15 +197,15 @@ type ExternalEtcd struct {
 	Endpoints []string `json:"endpoints"`
 
 	// CAData is an SSL Certificate Authority file used to secure etcd communication.
-	// Required if using a TLS connection.
+	// +optional
 	CAData []byte `json:"caData"`
 
 	// CertData is an SSL certification file used to secure etcd communication.
-	// Required if using a TLS connection.
+	// +optional
 	CertData []byte `json:"certData"`
 
 	// KeyData is an SSL key file used to secure etcd communication.
-	// Required if using a TLS connection.
+	// +optional
 	KeyData []byte `json:"keyData"`
 }
 

operator/pkg/controlplane/apiserver/apiserver.go

@@ -18,38 +18,51 @@ import (
 
 // EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource
 func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error {
-	if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace, featureGates); err != nil {
+	if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, cfg.Etcd, name, namespace, featureGates); err != nil {
 		return fmt.Errorf("failed to install karmada apiserver, err: %w", err)
 	}
 
 	return createKarmadaAPIServerService(client, cfg.KarmadaAPIServer, name, namespace)
 }
 
 // EnsureKarmadaAggregatedAPIServer creates karmada aggregated apiserver deployment and service resource
-func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error {
-	if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, name, namespace, featureGates); err != nil {
+func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, featureGates map[string]bool) error {
+	if err := installKarmadaAggregatedAPIServer(client, cfg, etcdCfg, name, namespace, featureGates); err != nil {
 		return err
 	}
 	return createKarmadaAggregatedAPIServerService(client, name, namespace)
 }
 
-func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, name, namespace string, featureGates map[string]bool) error {
+func getEtcdServers(name, namespace string, etcdCfg *operatorv1alpha1.Etcd) string {
+	endpoints := ""
+	if etcdCfg.Local != nil {
+		for _, v := range etcdCfg.External.Endpoints {
+			endpoints += v + ","
+		}
+	}
+	if etcdCfg.External != nil {
+		endpoints = fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", util.KarmadaEtcdClientName(name), namespace, constants.EtcdListenClientPort)
+	}
+	return endpoints
+}
+
+func installKarmadaAPIServer(client clientset.Interface, apiCfg *operatorv1alpha1.KarmadaAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, featureGates map[string]bool) error {
 	apiserverDeploymentBytes, err := util.ParseTemplate(KarmadaApiserverDeployment, struct {
-		DeploymentName, Namespace, Image, EtcdClientService string
-		ServiceSubnet, KarmadaCertsSecret, EtcdCertsSecret  string
-		Replicas                                            *int32
-		EtcdListenClientPort                                int32
+		DeploymentName, Namespace, Image                   string
+		ServiceSubnet, KarmadaCertsSecret, EtcdCertsSecret string
+		EtcdServers                                        string
+		Replicas                                           *int32
 	}{
-		DeploymentName:       util.KarmadaAPIServerName(name),
-		Namespace:            namespace,
-		Image:                cfg.Image.Name(),
-		EtcdClientService:    util.KarmadaEtcdClientName(name),
-		ServiceSubnet:        *cfg.ServiceSubnet,
-		KarmadaCertsSecret:   util.KarmadaCertSecretName(name),
-		EtcdCertsSecret:      util.EtcdCertSecretName(name),
-		Replicas:             cfg.Replicas,
-		EtcdListenClientPort: constants.EtcdListenClientPort,
+		DeploymentName:     util.KarmadaAPIServerName(name),
+		Namespace:          namespace,
+		Image:              apiCfg.Image.Name(),
+		EtcdServers:        getEtcdServers(name, namespace, etcdCfg),
+		ServiceSubnet:      *apiCfg.ServiceSubnet,
+		KarmadaCertsSecret: util.KarmadaCertSecretName(name),
+		EtcdCertsSecret:    util.EtcdCertSecretName(name),
+		Replicas:           apiCfg.Replicas,
 	})
+
 	if err != nil {
 		return fmt.Errorf("error when parsing karmadaApiserver deployment template: %w", err)
 	}
@@ -58,8 +71,8 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K
 	if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), apiserverDeploymentBytes, apiserverDeployment); err != nil {
 		return fmt.Errorf("error when decoding karmadaApiserver deployment: %w", err)
 	}
-	patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
-		WithExtraArgs(cfg.ExtraArgs).ForDeployment(apiserverDeployment)
+	patcher.NewPatcher().WithAnnotations(apiCfg.Annotations).WithLabels(apiCfg.Labels).
+		WithExtraArgs(apiCfg.ExtraArgs).ForDeployment(apiserverDeployment)
 
 	if err := apiclient.CreateOrUpdateDeployment(client, apiserverDeployment); err != nil {
 		return fmt.Errorf("error when creating deployment for %s, err: %w", apiserverDeployment.Name, err)
@@ -90,22 +103,21 @@ func createKarmadaAPIServerService(client clientset.Interface, cfg *operatorv1al
 	return nil
 }
 
-func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, name, namespace string, featureGates map[string]bool) error {
+func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, featureGates map[string]bool) error {
 	aggregatedAPIServerDeploymentBytes, err := util.ParseTemplate(KarmadaAggregatedAPIServerDeployment, struct {
-		DeploymentName, Namespace, Image, EtcdClientService   string
+		DeploymentName, Namespace, Image                      string
 		KubeconfigSecret, KarmadaCertsSecret, EtcdCertsSecret string
+		EtcdServers                                           string
 		Replicas                                              *int32
-		EtcdListenClientPort                                  int32
 	}{
-		DeploymentName:       util.KarmadaAggregatedAPIServerName(name),
-		Namespace:            namespace,
-		Image:                cfg.Image.Name(),
-		EtcdClientService:    util.KarmadaEtcdClientName(name),
-		KubeconfigSecret:     util.AdminKubeconfigSecretName(name),
-		KarmadaCertsSecret:   util.KarmadaCertSecretName(name),
-		EtcdCertsSecret:      util.EtcdCertSecretName(name),
-		Replicas:             cfg.Replicas,
-		EtcdListenClientPort: constants.EtcdListenClientPort,
+		DeploymentName:     util.KarmadaAggregatedAPIServerName(name),
+		Namespace:          namespace,
+		Image:              cfg.Image.Name(),
+		EtcdServers:        getEtcdServers(name, namespace, etcdCfg),
+		KubeconfigSecret:   util.AdminKubeconfigSecretName(name),
+		KarmadaCertsSecret: util.KarmadaCertSecretName(name),
+		EtcdCertsSecret:    util.EtcdCertSecretName(name),
+		Replicas:           cfg.Replicas,
 	})
 	if err != nil {
 		return fmt.Errorf("error when parsing karmadaAggregatedAPIServer deployment template: %w", err)

operator/pkg/controlplane/apiserver/mainfests.go

@@ -37,7 +37,7 @@ spec:
         - --etcd-cafile=/etc/etcd/pki/etcd-ca.crt
         - --etcd-certfile=/etc/etcd/pki/etcd-client.crt
         - --etcd-keyfile=/etc/etcd/pki/etcd-client.key
-        - --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
+        - --etcd-servers={{ .EtcdServers }}
         - --bind-address=0.0.0.0
         - --kubelet-client-certificate=/etc/karmada/pki/karmada.crt
         - --kubelet-client-key=/etc/karmada/pki/karmada.key
@@ -164,7 +164,7 @@ spec:
         - --etcd-cafile=/etc/etcd/pki/etcd-ca.crt
         - --etcd-certfile=/etc/etcd/pki/etcd-client.crt
         - --etcd-keyfile=/etc/etcd/pki/etcd-client.key
-        - --etcd-servers=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }}
+        - --etcd-servers={{ .EtcdServers }}
         - --tls-cert-file=/etc/karmada/pki/karmada.crt
         - --tls-private-key-file=/etc/karmada/pki/karmada.key
         - --audit-log-path=-

operator/pkg/tasks/init/apiserver.go

@@ -129,7 +129,8 @@ func runKarmadaAggregatedAPIServer(r workflow.RunData) error {
 
 	err := apiserver.EnsureKarmadaAggregatedAPIServer(
 		data.RemoteClient(),
-		cfg,
+		cfg.KarmadaAggregatedAPIServer,
+		cfg.Etcd,
 		data.GetName(),
 		data.GetNamespace(),
 		data.FeatureGates())

operator/pkg/tasks/init/crd.go

@@ -85,6 +85,7 @@ func runCrdsDownload(r workflow.RunData) error {
 	if err != nil {
 		return err
 	}
+
 	if !exist {
 		if err := os.MkdirAll(crdsDir, 0755); err != nil {
 			return err

operator/pkg/tasks/init/etcd.go

@@ -46,12 +46,11 @@ func runDeployEtcd(r workflow.RunData) error {
 	if !ok {
 		return errors.New("deploy-etcd task invoked with an invalid data struct")
 	}
-
-	cfg := data.Components()
-	if cfg.Etcd.External != nil {
+	if runWithExtEtcd(data) {
 		klog.V(2).InfoS("[etcd] use external etcd, skip install etcd job", "karmada", data.GetName())
 		return nil
 	}
+	cfg := data.Components()
 
 	if cfg.Etcd.Local == nil {
 		return errors.New("unexpect empty etcd local configuration")
@@ -71,6 +70,10 @@ func runWaitEtcd(r workflow.RunData) error {
 	if !ok {
 		return errors.New("wait-etcd task invoked with an invalid data struct")
 	}
+	if runWithExtEtcd(data) {
+		klog.V(2).InfoS("[etcd] use external etcd, skip waiting the etcd pod", "karmada", data.GetName())
+		return nil
+	}
 
 	waiter := apiclient.NewKarmadaWaiter(data.ControlplaneConfig(), data.RemoteClient(), time.Second*30)
 
@@ -83,3 +86,7 @@ func runWaitEtcd(r workflow.RunData) error {
 	klog.V(2).InfoS("[wait-etcd] the etcd pods is ready", "karmada", klog.KObj(data))
 	return nil
 }
+
+func runWithExtEtcd(data InitData) bool {
+	return data.Components().Etcd.External != nil
+}