Refactor singleton mode to use image built in ocm

Signed-off-by: Jian Qiu <[email protected]>

go.mod

@@ -28,7 +28,7 @@ require (
 	k8s.io/klog/v2 v2.90.1
 	k8s.io/kubectl v0.27.1
 	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
-	open-cluster-management.io/api v0.11.0
+	open-cluster-management.io/api v0.11.1-0.20230905055724-cf1ead467a83
 	open-cluster-management.io/cluster-proxy v0.1.2
 	open-cluster-management.io/managed-serviceaccount v0.2.1-0.20220427065210-de6a7b7b5be8
 	sigs.k8s.io/apiserver-network-proxy v0.1.2

go.sum

@@ -1601,8 +1601,8 @@ modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
 open-cluster-management.io/addon-framework v0.1.1-0.20211223101009-d6b1a7adae93/go.mod h1:reAEgSFo9UX+TzfGjhHMWKmrOC4hJYxBHeubqllDbhk=
 open-cluster-management.io/api v0.5.0/go.mod h1:9qiA5h/8kvPQnJEOlAPHVjRO9a1jCmDhGzvgMBvXEaE=
-open-cluster-management.io/api v0.11.0 h1:zBxa33Co3wseLBF4HEJobhl0P6ygj+Drhe7Wrfo0/h8=
-open-cluster-management.io/api v0.11.0/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
+open-cluster-management.io/api v0.11.1-0.20230905055724-cf1ead467a83 h1:3zbT3sT/tEAQbpjIk6uRiTQGknQ3kQlfd11ElVuXyyQ=
+open-cluster-management.io/api v0.11.1-0.20230905055724-cf1ead467a83/go.mod h1:nsQ/G5JpfjQUg7dHpblyywWC6BRqklNaF6fIswVCHyY=
 open-cluster-management.io/cluster-proxy v0.1.2 h1:0WYhPEZT6Wlt0dyE35GCC7zcqPo2poaALzdmVbQp9Dg=
 open-cluster-management.io/cluster-proxy v0.1.2/go.mod h1:f4HVfparQqOJsPrh4fTA4uyWx5KJ7MG2AcuSRypedws=
 open-cluster-management.io/managed-serviceaccount v0.2.1-0.20220427065210-de6a7b7b5be8 h1:foP5RkSec9DkUA8sJ5FLna5We8VxJyu0179id5zNXgA=

pkg/cmd/init/scenario/init/clustermanagers.crd.yaml

@@ -89,7 +89,7 @@ spec:
                   Default mode is used if DeployOption is not set.
                 properties:
                   hosted:
-                    description: Hosted includes configurations we needs for clustermanager
+                    description: Hosted includes configurations we need for clustermanager
                       in the Hosted mode.
                     properties:
                       registrationWebhookConfiguration:
@@ -161,8 +161,8 @@ spec:
                       on. The default is an empty list.
                     type: object
                   tolerations:
-                    description: Tolerations is attached by pods to tolerate any taint
-                      that matches the triple <key,value,effect> using the matching
+                    description: Tolerations are attached by pods to tolerate any
+                      taint that matches the triple <key,value,effect> using the matching
                       operator <operator>. The default is an empty list.
                     items:
                       description: The pod this Toleration is attached to tolerates

pkg/cmd/join/exec.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/kubectl/pkg/cmd/util"
 	ocmfeature "open-cluster-management.io/api/feature"
+	operatorv1 "open-cluster-management.io/api/operator/v1"
 	"open-cluster-management.io/clusteradm/pkg/cmd/join/preflight"
 	"open-cluster-management.io/clusteradm/pkg/cmd/join/scenario"
 	genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
@@ -88,42 +89,45 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 		Registry:       o.registry,
 		AgentNamespace: agentNamespace,
 	}
+	// deploy klusterlet
+	// operatorNamespace is the namespace to deploy klsuterlet;
+	// agentNamespace is the namesapce to deploy the agents(registration agent, work agent, etc.);
+	// klusterletNamespace is the namespace created on the managed cluster for each klusterlet.
+	//
+	// The operatorNamespace is fixed to "open-cluster-management".
+	// In default mode, agentNamespace is "open-cluster-management-agent", klusterletNamespace refers to agentNamespace, all of these three namesapces are on the managed cluster;
+	// In hosted mode, operatorNamespace is on the management cluster, agentNamesapce is "<cluster name>-<6-bit random string>" on the management cluster, and the klusterletNamespace is "open-cluster-management-<agentNamespace>" on the managed cluster.
+
+	// values for default mode
+	klusterletName := DefaultOperatorName
+	klusterletNamespace := agentNamespace
+	if o.mode == InstallModeHosted {
+		// add hash suffix to avoid conflict
+		klusterletName += "-hosted-" + helpers.RandStringRunes_az09(6)
+		agentNamespace = klusterletName
+		klusterletNamespace = AgentNamespacePrefix + agentNamespace
 
-	if o.singleton { // deploy singleton agent
-		if o.mode != InstallModeDefault {
-			return fmt.Errorf("only default mode is supported while deploy singleton agent, hosted mode will be supported in the future")
-		}
-	} else { // deploy klusterlet
-		// operatorNamespace is the namespace to deploy klsuterlet;
-		// agentNamespace is the namesapce to deploy the agents(registration agent, work agent, etc.);
-		// klusterletNamespace is the namespace created on the managed cluster for each klusterlet.
-		//
-		// The operatorNamespace is fixed to "open-cluster-management".
-		// In default mode, agentNamespace is "open-cluster-management-agent", klusterletNamespace refers to agentNamespace, all of these three namesapces are on the managed cluster;
-		// In hosted mode, operatorNamespace is on the management cluster, agentNamesapce is "<cluster name>-<6-bit random string>" on the management cluster, and the klusterletNamespace is "open-cluster-management-<agentNamespace>" on the managed cluster.
-
-		// values for default mode
-		klusterletName := DefaultOperatorName
-		klusterletNamespace := agentNamespace
-		if o.mode == InstallModeHosted {
-			// add hash suffix to avoid conflict
-			klusterletName += "-hosted-" + helpers.RandStringRunes_az09(6)
-			agentNamespace = klusterletName
-			klusterletNamespace = AgentNamespacePrefix + agentNamespace
+		// update AgentNamespace
+		o.values.AgentNamespace = agentNamespace
+	}
 
-			// update AgentNamespace
-			o.values.AgentNamespace = agentNamespace
-		}
+	o.values.Klusterlet = Klusterlet{
+		Name:                klusterletName,
+		KlusterletNamespace: klusterletNamespace,
+	}
+	o.values.ManagedKubeconfig = o.managedKubeconfigFile
+	o.values.RegistrationFeatures = genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeRegistrationFeatureGates)
+	o.values.WorkFeatures = genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeWorkFeatureGates)
 
-		o.values.Klusterlet = Klusterlet{
-			Mode:                o.mode,
-			Name:                klusterletName,
-			KlusterletNamespace: klusterletNamespace,
-		}
-		o.values.ManagedKubeconfig = o.managedKubeconfigFile
-		o.values.RegistrationFeatures = genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeRegistrationFeatureGates)
-		o.values.WorkFeatures = genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.SpokeMutableFeatureGate, ocmfeature.DefaultSpokeWorkFeatureGates)
+	// set mode based on mode and singleton
+	if o.mode == InstallModeHosted && o.singleton {
+		o.values.Klusterlet.Mode = string(operatorv1.InstallModeSingletonHosted)
+	} else if o.singleton {
+		o.values.Klusterlet.Mode = string(operatorv1.InstallModeSingleton)
+	} else {
+		o.values.Klusterlet.Mode = o.mode
 	}
+
 	versionBundle, err := version.GetVersionBundle(o.bundleVersion)
 
 	if err != nil {
@@ -132,11 +136,10 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 	}
 
 	o.values.BundleVersion = BundleVersion{
-		RegistrationImageVersion:   versionBundle.Registration,
-		PlacementImageVersion:      versionBundle.Placement,
-		WorkImageVersion:           versionBundle.Work,
-		OperatorImageVersion:       versionBundle.Operator,
-		SingletonAgentImageVersion: versionBundle.MulticlusterControlplane,
+		RegistrationImageVersion: versionBundle.Registration,
+		PlacementImageVersion:    versionBundle.Placement,
+		WorkImageVersion:         versionBundle.Work,
+		OperatorImageVersion:     versionBundle.Operator,
 	}
 	klog.V(3).InfoS("Image version:",
 		"'registration image version'", versionBundle.Registration,
@@ -243,35 +246,9 @@ func (o *Options) run() error {
 
 	r := reader.NewResourceReader(o.builder, o.ClusteradmFlags.DryRun, o.Streams)
 
-	_, err = kubeClient.CoreV1().Namespaces().Get(context.TODO(), o.values.AgentNamespace, metav1.GetOptions{})
+	err = o.applyKlusterlet(r, kubeClient, apiExtensionsClient)
 	if err != nil {
-		if errors.IsNotFound(err) {
-			_, err = kubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{
-				ObjectMeta: metav1.ObjectMeta{
-					Name: o.values.AgentNamespace,
-					Annotations: map[string]string{
-						"workload.openshift.io/allowed": "management",
-					},
-				},
-			}, metav1.CreateOptions{})
-			if err != nil {
-				return err
-			}
-		} else {
-			return err
-		}
-	}
-
-	if o.singleton {
-		err = o.applySingletonAgent(r, kubeClient)
-		if err != nil {
-			return err
-		}
-	} else {
-		err = o.applyKlusterlet(r, kubeClient, apiExtensionsClient)
-		if err != nil {
-			return err
-		}
+		return err
 	}
 
 	if len(o.outputFile) > 0 {
@@ -294,32 +271,6 @@ func (o *Options) run() error {
 
 }
 
-func (o *Options) applySingletonAgent(r *reader.ResourceReader, kubeClient kubernetes.Interface) error {
-	files := []string{
-		"bootstrap_hub_kubeconfig.yaml",
-		"singleton/clusterrole.yaml",
-		"singleton/clusterrolebinding-admin.yaml",
-		"singleton/clusterrolebinding.yaml",
-		"singleton/role.yaml",
-		"singleton/rolebinding.yaml",
-		"singleton/serviceaccount.yaml",
-		"singleton/deployment.yaml",
-	}
-
-	err := r.Apply(scenario.Files, o.values, files...)
-	if err != nil {
-		return err
-	}
-
-	if o.wait && !o.ClusteradmFlags.DryRun {
-		err = waitUntilSingletonAgentConditionIsTrue(o.ClusteradmFlags.KubectlFactory, int64(o.ClusteradmFlags.Timeout), o.values.AgentNamespace)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (o *Options) applyKlusterlet(r *reader.ResourceReader, kubeClient kubernetes.Interface, apiExtensionsClient apiextensionsclient.Interface) error {
 	available, err := checkIfRegistrationOperatorAvailable(o.ClusteradmFlags.KubectlFactory)
 	if err != nil {
@@ -525,52 +476,6 @@ func waitUntilKlusterletConditionIsTrue(f util.Factory, timeout int64, agentName
 	)
 }
 
-func waitUntilSingletonAgentConditionIsTrue(f util.Factory, timeout int64, agentNamespace string) error {
-	client, err := f.KubernetesClientSet()
-	if err != nil {
-		return err
-	}
-
-	phase := &atomic.Value{}
-	phase.Store("")
-	agentSpinner := printer.NewSpinnerWithStatus(
-		"Waiting for controlplane agent to become ready...",
-		time.Millisecond*500,
-		"Controlplane agent is now available.\n",
-		func() string {
-			return phase.Load().(string)
-		})
-	agentSpinner.Start()
-	defer agentSpinner.Stop()
-
-	return helpers.WatchUntil(
-		func() (watch.Interface, error) {
-			return client.CoreV1().Pods(agentNamespace).
-				Watch(context.TODO(), metav1.ListOptions{
-					TimeoutSeconds: &timeout,
-					LabelSelector:  "app=multicluster-controlplane-agent",
-				})
-		},
-		func(event watch.Event) bool {
-			pod, ok := event.Object.(*corev1.Pod)
-			if !ok {
-				return false
-			}
-			phase.Store(printer.GetSpinnerPodStatus(pod))
-			conds := make([]metav1.Condition, len(pod.Status.Conditions))
-			for i := range pod.Status.Conditions {
-				conds[i] = metav1.Condition{
-					Type:    string(pod.Status.Conditions[i].Type),
-					Status:  metav1.ConditionStatus(pod.Status.Conditions[i].Status),
-					Reason:  pod.Status.Conditions[i].Reason,
-					Message: pod.Status.Conditions[i].Message,
-				}
-			}
-			return meta.IsStatusConditionTrue(conds, "Ready")
-		},
-	)
-}
-
 // Create bootstrap with token but without CA
 func (o *Options) createExternalBootstrapConfig() clientcmdapiv1.Config {
 	return clientcmdapiv1.Config{

pkg/cmd/join/options.go

@@ -110,8 +110,6 @@ type BundleVersion struct {
 	WorkImageVersion string
 	// operator image version
 	OperatorImageVersion string
-	// singleton agent image version
-	SingletonAgentImageVersion string
 }
 
 func newOptions(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, streams genericclioptions.IOStreams) *Options {

pkg/cmd/join/scenario/join/klusterlets.cr.yaml

@@ -7,7 +7,8 @@ spec:
   deployOption:
     mode: {{ .Klusterlet.Mode }}
   registrationImagePullSpec: {{ .Registry }}/registration:{{ .BundleVersion.RegistrationImageVersion }}
-  workImagePullSpec: {{ .Registry }}/work:{{ .BundleVersion.RegistrationImageVersion }}  
+  workImagePullSpec: {{ .Registry }}/work:{{ .BundleVersion.RegistrationImageVersion }}
+  imagePullSpec: {{.Registry}}/registration-operator:{{.BundleVersion.OperatorImageVersion}}
   clusterName: {{ .ClusterName }}
   namespace: {{ .Klusterlet.KlusterletNamespace }} 
   externalServerURLs:

pkg/cmd/join/scenario/join/klusterlets.crd.yaml

@@ -50,20 +50,22 @@ spec:
                 description: DeployOption contains the options of deploying a klusterlet
                 properties:
                   mode:
-                    description: 'Mode can be Default or Hosted. It is Default mode
-                      if not specified In Default mode, all klusterlet related resources
-                      are deployed on the managed cluster. In Hosted mode, only crd
-                      and configurations are installed on the spoke/managed cluster.
-                      Controllers run in another cluster (defined as management-cluster)
+                    description: 'Mode can be Default, Hosted, Singleton or SingletonHosted.
+                      It is Default mode if not specified In Default mode, all klusterlet
+                      related resources are deployed on the managed cluster. In Hosted
+                      mode, only crd and configurations are installed on the spoke/managed
+                      cluster. Controllers run in another cluster (defined as management-cluster)
                       and connect to the mangaged cluster with the kubeconfig in secret
                       of "external-managed-kubeconfig"(a kubeconfig of managed-cluster
-                      with cluster-admin permission). Note: Do not modify the Mode
-                      field once it''s applied.'
+                      with cluster-admin permission). In Singleton mode, registration/work
+                      agent is started as a single deployment. In SingletonHosted
+                      mode, agent is started as a single deployment in hosted mode.
+                      Note: Do not modify the Mode field once it''s applied.'
                     type: string
                 type: object
               externalServerURLs:
-                description: ExternalServerURLs represents the a list of apiserver
-                  urls and ca bundles that is accessible externally If it is set empty,
+                description: ExternalServerURLs represents a list of apiserver urls
+                  and ca bundles that is accessible externally If it is set empty,
                   managed cluster has no externally accessible url that hub cluster
                   can visit.
                 items:
@@ -99,6 +101,11 @@ spec:
                 - hostname
                 - ip
                 type: object
+              imagePullSpec:
+                description: ImagePullSpec represents the desired image configuration
+                  of agent, it takes effect only when singleton mode is set. quay.io/open-cluster-management.io/registration-operator:latest
+                  will be used if unspecified
+                type: string
               namespace:
                 description: Namespace is the namespace to deploy the agent on the
                   managed cluster. The namespace must have a prefix of "open-cluster-management-",
@@ -123,8 +130,8 @@ spec:
                       on. The default is an empty list.
                     type: object
                   tolerations:
-                    description: Tolerations is attached by pods to tolerate any taint
-                      that matches the triple <key,value,effect> using the matching
+                    description: Tolerations are attached by pods to tolerate any
+                      taint that matches the triple <key,value,effect> using the matching
                       operator <operator>. The default is an empty list.
                     items:
                       description: The pod this Toleration is attached to tolerates
@@ -179,6 +186,13 @@ spec:
                       set.
                     format: int32
                     type: integer
+                  clusterAnnotations:
+                    additionalProperties:
+                      type: string
+                    description: ClusterAnnotations is annotations with the reserve
+                      prefix "agent.open-cluster-management.io" set on ManagedCluster
+                      when creating only, other actors can update it afterwards.
+                    type: object
                   featureGates:
                     description: 'FeatureGates represents the list of feature gates
                       for registration If it is set empty, default feature gates will

pkg/cmd/join/scenario/join/operator.yaml

@@ -42,7 +42,7 @@ spec:
       serviceAccountName: klusterlet
       containers:
       - name: klusterlet
-        image: {{ .Registry }}/registration-operator:{{ .BundleVersion.RegistrationImageVersion }}
+        image: {{ .Registry }}/registration-operator:{{ .BundleVersion.OperatorImageVersion }}
         args:
           - "/registration-operator"
           - "klusterlet"

vendor/modules.txt

@@ -1198,7 +1198,7 @@ k8s.io/utils/net
 k8s.io/utils/pointer
 k8s.io/utils/strings/slices
 k8s.io/utils/trace
-# open-cluster-management.io/api v0.11.0
+# open-cluster-management.io/api v0.11.1-0.20230905055724-cf1ead467a83
 ## explicit; go 1.19
 open-cluster-management.io/api/addon/v1alpha1
 open-cluster-management.io/api/client/addon/clientset/versioned