create empty pull secret in the operator ns (#438)

Signed-off-by: Zhiwei Yin <[email protected]>

.gitignore

@@ -28,3 +28,6 @@ launch.json
 
 # files generated by editors
 .vscode/
+
+# used in e2e test
+config.json

pkg/cmd/init/cmd.go

@@ -59,10 +59,12 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
 	cmd.Flags().StringToStringVar(&o.resourceRequests, "resource-requests", nil, "the resource requests of all the containers managed by the cluster manager and the cluster manager operator, for example: cpu=500m,memory=500Mi")
 	cmd.Flags().BoolVar(&o.createNamespace, "create-namespace", true, "If true, create open-cluster-management namespace, otherwise use existing one")
 
-	//clusterManagetSet contains the flags for deploy cluster-manager
+	// clusterManagetSet contains the flags for deploy cluster-manager
 	clusterManagerSet := pflag.NewFlagSet("clusterManagerSet", pflag.ExitOnError)
 	cmd.Flags().StringVar(&o.registry, "image-registry", "quay.io/open-cluster-management",
 		"The name of the image registry serving OCM images, which will be applied to all the deploying OCM components.")
+	cmd.Flags().StringVar(&o.imagePullCredFile, "image-pull-credential-file", "",
+		"The credential file is the docker config json file and will be filled into the default image pull secret named open-cluster-management-image-pull-credentials.")
 	cmd.Flags().StringVar(&o.bundleVersion, "bundle-version", "default",
 		"The version of predefined compatible image versions (e.g. v0.6.0). Defaults to the latest released version. You can also set \"latest\" to install the latest development version.")
 	clusterManagerSet.BoolVar(&o.useBootstrapToken, "use-bootstrap-token", false, "If set then the bootstrap token will used instead of a service account token")

pkg/cmd/init/exec.go

@@ -3,6 +3,7 @@ package init
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"os"
 	"time"
@@ -59,15 +60,25 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 	if !o.singleton {
 		o.values = scenario.Values{
 			Hub: scenario.Hub{
-				TokenID:     helpers.RandStringRunes_az09(6),
-				TokenSecret: helpers.RandStringRunes_az09(16),
-				Registry:    o.registry,
+				TokenID:       helpers.RandStringRunes_az09(6),
+				TokenSecret:   helpers.RandStringRunes_az09(16),
+				Registry:      o.registry,
+				ImagePullCred: "e30K", // e30K is the base64 string of '{}'
 			},
 			AutoApprove:          genericclioptionsclusteradm.HubMutableFeatureGate.Enabled(ocmfeature.ManagedClusterAutoApproval),
 			RegistrationFeatures: genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.HubMutableFeatureGate, ocmfeature.DefaultHubRegistrationFeatureGates),
 			WorkFeatures:         genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.HubMutableFeatureGate, ocmfeature.DefaultHubWorkFeatureGates),
 			AddonFeatures:        genericclioptionsclusteradm.ConvertToFeatureGateAPI(genericclioptionsclusteradm.HubMutableFeatureGate, ocmfeature.DefaultHubAddonManagerFeatureGates),
 		}
+
+		if o.imagePullCredFile != "" {
+			data, err := os.ReadFile(o.imagePullCredFile)
+			if err != nil {
+				return fmt.Errorf("failed read the image pull credential file %v: %v", o.imagePullCredFile, err)
+			}
+			o.values.Hub.ImagePullCred = base64.StdEncoding.EncodeToString(data)
+		}
+
 		resourceRequirement, err := resourcerequirement.NewResourceRequirement(o.resourceQosClass, o.resourceLimits, o.resourceRequests)
 		if err != nil {
 			return err
@@ -180,6 +191,7 @@ func (o *Options) run() error {
 			"init/clustermanager_cluster_role_binding.yaml",
 			"init/clustermanagers.crd.yaml",
 			"init/clustermanager_sa.yaml",
+			"init/image-pull-secret.yaml",
 		)
 
 		r := reader.NewResourceReader(o.ClusteradmFlags.KubectlFactory, o.ClusteradmFlags.DryRun, o.Streams)
@@ -219,7 +231,7 @@ func (o *Options) run() error {
 			}
 		}
 
-		//if service-account wait for the sa secret
+		// if service-account wait for the sa secret
 		if !o.useBootstrapToken && !o.ClusteradmFlags.DryRun {
 			token, err = helpers.GetBootstrapTokenFromSA(context.TODO(), kubeClient)
 			if err != nil {

pkg/cmd/init/options.go

@@ -10,21 +10,26 @@ import (
 
 // Options is holding all the command-line options
 type Options struct {
-	//ClusteradmFlags: The generic options from the clusteradm cli-runtime.
+	// ClusteradmFlags: The generic options from the clusteradm cli-runtime.
 	ClusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags
 	values          scenario.Values
-	//The file to output the resources will be sent to the file.
+	// The file to output the resources will be sent to the file.
 	outputFile string
-	//If true the bootstrap token will be used instead of the service account token
+	// If true the bootstrap token will be used instead of the service account token
 	useBootstrapToken bool
-	//if true the hub will be reinstalled
+	// if true the hub will be reinstalled
 	force bool
-	//Pulling image registry of OCM
+	// Pulling image registry of OCM
 	registry string
-	//version of predefined compatible image versions
+
+	// imagePullCredFile is a credential file is used to pull image which should be docker credentials json file and
+	// will be filled into the secret open-cluster-management-image-pull-credentials.
+	imagePullCredFile string
+
+	// version of predefined compatible image versions
 	bundleVersion string
 
-	//If set, deploy the singleton controlplane
+	// If set, deploy the singleton controlplane
 	singleton     bool
 	SingletonName string
 	Helm          *helm.Helm
@@ -37,9 +42,9 @@ type Options struct {
 	// If create ns or use existing ns
 	createNamespace bool
 
-	//If set, will be persisting the generated join command to a local file
+	// If set, will be persisting the generated join command to a local file
 	outputJoinCommandFile string
-	//If set, the command will hold until the OCM control plane initialized
+	// If set, the command will hold until the OCM control plane initialized
 	wait bool
 	//
 	output string

pkg/cmd/init/scenario/init/clustermanager_sa.yaml

@@ -4,3 +4,5 @@ kind: ServiceAccount
 metadata:
   name: cluster-manager
   namespace: open-cluster-management
+imagePullSecrets:
+  - name: open-cluster-management-image-pull-credentials

pkg/cmd/init/scenario/init/image-pull-secret.yaml

@@ -0,0 +1,11 @@
+# Copyright Contributors to the Open Cluster Management project
+apiVersion: v1
+kind: Secret
+metadata:
+  name: open-cluster-management-image-pull-credentials
+  namespace: open-cluster-management
+  labels:
+    app: cluster-manager
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: {{ .Hub.ImagePullCred }}

pkg/cmd/init/scenario/resources.go

@@ -26,9 +26,9 @@ type BundleVersion struct {
 
 // Values: The values used in the template
 type Values struct {
-	//The values related to the hub
+	// The values related to the hub
 	Hub Hub `json:"hub"`
-	//bundle version
+	// bundle version
 	BundleVersion BundleVersion
 
 	// if enable auto approve
@@ -50,10 +50,14 @@ type Values struct {
 
 // Hub: The hub values for the template
 type Hub struct {
-	//TokenID: A token id allowing the cluster to connect back to the hub
+	// TokenID: A token id allowing the cluster to connect back to the hub
 	TokenID string `json:"tokenID"`
-	//TokenSecret: A token secret allowing the cluster to connect back to the hub
+	// TokenSecret: A token secret allowing the cluster to connect back to the hub
 	TokenSecret string `json:"tokenSecret"`
 	// Registry is the name of the image registry to pull.
 	Registry string `json:"registry"`
+
+	// ImagePullCred is the credential used to pull image. should be a base64 string and will be filled into the default image pull secret
+	// named open-cluster-management-image-pull-credentials.
+	ImagePullCred string `json:"imagePullCred"`
 }

pkg/cmd/join/cmd.go

@@ -55,6 +55,8 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
 	cmd.Flags().StringVar(&o.clusterName, "cluster-name", "", "The name of the joining cluster")
 	cmd.Flags().StringVar(&o.outputFile, "output-file", "", "The generated resources will be copied in the specified file")
 	cmd.Flags().StringVar(&o.registry, "image-registry", "quay.io/open-cluster-management", "The name of the image registry serving OCM images.")
+	cmd.Flags().StringVar(&o.imagePullCredFile, "image-pull-credential-file", "",
+		"The credential file is the docker config json file and will be filled into the default image pull secret named open-cluster-management-image-pull-credentials.")
 	cmd.Flags().StringVar(&o.bundleVersion, "bundle-version", "default",
 		"The version of predefined compatible image versions (e.g. v0.6.0). Defaults to the latest released version. You can also set \"latest\" to install the latest development version.")
 	cmd.Flags().BoolVar(&o.forceHubInClusterEndpointLookup, "force-internal-endpoint-lookup", false,

pkg/cmd/join/exec.go

@@ -96,9 +96,19 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 			APIServer: o.hubAPIServer,
 		},
 		Registry:         o.registry,
+		ImagePullCred:    "e30K", // e30K is the base64 string of '{}'
 		AgentNamespace:   agentNamespace,
 		EnableSyncLabels: o.enableSyncLabels,
 	}
+
+	if o.imagePullCredFile != "" {
+		data, err := os.ReadFile(o.imagePullCredFile)
+		if err != nil {
+			return fmt.Errorf("failed read the image pull credential file %v: %v", o.imagePullCredFile, err)
+		}
+		o.values.ImagePullCred = base64.StdEncoding.EncodeToString(data)
+	}
+
 	// deploy klusterlet
 	// operatorNamespace is the namespace to deploy klsuterlet;
 	// agentNamespace is the namesapce to deploy the agents(registration agent, work agent, etc.);
@@ -179,14 +189,14 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 	// 2. if not found, assume using an authorized ca.
 	// 3. use the ca and token to build a secured client and call hub
 
-	//Create an unsecure bootstrap
+	// Create an unsecure bootstrap
 	bootstrapExternalConfigUnSecure := o.createExternalBootstrapConfig()
-	//create external client from the bootstrap
+	// create external client from the bootstrap
 	externalClientUnSecure, err := helpers.CreateClientFromClientcmdapiv1Config(bootstrapExternalConfigUnSecure)
 	if err != nil {
 		return err
 	}
-	//Create the kubeconfig for the internal client
+	// Create the kubeconfig for the internal client
 	o.HubConfig, err = o.createClientcmdapiv1Config(externalClientUnSecure, bootstrapExternalConfigUnSecure)
 	if err != nil {
 		return err
@@ -366,6 +376,7 @@ func (o *Options) applyKlusterlet(r *reader.ResourceReader, operatorClient opera
 		files = append(files,
 			"join/klusterlets.crd.yaml",
 			"join/service_account.yaml",
+			"join/image-pull-secret.yaml",
 			"join/cluster_role.yaml",
 			"join/cluster_role_binding.yaml",
 			"join/operator.yaml",
@@ -466,7 +477,7 @@ func checkIfRegistrationOperatorAvailable(f util.Factory) (bool, error) {
 }
 
 func (o *Options) waitUntilManagedClusterIsCreated(timeout int64, clusterName string) error {
-	//Create an unsecure bootstrap
+	// Create an unsecure bootstrap
 	bootstrapExternalConfigUnSecure := o.createExternalBootstrapConfig()
 	restConfig, err := helpers.CreateRESTConfigFromClientcmdapiv1Config(bootstrapExternalConfigUnSecure)
 	if err != nil {

pkg/cmd/join/options.go

@@ -30,6 +30,11 @@ type Options struct {
 	managedKubeconfigFile string
 	// Pulling image registry of OCM
 	registry string
+
+	// imagePullCredFile is a credential file is used to pull image which should be docker credentials json file and
+	// will be filled into the secret open-cluster-management-image-pull-credentials.
+	imagePullCredFile string
+
 	// version of predefined compatible image versions
 	bundleVersion string
 

pkg/cmd/join/scenario/join/image-pull-secret.yaml

@@ -0,0 +1,11 @@
+# Copyright Contributors to the Open Cluster Management project
+apiVersion: v1
+kind: Secret
+metadata:
+  name: open-cluster-management-image-pull-credentials
+  namespace: open-cluster-management
+  labels:
+    app: cluster-manager
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: {{ .ImagePullCred }}

pkg/cmd/join/scenario/join/service_account.yaml

@@ -4,3 +4,5 @@ kind: ServiceAccount
 metadata:
   name: klusterlet
   namespace: open-cluster-management
+imagePullSecrets:
+  - name: open-cluster-management-image-pull-credentials

pkg/cmd/join/scenario/resources.go

@@ -23,6 +23,11 @@ type Values struct {
 	Klusterlet Klusterlet
 	// Registry is the image registry related configuration
 	Registry string
+
+	// ImagePullCred is the credential used to pull image. should be a base64 string and will be filled into the
+	// default image pull secret named open-cluster-management-image-pull-credentials.
+	ImagePullCred string
+
 	// bundle version
 	BundleVersion BundleVersion
 	// managed kubeconfig

pkg/config/env.go

@@ -14,4 +14,5 @@ const (
 	HubClusterNamespace               = "open-cluster-management-hub"
 	ManagedClusterNamespace           = "open-cluster-management-agent"
 	ManagedProxyConfigurationName     = "cluster-proxy"
+	ImagePullSecret                   = "open-cluster-management-image-pull-credentials"
 )

pkg/helpers/client.go

@@ -101,7 +101,7 @@ func GetToken(ctx context.Context, kubeClient kubernetes.Interface) (string, Tok
 	token, err := GetBootstrapTokenFromSA(ctx, kubeClient)
 	if err != nil {
 		if errors.IsNotFound(err) {
-			//As no SA search for bootstrap token
+			// As no SA search for bootstrap token
 			var token string
 			token, err = GetBootstrapToken(ctx, kubeClient)
 			if err == nil {
@@ -122,7 +122,7 @@ func GetBootstrapSecret(ctx context.Context, kubeClient kubernetes.Interface) (*
 	if err != nil {
 		return nil, err
 	}
-	//sort items by creationTimestamp
+	// sort items by creationTimestamp
 	sort.Slice(l.Items, func(i, j int) bool {
 		return l.Items[j].CreationTimestamp.Before(&l.Items[i].CreationTimestamp)
 	})
@@ -213,7 +213,7 @@ func WatchUntil(
 	defer w.Stop()
 	for {
 		event, ok := <-w.ResultChan()
-		if !ok { //The channel is closed by Kubernetes, thus, user should check the pod status manually
+		if !ok { // The channel is closed by Kubernetes, thus, user should check the pod status manually
 			return fmt.Errorf("unexpected watch event received")
 		}
 

test/e2e/clusteradm/e2e_suite_test.go

@@ -26,7 +26,7 @@ var e2e *util.TestE2eConfig
 
 // var testEnv *envtest.Environment
 var restConfig *rest.Config
-var kubeClient kubernetes.Interface
+var kubeClient, managedClusterKubeClient kubernetes.Interface
 var apiExtensionsClient apiextensionsclient.Interface
 var dynamicClient dynamic.Interface
 var clusterClient clusterv1client.Interface
@@ -39,7 +39,7 @@ func TestE2EClusteradm(t *testing.T) {
 	suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration()
 	// adjust it
 	suiteConfig.SkipStrings = []string{"NEVER-RUN"}
-	//suiteConfig.FocusStrings = []string{"test clusteradm with manual bootstrap token"}
+	// suiteConfig.FocusStrings = []string{"test clusteradm with manual bootstrap token"}
 	reporterConfig.FullTrace = true
 
 	ginkgo.RunSpecs(t, "E2E clusteradm test", suiteConfig, reporterConfig)
@@ -66,6 +66,15 @@ var _ = ginkgo.BeforeSuite(func() {
 	kubeClient, err = kubernetes.NewForConfig(hubConfig)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
+	managedClusterCtx := os.Getenv("MANAGED_CLUSTER1_CTX")
+	managedClusterClientConfig := clientcmd.NewNonInteractiveClientConfig(*configapi,
+		managedClusterCtx, nil, clientcmd.NewDefaultClientConfigLoadingRules())
+	managedClusterConfig, err := managedClusterClientConfig.ClientConfig()
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	managedClusterKubeClient, err = kubernetes.NewForConfig(managedClusterConfig)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
 	apiExtensionsClient, err = apiextensionsclient.NewForConfig(hubConfig)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	dynamicClient, err = dynamic.NewForConfig(hubConfig)

test/e2e/clusteradm/init_test.go

@@ -4,10 +4,12 @@ package clusteradme2e
 
 import (
 	"context"
+	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"open-cluster-management.io/clusteradm/test/e2e/util"
 )
 
 var _ = ginkgo.Describe("test clusteradm with bootstrap token in singleton mode", func() {
@@ -18,6 +20,7 @@ var _ = ginkgo.Describe("test clusteradm with bootstrap token in singleton mode"
 	})
 
 	ginkgo.Context("init cluster manager", func() {
+
 		ginkgo.It("should init multiple times with different flags", func() {
 			ginkgo.By("init hub with bootstrap token")
 			err := e2e.Clusteradm().Init(
@@ -41,6 +44,28 @@ var _ = ginkgo.Describe("test clusteradm with bootstrap token in singleton mode"
 			cm, err = operatorClient.OperatorV1().ClusterManagers().Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 			gomega.Expect(len(cm.Spec.RegistrationConfiguration.FeatureGates)).Should(gomega.Equal(2))
+
+			gomega.Eventually(func() error {
+				return util.ValidateImagePullSecret(kubeClient, "e30K",
+					"open-cluster-management")
+			}, time.Second*120, time.Second*2).ShouldNot(gomega.HaveOccurred())
+
+			// set image-pull-credential
+			encodedString := util.NewTestImagePullCredentialFile("config.json")
+			err = e2e.Clusteradm().Init(
+				"--use-bootstrap-token",
+				"--context", e2e.Cluster().Hub().Context(),
+				"--feature-gates=ManagedClusterAutoApproval=true",
+				"--bundle-version=latest",
+				"--image-pull-credential-file=./config.json",
+			)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
+
+			util.CleanupTestImagePullCredentialFile("config.json")
+			gomega.Eventually(func() error {
+				return util.ValidateImagePullSecret(kubeClient, encodedString,
+					"open-cluster-management")
+			}, time.Second*120, time.Second*2).ShouldNot(gomega.HaveOccurred())
 		})
 	})
 })