feat: V1beta CSR api compatible by apply agent-client secret via work (…

…#83)

Signed-off-by: yue9944882 <[email protected]>

cmd/addon-manager/main.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
+	addonutil "open-cluster-management.io/addon-framework/pkg/utils"
 	addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	"open-cluster-management.io/api/client/addon/clientset/versioned"
 	"open-cluster-management.io/api/client/addon/informers/externalversions"
@@ -114,6 +115,21 @@ func main() {
 		os.Exit(1)
 	}
 
+	supportsV1CSR, supportsV1beta1CSR, err := addonutil.IsCSRSupported(nativeClient)
+	if err != nil {
+		setupLog.Error(err, "unable to detect available CSR API versions")
+		os.Exit(1)
+	}
+
+	if supportsV1CSR {
+		setupLog.Info("V1 CSR API found")
+	} else if supportsV1beta1CSR {
+		setupLog.Info("V1 CSR API not found, falling back to v1beta1")
+	} else {
+		setupLog.Error(err, "No supported CSR api found")
+		os.Exit(1)
+	}
+
 	informerFactory := externalversions.NewSharedInformerFactory(client, 0)
 	nativeInformer := informers.NewSharedInformerFactoryWithOptions(nativeClient, 0)
 
@@ -130,6 +146,7 @@ func main() {
 		selfSigner,
 		nativeClient,
 		nativeInformer.Core().V1().Secrets(),
+		supportsV1CSR,
 	); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ClusterManagementAddonReconciler")
 		os.Exit(1)
@@ -154,16 +171,11 @@ func main() {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}
-	caCertData, caKeyData, err := selfSigner.CA().Config.GetPEMBytes()
-	if err != nil {
-		setupLog.Error(err, "unable to get content from signer CA")
-		os.Exit(1)
-	}
 
 	clusterProxyAddon, err := agent.NewAgentAddon(
-		mgr.GetScheme(),
-		caCertData,
-		caKeyData,
+		selfSigner,
+		signerSecretNamespace,
+		supportsV1CSR,
 		mgr.GetClient(),
 		nativeClient)
 	if err != nil {

go.mod

@@ -14,7 +14,7 @@ require (
 	k8s.io/client-go v0.23.0
 	k8s.io/klog/v2 v2.30.0
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
-	open-cluster-management.io/addon-framework v0.2.1-0.20220217081958-3a1dfe85a86f
+	open-cluster-management.io/addon-framework v0.2.1-0.20220317063747-100a0230a883
 	open-cluster-management.io/api v0.5.1-0.20220112073018-2d280a97a052
 	sigs.k8s.io/apiserver-network-proxy v0.0.27
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.27

go.sum

@@ -976,6 +976,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yue9944882/addon-framework v0.0.0-20220315104416-d9192aa14201 h1:kRRX8O2g/sMT+m7mbTqGV+zRBFvSbEQHAiUVnmk9GNU=
+github.com/yue9944882/addon-framework v0.0.0-20220315104416-d9192aa14201/go.mod h1:8QG1fHwPCEdyRrzY2Jji+qhMmJl+/HzTpuSDVmeG+7s=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -1681,8 +1683,8 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
-open-cluster-management.io/addon-framework v0.2.1-0.20220217081958-3a1dfe85a86f h1:XEFw2gaAaS0nPKmE7uJNEZ1XiSD7D4XcQVo8bPZVov0=
-open-cluster-management.io/addon-framework v0.2.1-0.20220217081958-3a1dfe85a86f/go.mod h1:8QG1fHwPCEdyRrzY2Jji+qhMmJl+/HzTpuSDVmeG+7s=
+open-cluster-management.io/addon-framework v0.2.1-0.20220317063747-100a0230a883 h1:NJdMI1w60WArizywHhUwEHU+Z/i3/fynuNU45e8pOKQ=
+open-cluster-management.io/addon-framework v0.2.1-0.20220317063747-100a0230a883/go.mod h1:8QG1fHwPCEdyRrzY2Jji+qhMmJl+/HzTpuSDVmeG+7s=
 open-cluster-management.io/api v0.5.1-0.20220112073018-2d280a97a052 h1:W+0vJSBH5o2uKWNhCVySIjsiew7fe8KAN9aGnWXo9W8=
 open-cluster-management.io/api v0.5.1-0.20220112073018-2d280a97a052/go.mod h1:0IUTh8J+p4pv1THh1r9oO0luX9Z1FLDEAmvzW09qC0o=
 oras.land/oras-go v0.4.0/go.mod h1:VJcU+VE4rkclUbum5C0O7deEZbBYnsnpbGSACwTjOcg=

pkg/common/constants.go

@@ -33,3 +33,7 @@ const (
 	LabelKeyComponentName                = "proxy.open-cluster-management.io/component-name"
 	AnnotationKeyConfigurationGeneration = "proxy.open-cluster-management.io/configuration-generation"
 )
+
+const (
+	AgentClientSecretName = "agent-client"
+)

pkg/proxyagent/agent/agent.go

@@ -14,7 +14,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"open-cluster-management.io/addon-framework/pkg/addonfactory"
@@ -25,6 +24,7 @@ import (
 	proxyv1alpha1 "open-cluster-management.io/cluster-proxy/pkg/apis/proxy/v1alpha1"
 	"open-cluster-management.io/cluster-proxy/pkg/common"
 	"open-cluster-management.io/cluster-proxy/pkg/config"
+	"open-cluster-management.io/cluster-proxy/pkg/proxyserver/operator/authentication/selfsigned"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -35,30 +35,39 @@ const (
 	ProxyAgentSignerName = "open-cluster-management.io/proxy-agent-signer"
 )
 
-func NewAgentAddon(scheme *runtime.Scheme, caCertData, caKeyData []byte, runtimeClient client.Client, nativeClient kubernetes.Interface) (agent.AgentAddon, error) {
+func NewAgentAddon(signer selfsigned.SelfSigner, signerNamespace string, v1CSRSupported bool, runtimeClient client.Client, nativeClient kubernetes.Interface) (agent.AgentAddon, error) {
+	caCertData, caKeyData, err := signer.CA().Config.GetPEMBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	regConfigs := []addonv1alpha1.RegistrationConfig{
+		{
+			SignerName: csrv1.KubeAPIServerClientSignerName,
+			Subject: addonv1alpha1.Subject{
+				User: common.SubjectUserClusterAddonAgent,
+				Groups: []string{
+					common.SubjectGroupClusterProxy,
+				},
+			},
+		},
+	}
+	// Register the custom signer CSR option if V1 csr is supported
+	if v1CSRSupported {
+		regConfigs = append(regConfigs, addonv1alpha1.RegistrationConfig{
+			SignerName: ProxyAgentSignerName,
+			Subject: addonv1alpha1.Subject{
+				User: common.SubjectUserClusterProxyAgent,
+				Groups: []string{
+					common.SubjectGroupClusterProxy,
+				},
+			},
+		})
+	}
 	return addonfactory.NewAgentAddonFactory(common.AddonName, FS, "manifests/charts/addon-agent").
 		WithAgentRegistrationOption(&agent.RegistrationOption{
 			CSRConfigurations: func(cluster *clusterv1.ManagedCluster) []addonv1alpha1.RegistrationConfig {
-				return []addonv1alpha1.RegistrationConfig{
-					{
-						SignerName: ProxyAgentSignerName,
-						Subject: addonv1alpha1.Subject{
-							User: common.SubjectUserClusterProxyAgent,
-							Groups: []string{
-								common.SubjectGroupClusterProxy,
-							},
-						},
-					},
-					{
-						SignerName: csrv1.KubeAPIServerClientSignerName,
-						Subject: addonv1alpha1.Subject{
-							User: common.SubjectUserClusterAddonAgent,
-							Groups: []string{
-								common.SubjectGroupClusterProxy,
-							},
-						},
-					},
-				}
+				return regConfigs
 			},
 			CSRApproveCheck: func(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn, csr *csrv1.CertificateSigningRequest) bool {
 				return cluster.Spec.HubAcceptsClient
@@ -95,14 +104,20 @@ func NewAgentAddon(scheme *runtime.Scheme, caCertData, caKeyData []byte, runtime
 			CSRSign: CustomSignerWithExpiry(ProxyAgentSignerName, caKeyData, caCertData, time.Hour*24*180),
 		}).
 		WithInstallStrategy(agent.InstallAllStrategy(common.AddonInstallNamespace)).
-		WithGetValuesFuncs(GetClusterProxyValueFunc(runtimeClient, nativeClient, caCertData)).
+		WithGetValuesFuncs(GetClusterProxyValueFunc(runtimeClient, nativeClient, signerNamespace, caCertData, v1CSRSupported)).
 		BuildHelmAgentAddon()
 
 }
 
-func GetClusterProxyValueFunc(runtimeClient client.Client, nativeClient kubernetes.Interface, caCertData []byte) addonfactory.GetValuesFunc {
+func GetClusterProxyValueFunc(
+	runtimeClient client.Client,
+	nativeClient kubernetes.Interface,
+	signerNamespace string,
+	caCertData []byte,
+	v1CSRSupported bool) addonfactory.GetValuesFunc {
 	return func(cluster *clusterv1.ManagedCluster,
 		addon *addonv1alpha1.ManagedClusterAddOn) (addonfactory.Values, error) {
+
 		// prepping
 		clusterAddon := &addonv1alpha1.ClusterManagementAddOn{}
 		if err := runtimeClient.Get(context.TODO(), types.NamespacedName{
@@ -161,6 +176,20 @@ func GetClusterProxyValueFunc(runtimeClient client.Client, nativeClient kubernet
 			serviceEntryPointPort = 8091
 		}
 
+		// If v1 CSR is not supported in the hub cluster, copy and apply the secret named
+		// "agent-client" to the managed clusters.
+		certDataBase64, keyDataBase64 := "", ""
+		if !v1CSRSupported {
+			agentClientSecret, err := nativeClient.CoreV1().
+				Secrets(signerNamespace).
+				Get(context.TODO(), common.AgentClientSecretName, metav1.GetOptions{})
+			if err != nil {
+				return nil, err
+			}
+			certDataBase64 = base64.StdEncoding.EncodeToString(agentClientSecret.Data[corev1.TLSCertKey])
+			keyDataBase64 = base64.StdEncoding.EncodeToString(agentClientSecret.Data[corev1.TLSPrivateKeyKey])
+		}
+
 		registry, image, tag := config.GetParsedAgentImage()
 		return map[string]interface{}{
 			"agentDeploymentName":      "cluster-proxy-proxy-agent",
@@ -169,17 +198,20 @@ func GetClusterProxyValueFunc(runtimeClient client.Client, nativeClient kubernet
 			"spokeAddonNamespace":      "open-cluster-management-cluster-proxy",
 			"additionalProxyAgentArgs": proxyConfig.Spec.ProxyAgent.AdditionalArgs,
 
-			"clusterName":                cluster.Name,
-			"registry":                   registry,
-			"image":                      image,
-			"tag":                        tag,
-			"proxyAgentImage":            proxyConfig.Spec.ProxyAgent.Image,
-			"replicas":                   proxyConfig.Spec.ProxyAgent.Replicas,
-			"base64EncodedCAData":        base64.StdEncoding.EncodeToString(caCertData),
-			"serviceEntryPoint":          serviceEntryPoint,
-			"serviceEntryPointPort":      serviceEntryPointPort,
-			"agentDeploymentAnnotations": annotations,
-			"addonAgentArgs":             addonAgentArgs,
+			"clusterName":                   cluster.Name,
+			"registry":                      registry,
+			"image":                         image,
+			"tag":                           tag,
+			"proxyAgentImage":               proxyConfig.Spec.ProxyAgent.Image,
+			"replicas":                      proxyConfig.Spec.ProxyAgent.Replicas,
+			"base64EncodedCAData":           base64.StdEncoding.EncodeToString(caCertData),
+			"serviceEntryPoint":             serviceEntryPoint,
+			"serviceEntryPointPort":         serviceEntryPointPort,
+			"agentDeploymentAnnotations":    annotations,
+			"addonAgentArgs":                addonAgentArgs,
+			"includeStaticProxyAgentSecret": !v1CSRSupported,
+			"staticProxyAgentSecretCert":    certDataBase64,
+			"staticProxyAgentSecretKey":     keyDataBase64,
 		}, nil
 	}
 }

pkg/proxyagent/agent/manifests/charts/addon-agent/templates/agent-client-secret.yaml

@@ -0,0 +1,10 @@
+{{ if .Values.includeStaticProxyAgentSecret }}
+apiVersion: v1
+kind: Secret
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: cluster-proxy-open-cluster-management.io-proxy-agent-signer-client-cert
+data:
+  "tls.crt": {{ .Values.staticProxyAgentSecretCert }}
+  "tls.key": {{ .Values.staticProxyAgentSecretKey }}
+{{ end }}

pkg/proxyserver/controllers/clustermanagementaddon_controller.go

@@ -51,7 +51,9 @@ func RegisterClusterManagementAddonReconciler(
 	mgr manager.Manager,
 	selfSigner selfsigned.SelfSigner,
 	nativeClient kubernetes.Interface,
-	secretInformer informercorev1.SecretInformer) error {
+	secretInformer informercorev1.SecretInformer,
+	supportsV1CSR bool,
+) error {
 	r := &ClusterManagementAddonReconciler{
 		Client:     mgr.GetClient(),
 		SelfSigner: selfSigner,
@@ -72,6 +74,7 @@ func RegisterClusterManagementAddonReconciler(
 		ServiceGetter:    nativeClient.CoreV1(),
 		DeploymentGetter: nativeClient.AppsV1(),
 		EventRecorder:    events.NewInMemoryRecorder("ClusterManagementAddonReconciler"),
+		supportsV1CSR:    supportsV1CSR,
 	}
 	return r.SetupWithManager(mgr)
 }
@@ -87,6 +90,7 @@ type ClusterManagementAddonReconciler struct {
 	EventRecorder    events.Recorder
 
 	newCertRotatorFunc func(namespace, name string, sans ...string) selfsigned.CertRotation
+	supportsV1CSR      bool
 }
 
 func (c *ClusterManagementAddonReconciler) SetupWithManager(mgr ctrl.Manager) error {
@@ -452,6 +456,20 @@ func (c *ClusterManagementAddonReconciler) ensureRotation(config *proxyv1alpha1.
 		return errors.Wrapf(err, "fails to rotate proxy client cert")
 	}
 
+	if !c.supportsV1CSR {
+		// agent client cert rotation
+		// beta CSR running under Kubernetes release lower than 1.18 doesn't
+		// support custom CSR signer. so we need to sign the secret in the hub
+		// then apply it to the managed cluster via ManifestWork.
+		proxyClientRotator := c.newCertRotatorFunc(
+			config.Spec.ProxyServer.Namespace,
+			common.AgentClientSecretName,
+			sans...)
+		if err := proxyClientRotator.EnsureTargetCertKeyPair(c.CAPair, c.CAPair.Config.Certs, tweakClientCertUsageFunc); err != nil {
+			return errors.Wrapf(err, "fails to rotate proxy client cert")
+		}
+	}
+
 	return nil
 }
 

pkg/proxyserver/controllers/suite_test.go

@@ -112,8 +112,7 @@ var _ = BeforeSuite(func() {
 	selfSigner, err := selfsigned.NewSelfSignerFromSecretOrGenerate(kubeClient, "default", "test-ca")
 	Expect(err).NotTo(HaveOccurred())
 
-	err = RegisterClusterManagementAddonReconciler(
-		mgr, selfSigner, kubeClient, kubeInformer.Core().V1().Secrets())
+	err = RegisterClusterManagementAddonReconciler(mgr, selfSigner, kubeClient, kubeInformer.Core().V1().Secrets(), true)
 	Expect(err).NotTo(HaveOccurred())
 
 	By("start manager")