diff --git a/artifacts/deploy/bootstrap-token-configuration.yaml b/artifacts/deploy/bootstrap-token-configuration.yaml index 1904eaec996e..9273e7da0762 100644 --- a/artifacts/deploy/bootstrap-token-configuration.yaml +++ b/artifacts/deploy/bootstrap-token-configuration.yaml @@ -13,6 +13,7 @@ data: kind: Config --- +# Define a role with permission to get the cluster-info configmap apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -31,6 +32,8 @@ rules: - get --- +# An anonymous user can get `cluster-info` configmap, which is used to obtain the control plane API server's server +# address and `certificate-authority-data` during the `karmadactl register` process. apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -48,6 +51,8 @@ subjects: name: system:anonymous --- +# Group `system:bootstrappers:karmada:default-cluster-token` is the user group of the bootstrap token +# used by `karmadactl register` when registering a new pull mode cluster. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -64,6 +69,26 @@ subjects: name: system:bootstrappers:karmada:default-cluster-token --- +# Define a ClusterRole with permissions to automatically approve the agent CSRs when the agentcsrapproving controller is enabled by karmada-controller-manager. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + karmada.io/bootstrapping: rbac-defaults + name: system:karmada:certificatesigningrequest:autoapprover +rules: + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/clusteragent + verbs: + - create + +--- +# Group `system:bootstrappers:karmada:default-cluster-token` is the user group of the bootstrap token +# used by `karmadactl register` when registering a new pull mode cluster. +# When the `agentcsrapproving` controller is enabled by the karmada-controller-manager, +# it can automatically approve the agent CSRs requested by the user group system:bootstrappers:karmada:default-cluster-token. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -73,13 +98,33 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:nodeclient + name: system:karmada:certificatesigningrequest:autoapprover subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:karmada:default-cluster-token --- +# Define a ClusterRole with permissions to automatically approve the agent CSRs +# where the user name and group of requester match those in the CSRs when the agentcsrapproving controller is enabled by karmada-controller-manager. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + karmada.io/bootstrapping: rbac-defaults + name: system:karmada:certificatesigningrequest:selfautoapprover +rules: + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/selfclusteragent + verbs: + - create + +--- +# Group `system:karmada:agents` is the user group used by the karmada-agent to access the Karmada API server. +# When the agentcsrapproving controller is enabled by the karmada-controller-manager, it can automatically approve +# the agent CSRs(csr.Subject.CommonName = agent username) requested by the user group system:karmada:agents. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -89,280 +134,39 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + name: system:karmada:certificatesigningrequest:selfautoapprover subjects: - apiGroup: rbac.authorization.k8s.io kind: Group - name: system:nodes + name: system:karmada:agents --- +# ClusterRole `system:karmada:agent-rbac-generator` is not used for the connection between the karmada-agent and the control plane, +# but is used by karmadactl register to generate the RBAC resources required by the karmada-agent. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: karmada.io/bootstrapping: rbac-defaults - name: system:karmada:agent + name: system:karmada:agent-rbac-generator rules: -- apiGroups: - - cluster.karmada.io - resources: - - clusters - verbs: - - create - - get - - list - - watch - - delete -- apiGroups: - - cluster.karmada.io - resources: - - clusters/status - verbs: - - update -- apiGroups: - - work.karmada.io - resources: - - works - verbs: - - create - - get - - list - - watch - - update - - delete -- apiGroups: - - work.karmada.io - resources: - - works/status - verbs: - - patch - - update -- apiGroups: - - config.karmada.io - resources: - - resourceinterpreterwebhookconfigurations - - resourceinterpretercustomizations - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create - - patch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - verbs: - - create - - get -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update + - apiGroups: ['*'] + resources: ['*'] + verbs: ['*'] --- +# User `system:karmada:agent:rbac-generator` is specifically used during the `karmadactl register` process to generate restricted RBAC resources for the `karmada-agent`. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: karmada.io/bootstrapping: rbac-defaults - name: system:karmada:agent + name: system:karmada:agent-rbac-generator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:karmada:agent + name: system:karmada:agent-rbac-generator subjects: -- apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes - -# To ensure the agent has the minimal RBAC permissions, the ideal approach is to -# use different RBAC configurations for different agents of member clusters with pull mode. -# Below is the minimal set of RBAC permissions required for a single pull mode member cluster. -# Here are the definitions of the variables used: -# -# - clustername: the name of the member cluster. -# - cluster_namespace: the namespace where the member cluster secrets are stored, default to karmada-cluster. -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRole -# metadata: -# name: system:karmada:agent -# rules: -# - apiGroups: -# - cluster.karmada.io -# resources: -# - clusters -# resourceNames: -# - {{clustername}} -# verbs: -# - create -# - get -# - delete -# - apiGroups: -# - cluster.karmada.io -# resources: -# - clusters -# verbs: -# - list -# - watch -# - apiGroups: -# - cluster.karmada.io -# resources: -# - clusters/status -# resourceNames: -# - {{clustername}} -# verbs: -# - update -# - apiGroups: -# - config.karmada.io -# resources: -# - resourceinterpreterwebhookconfigurations -# - resourceinterpretercustomizations -# verbs: -# - get -# - list -# - watch -# - apiGroups: -# - "" -# resources: -# - namespaces -# verbs: -# - get -# - apiGroups: -# - coordination.k8s.io -# resources: -# - leases -# verbs: -# - create -# - get -# - update -# - apiGroups: -# - certificates.k8s.io -# resources: -# - certificatesigningrequests -# verbs: -# - create -# - get -# - apiGroups: -# - "" -# resources: -# - events -# verbs: -# - create -# - patch -# - update -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRoleBinding -# metadata: -# name: system:karmada:agent -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: ClusterRole -# name: system:karmada:agent -# subjects: -# - apiGroup: rbac.authorization.k8s.io -# kind: Group -# name: system:nodes -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: Role -# metadata: -# name: system:karmada:agent-secret -# namespace: "{{cluster_namespace}}" -# rules: -# - apiGroups: -# - "" -# resources: -# - secrets -# resourceNames: -# - {{clustername}}-impersonator -# - {{clustername}} -# verbs: -# - get -# - create -# - patch -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: RoleBinding -# metadata: -# name: system:karmada:agent-secret -# namespace: "{{cluster_namespace}}" -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: Role -# name: system:karmada:agent-secret -# subjects: -# - apiGroup: rbac.authorization.k8s.io -# kind: Group -# name: system:nodes -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: Role -# metadata: -# name: system:karmada:agent-work -# namespace: "karmada-es-{{clustername}}" -# rules: -# - apiGroups: -# - work.karmada.io -# resources: -# - works -# verbs: -# - create -# - get -# - list -# - watch -# - update -# - delete -# - apiGroups: -# - work.karmada.io -# resources: -# - works/status -# verbs: -# - patch -# - update -# -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: RoleBinding -# metadata: -# name: system:karmada:agent-work -# namespace: "karmada-es-{{clustername}}" -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: Role -# name: system:karmada:agent-work -# subjects: -# - apiGroup: rbac.authorization.k8s.io -# kind: Group -# name: system:nodes + - apiGroup: rbac.authorization.k8s.io + kind: User + name: system:karmada:agent:rbac-generator diff --git a/charts/karmada/templates/_karmada_bootstrap_token_configuration.tpl b/charts/karmada/templates/_karmada_bootstrap_token_configuration.tpl index f9852290721d..776bc8ca0bf4 100644 --- a/charts/karmada/templates/_karmada_bootstrap_token_configuration.tpl +++ b/charts/karmada/templates/_karmada_bootstrap_token_configuration.tpl @@ -75,6 +75,22 @@ subjects: name: system:bootstrappers:karmada:default-cluster-token --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:karmada:certificatesigningrequest:autoapprover + {{- if "karmada.commonLabels" }} + labels: + {{- include "karmada.commonLabels" . | nindent 4 }} + {{- end }} +rules: +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/clusteragent + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:karmada:agent-autoapprove-bootstrap @@ -85,13 +101,29 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:nodeclient + name: system:karmada:certificatesigningrequest:autoapprover subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:bootstrappers:karmada:default-cluster-token --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:karmada:certificatesigningrequest:selfautoapprover + {{- if "karmada.commonLabels" }} + labels: + {{- include "karmada.commonLabels" . | nindent 4 }} + {{- end }} +rules: +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/selfclusteragent + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:karmada:agent-autoapprove-certificate-rotation @@ -102,124 +134,32 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + name: system:karmada:certificatesigningrequest:selfautoapprover subjects: - apiGroup: rbac.authorization.k8s.io kind: Group - name: system:nodes + name: system:karmada:agents --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: system:karmada:agent + name: system:karmada:agent-rbac-generator {{- if "karmada.commonLabels" }} labels: {{- include "karmada.commonLabels" . | nindent 4 }} {{- end }} rules: - apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - cluster.karmada.io - resources: - - clusters - verbs: - - create - - get - - list - - watch - - patch - - update - - delete -- apiGroups: - - cluster.karmada.io - resources: - - clusters/status - verbs: - - patch - - update -- apiGroups: - - work.karmada.io + - "*" resources: - - works + - "*" verbs: - - create - - get - - list - - watch - - update - - delete -- apiGroups: - - work.karmada.io - resources: - - works/status - verbs: - - patch - - update -- apiGroups: - - config.karmada.io - resources: - - resourceinterpreterwebhookconfigurations - - resourceinterpretercustomizations - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch - - create -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - create - - patch -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - delete - - get - - patch - - update -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - verbs: - - create - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update + - "*" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: system:karmada:agent + name: system:karmada:agent-rbac-generator {{- if "karmada.commonLabels" }} labels: {{- include "karmada.commonLabels" . | nindent 4 }} @@ -227,9 +167,9 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:karmada:agent + name: system:karmada:agent-rbac-generator subjects: - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes + kind: User + name: system:karmada:agent:rbac-generator {{- end -}} diff --git a/pkg/karmadactl/cmdinit/bootstraptoken/agent/tlsbootstrap.go b/pkg/karmadactl/cmdinit/bootstraptoken/agent/tlsbootstrap.go index aaa81e14d268..11f1a8a89dcf 100644 --- a/pkg/karmadactl/cmdinit/bootstraptoken/agent/tlsbootstrap.go +++ b/pkg/karmadactl/cmdinit/bootstraptoken/agent/tlsbootstrap.go @@ -31,15 +31,15 @@ const ( // KarmadaAgentBootstrap defines the name of the ClusterRoleBinding that lets Karmada Agent post CSRs KarmadaAgentBootstrap = "system:karmada:agent-bootstrap" // KarmadaAgentGroup defines the group of Karmada Agent - KarmadaAgentGroup = "system:nodes" + KarmadaAgentGroup = "system:karmada:agents" // KarmadaAgentAutoApproveBootstrapClusterRoleBinding defines the name of the ClusterRoleBinding that makes the csrapprover approve agent CSRs KarmadaAgentAutoApproveBootstrapClusterRoleBinding = "system:karmada:agent-autoapprove-bootstrap" // KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding defines name of the ClusterRoleBinding that makes the csrapprover approve agent auto rotated CSRs KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding = "system:karmada:agent-autoapprove-certificate-rotation" // CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR - CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" + CSRAutoApprovalClusterRoleName = "system:karmada:certificatesigningrequest:autoapprover" // KarmadaAgentSelfCSRAutoApprovalClusterRoleName is a role for automatic CSR approvals for automatically rotated agent certificates - KarmadaAgentSelfCSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:selfnodeclient" + KarmadaAgentSelfCSRAutoApprovalClusterRoleName = "system:karmada:certificatesigningrequest:selfautoapprover" // KarmadaAgentBootstrapTokenAuthGroup specifies which group a Karmada Agent Bootstrap Token should be authenticated in KarmadaAgentBootstrapTokenAuthGroup = "system:bootstrappers:karmada:default-cluster-token" ) @@ -60,7 +60,18 @@ func AllowBootstrapTokensToPostCSRs(clientSet kubernetes.Interface) error { // AutoApproveKarmadaAgentBootstrapTokens creates RBAC rules in a way that makes Karmada Agent Bootstrap Tokens' CSR auto-approved by the csrapprover controller func AutoApproveKarmadaAgentBootstrapTokens(clientSet kubernetes.Interface) error { - klog.Infoln("[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Karmada Agent Bootstrap Token") + klog.Infoln("[bootstrap-token] configured RBAC rules to allow the agentcsrapproving controller automatically approve CSRs from a Karmada Agent Bootstrap Token") + csrAutoApprovalClusterRole := utils.ClusterRoleFromRules(CSRAutoApprovalClusterRoleName, []rbacv1.PolicyRule{ + { + APIGroups: []string{"certificates.k8s.io"}, + Resources: []string{"certificatesigningrequests/clusteragent"}, + Verbs: []string{"create"}, + }, + }, nil, nil) + err := cmdutil.CreateOrUpdateClusterRole(clientSet, csrAutoApprovalClusterRole) + if err != nil { + return err + } clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(KarmadaAgentAutoApproveBootstrapClusterRoleBinding, CSRAutoApprovalClusterRoleName, []rbacv1.Subject{ @@ -75,6 +86,17 @@ func AutoApproveKarmadaAgentBootstrapTokens(clientSet kubernetes.Interface) erro // AutoApproveAgentCertificateRotation creates RBAC rules in a way that makes Agent certificate rotation CSR auto-approved by the csrapprover controller func AutoApproveAgentCertificateRotation(clientSet kubernetes.Interface) error { klog.Infoln("[bootstrap-token] configured RBAC rules to allow certificate rotation for all agent client certificates in the member cluster") + karmadaAgentSelfCSRAutoApprovalClusterRole := utils.ClusterRoleFromRules(KarmadaAgentSelfCSRAutoApprovalClusterRoleName, []rbacv1.PolicyRule{ + { + APIGroups: []string{"certificates.k8s.io"}, + Resources: []string{"certificatesigningrequests/selfclusteragent"}, + Verbs: []string{"create"}, + }, + }, nil, nil) + err := cmdutil.CreateOrUpdateClusterRole(clientSet, karmadaAgentSelfCSRAutoApprovalClusterRole) + if err != nil { + return err + } clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding, KarmadaAgentSelfCSRAutoApprovalClusterRoleName, []rbacv1.Subject{ diff --git a/pkg/karmadactl/cmdinit/karmada/deploy.go b/pkg/karmadactl/cmdinit/karmada/deploy.go index 7af7791d3601..899adfead9cf 100644 --- a/pkg/karmadactl/cmdinit/karmada/deploy.go +++ b/pkg/karmadactl/cmdinit/karmada/deploy.go @@ -185,8 +185,8 @@ func createExtraResources(clientSet *kubernetes.Clientset, dir string) error { return fmt.Errorf("error creating clusterinfo RBAC rules: %v", err) } - // grant limited access permission to 'karmada-agent' - if err := grantAccessPermissionToAgent(clientSet); err != nil { + // grant access permission to 'karmada-agent-rbac-generator' + if err := grantAccessPermissionToAgentRBACGenerator(clientSet); err != nil { return err } diff --git a/pkg/karmadactl/cmdinit/karmada/rbac.go b/pkg/karmadactl/cmdinit/karmada/rbac.go index 7ee4281fee83..abf8e5b61f78 100644 --- a/pkg/karmadactl/cmdinit/karmada/rbac.go +++ b/pkg/karmadactl/cmdinit/karmada/rbac.go @@ -26,10 +26,11 @@ import ( ) const ( - karmadaViewClusterRole = "karmada-view" - karmadaEditClusterRole = "karmada-edit" - karmadaAgentAccessClusterRole = "system:karmada:agent" - karmadaAgentGroup = "system:nodes" + karmadaViewClusterRole = "karmada-view" + karmadaEditClusterRole = "karmada-edit" + karmadaAgentRBACGeneratorClusterRole = "system:karmada:agent-rbac-generator" + karmadaAgentRBACGeneratorClusterRoleBinding = "system:karmada:agent-rbac-generator" + agentRBACGenerator = "system:karmada:agent:rbac-generator" ) // grantProxyPermissionToAdmin grants the proxy permission to "system:admin" @@ -62,63 +63,13 @@ func grantProxyPermissionToAdmin(clientSet kubernetes.Interface) error { return nil } -// grantAccessPermissionToAgent grants the limited access permission to 'karmada-agent' -func grantAccessPermissionToAgent(clientSet kubernetes.Interface) error { - clusterRole := utils.ClusterRoleFromRules(karmadaAgentAccessClusterRole, []rbacv1.PolicyRule{ +// grantAccessPermissionToAgentRBACGenerator grants the access permission to 'karmada-agent-rbac-generator' +func grantAccessPermissionToAgentRBACGenerator(clientSet kubernetes.Interface) error { + clusterRole := utils.ClusterRoleFromRules(karmadaAgentRBACGeneratorClusterRole, []rbacv1.PolicyRule{ { - APIGroups: []string{"authentication.k8s.io"}, - Resources: []string{"tokenreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"cluster.karmada.io"}, - Resources: []string{"clusters"}, - Verbs: []string{"create", "get", "list", "watch", "patch", "update", "delete"}, - }, - { - APIGroups: []string{"cluster.karmada.io"}, - Resources: []string{"clusters/status"}, - Verbs: []string{"patch", "update"}, - }, - { - APIGroups: []string{"work.karmada.io"}, - Resources: []string{"works"}, - Verbs: []string{"create", "get", "list", "watch", "update", "delete"}, - }, - { - APIGroups: []string{"work.karmada.io"}, - Resources: []string{"works/status"}, - Verbs: []string{"patch", "update"}, - }, - { - APIGroups: []string{"config.karmada.io"}, - Resources: []string{"resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list", "watch", "create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch", "create", "patch"}, - }, - { - APIGroups: []string{"coordination.k8s.io"}, - Resources: []string{"leases"}, - Verbs: []string{"create", "delete", "get", "patch", "update"}, - }, - { - APIGroups: []string{"certificates.k8s.io"}, - Resources: []string{"certificatesigningrequests"}, - Verbs: []string{"create", "get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch", "update"}, + APIGroups: []string{"*"}, + Resources: []string{"*"}, + Verbs: []string{"*"}, }, }, nil, nil) err := cmdutil.CreateOrUpdateClusterRole(clientSet, clusterRole) @@ -126,14 +77,14 @@ func grantAccessPermissionToAgent(clientSet kubernetes.Interface) error { return err } - clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(karmadaAgentAccessClusterRole, karmadaAgentAccessClusterRole, + clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(karmadaAgentRBACGeneratorClusterRoleBinding, karmadaAgentRBACGeneratorClusterRole, []rbacv1.Subject{ { - Kind: rbacv1.GroupKind, - Name: karmadaAgentGroup, + Kind: rbacv1.UserKind, + Name: agentRBACGenerator, }}, nil) - klog.V(1).Info("Grant the limited access permission to 'karmada-agent'") + klog.V(1).Info("Grant the access permission to 'karmada-agent-rbac-generator'") err = cmdutil.CreateOrUpdateClusterRoleBinding(clientSet, clusterRoleBinding) if err != nil { return err diff --git a/pkg/karmadactl/cmdinit/karmada/rbac_test.go b/pkg/karmadactl/cmdinit/karmada/rbac_test.go index 501adae9325d..7767cb6aa384 100644 --- a/pkg/karmadactl/cmdinit/karmada/rbac_test.go +++ b/pkg/karmadactl/cmdinit/karmada/rbac_test.go @@ -31,8 +31,8 @@ func Test_grantProxyPermissionToAdmin(t *testing.T) { func Test_grantAccessPermissionToAgent(t *testing.T) { client := fake.NewSimpleClientset() - if err := grantAccessPermissionToAgent(client); err != nil { - t.Errorf("grantAccessPermissionToAgent() expected no error, but got err: %v", err) + if err := grantAccessPermissionToAgentRBACGenerator(client); err != nil { + t.Errorf("grantAccessPermissionToAgentRBACGenerator() expected no error, but got err: %v", err) } } diff --git a/pkg/karmadactl/register/register.go b/pkg/karmadactl/register/register.go index c07db7cc78f7..738a58c845ff 100644 --- a/pkg/karmadactl/register/register.go +++ b/pkg/karmadactl/register/register.go @@ -51,7 +51,6 @@ import ( "github.com/karmada-io/karmada/pkg/apis/cluster/validation" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" - addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils" "github.com/karmada-io/karmada/pkg/karmadactl/options" cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient" @@ -82,9 +81,11 @@ const ( // CACertPath defines default location of CA certificate on Linux CACertPath = "/etc/karmada/pki/ca.crt" // ClusterPermissionPrefix defines the common name of karmada agent certificate - ClusterPermissionPrefix = "system:node:" + ClusterPermissionPrefix = "system:karmada:agent:" // ClusterPermissionGroups defines the organization of karmada agent certificate - ClusterPermissionGroups = "system:nodes" + ClusterPermissionGroups = "system:karmada:agents" + // AgentRBACGenerator defines the common name of karmada agent rbac generator certificate + AgentRBACGenerator = "system:karmada:agent:rbac-generator" // KarmadaAgentBootstrapKubeConfigFileName defines the file name for the kubeconfig that the karmada-agent will use to do // the TLS bootstrap to get itself an unique credential KarmadaAgentBootstrapKubeConfigFileName = "bootstrap-karmada-agent.conf" @@ -97,8 +98,8 @@ const ( KarmadaAgentName = "karmada-agent" // KarmadaAgentServiceAccountName is the name of karmada-agent serviceaccount KarmadaAgentServiceAccountName = "karmada-agent-sa" - // SignerName defines the signer name for csr, 'kubernetes.io/kube-apiserver-client-kubelet' can sign the csr automatically - SignerName = "kubernetes.io/kube-apiserver-client-kubelet" + // SignerName defines the signer name for csr, 'kubernetes.io/kube-apiserver-client' can sign the csr with `O=system:agents,CN=system:agent:` automatically if agentcsrapproving controller if enabled. + SignerName = "kubernetes.io/kube-apiserver-client" // BootstrapUserName defines bootstrap user name BootstrapUserName = "token-bootstrap-client" // DefaultClusterName defines the default cluster name @@ -260,6 +261,9 @@ type CommandRegisterOption struct { memberClusterEndpoint string memberClusterClient *kubeclient.Clientset + + // rbacResources contains RBAC resources that grant the necessary permissions for pull mode cluster to access to Karmada control plane. + rbacResources *RBACResources } // Complete ensures that options are valid and marshals them if necessary. @@ -288,6 +292,8 @@ func (o *CommandRegisterOption) Complete(args []string) error { o.ClusterName = config.Contexts[config.CurrentContext].Cluster } + o.rbacResources = GenerateRBACResources(o.ClusterName, o.ClusterNamespace) + o.memberClusterEndpoint = restConfig.Host o.memberClusterClient, err = apiclient.NewClientSet(restConfig) @@ -358,23 +364,79 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { return err } - // construct the final kubeconfig file used by karmada agent to connect to karmada apiserver - fmt.Println("[karmada-agent-start] Waiting to construct karmada-agent kubeconfig") - karmadaAgentCfg, err := o.constructKarmadaAgentConfig(bootstrapClient, karmadaClusterInfo) + var rbacClient *kubeclient.Clientset + defer func() { + if err != nil && rbacClient != nil { + fmt.Println("karmadactl register failed and started deleting the created resources") + err = o.rbacResources.Delete(rbacClient) + if err != nil { + klog.Warningf("Failed to delete rbac resources: %v", err) + } + } + }() + + rbacClient, err = o.EnsureNecessaryResourcesExistInControlPlane(bootstrapClient, karmadaClusterInfo) if err != nil { return err } - fmt.Println("[karmada-agent-start] Waiting to check cluster exists") - karmadaClient, err := ToKarmadaClient(karmadaAgentCfg) + err = o.EnsureNecessaryResourcesExistInMemberCluster(bootstrapClient, karmadaClusterInfo) if err != nil { return err } + + fmt.Printf("\ncluster(%s) is joined successfully\n", o.ClusterName) + + return nil +} + +// EnsureNecessaryResourcesExistInControlPlane ensures that all necessary resources are exist in Karmada control plane. +func (o *CommandRegisterOption) EnsureNecessaryResourcesExistInControlPlane(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*kubeclient.Clientset, error) { + csrName := "agent-rbac-generator-" + o.ClusterName + k8srand.String(5) + rbacCfg, err := o.constructAgentRBACGeneratorConfig(bootstrapClient, karmadaClusterInfo, csrName) + if err != nil { + return nil, err + } + + kubelient, err := ToClientSet(rbacCfg) + if err != nil { + return nil, err + } + defer func() { + err = kubelient.CertificatesV1().CertificateSigningRequests().Delete(context.Background(), csrName, metav1.DeleteOptions{}) + if err != nil { + klog.Warningf("Failed to delete CertificateSigningRequests %s: %v", csrName, err) + } + }() + + fmt.Println("[karmada-agent-start] Waiting to check cluster exists") + karmadaClient, err := ToKarmadaClient(rbacCfg) + if err != nil { + return kubelient, err + } _, exist, err := karmadautil.GetClusterWithKarmadaClient(karmadaClient, o.ClusterName) if err != nil { - return err + return kubelient, err } else if exist { - return fmt.Errorf("failed to register as cluster with name %s already exists", o.ClusterName) + return kubelient, fmt.Errorf("failed to register as cluster with name %s already exists", o.ClusterName) + } + + fmt.Println("[karmada-agent-start] Assign the necessary RBAC permissions to the agent") + err = o.ensureAgentRBACResourcesExistInControlPlane(kubelient) + if err != nil { + return kubelient, err + } + + return kubelient, nil +} + +// EnsureNecessaryResourcesExistInMemberCluster ensures that all necessary resources are exist in the registering cluster. +func (o *CommandRegisterOption) EnsureNecessaryResourcesExistInMemberCluster(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) error { + // construct the final kubeconfig file used by karmada agent to connect to karmada apiserver + fmt.Println("[karmada-agent-start] Waiting to construct karmada-agent kubeconfig") + karmadaAgentCfg, err := o.constructKarmadaAgentConfig(bootstrapClient, karmadaClusterInfo) + if err != nil { + return err } // It's necessary to set the label of namespace to make sure that the namespace is created by Karmada. @@ -382,29 +444,27 @@ func (o *CommandRegisterOption) Run(parentCommand string) error { karmadautil.KarmadaSystemLabel: karmadautil.KarmadaSystemLabelValue, } // ensure namespace where the karmada-agent resources be deployed exists in the member cluster - if _, err := karmadautil.EnsureNamespaceExistWithLabels(o.memberClusterClient, o.Namespace, o.DryRun, labels); err != nil { + if _, err = karmadautil.EnsureNamespaceExistWithLabels(o.memberClusterClient, o.Namespace, o.DryRun, labels); err != nil { return err } // create the necessary secret and RBAC in the member cluster fmt.Println("[karmada-agent-start] Waiting the necessary secret and RBAC") - if err := o.createSecretAndRBACInMemberCluster(karmadaAgentCfg); err != nil { + if err = o.createSecretAndRBACInMemberCluster(karmadaAgentCfg); err != nil { return err } // create karmada-agent Deployment in the member cluster fmt.Println("[karmada-agent-start] Waiting karmada-agent Deployment") KarmadaAgentDeployment := o.makeKarmadaAgentDeployment() - if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil { + if _, err = o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil { return err } - if err := addonutils.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { + if err = cmdutil.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil { return err } - fmt.Printf("\ncluster(%s) is joined successfully\n", o.ClusterName) - return nil } @@ -505,11 +565,316 @@ func (o *CommandRegisterOption) discoveryBootstrapConfigAndClusterInfo(bootstrap return bootstrapClient, clusterinfo, nil } -// constructKarmadaAgentConfig construct the final kubeconfig used by karmada-agent -func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*clientcmdapi.Config, error) { +// ensureAgentRBACResourcesExistInControlPlane ensures that necessary RBAC resources for karmada-agent are exist in control plane. +func (o *CommandRegisterOption) ensureAgentRBACResourcesExistInControlPlane(client kubeclient.Interface) error { + for i := range o.rbacResources.ClusterRoles { + _, err := karmadautil.CreateClusterRole(client, o.rbacResources.ClusterRoles[i]) + if err != nil { + return err + } + } + for i := range o.rbacResources.ClusterRoleBindings { + _, err := karmadautil.CreateClusterRoleBinding(client, o.rbacResources.ClusterRoleBindings[i]) + if err != nil { + return err + } + } + + for i := range o.rbacResources.Roles { + roleNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: o.rbacResources.Roles[i].GetNamespace(), + Labels: map[string]string{ + karmadautil.KarmadaSystemLabel: karmadautil.KarmadaSystemLabelValue, + }, + }, + } + _, err := karmadautil.CreateNamespace(client, roleNamespace) + if err != nil { + return err + } + _, err = karmadautil.CreateRole(client, o.rbacResources.Roles[i]) + if err != nil { + return err + } + } + + for i := range o.rbacResources.RoleBindings { + _, err := karmadautil.CreateRoleBinding(client, o.rbacResources.RoleBindings[i]) + if err != nil { + return err + } + } + + return nil +} + +// RBACResources defines the list of rbac resources. +type RBACResources struct { + ClusterRoles []*rbacv1.ClusterRole + ClusterRoleBindings []*rbacv1.ClusterRoleBinding + Roles []*rbacv1.Role + RoleBindings []*rbacv1.RoleBinding +} + +// GenerateRBACResources generates rbac resources. +func GenerateRBACResources(clusterName, clusterNamespace string) *RBACResources { + return &RBACResources{ + ClusterRoles: []*rbacv1.ClusterRole{GenerateClusterRole(clusterName)}, + ClusterRoleBindings: []*rbacv1.ClusterRoleBinding{GenerateClusterRoleBinding(clusterName)}, + Roles: []*rbacv1.Role{GenerateSecretAccessRole(clusterName, clusterNamespace), GenerateWorkAccessRole(clusterName)}, + RoleBindings: []*rbacv1.RoleBinding{GenerateSecretAccessRoleBinding(clusterName, clusterNamespace), GenerateWorkAccessRoleBinding(clusterName)}, + } +} + +// List return the list of rbac resources. +func (r *RBACResources) List() []Obj { + var obj []Obj + for i := range r.ClusterRoles { + obj = append(obj, Obj{Kind: "ClusterRole", Name: r.ClusterRoles[i].GetName()}) + } + for i := range r.ClusterRoleBindings { + obj = append(obj, Obj{Kind: "ClusterRoleBinding", Name: r.ClusterRoleBindings[i].GetName()}) + } + for i := range r.Roles { + obj = append(obj, Obj{Kind: "Role", Name: r.Roles[i].GetName(), Namespace: r.Roles[i].GetNamespace()}) + } + for i := range r.RoleBindings { + obj = append(obj, Obj{Kind: "RoleBinding", Name: r.RoleBindings[i].GetName(), Namespace: r.RoleBindings[i].GetNamespace()}) + } + return obj +} + +// ToString returns a list of RBAC resources in string format. +func (r *RBACResources) ToString() string { + var resources []string + for i := range r.List() { + resources = append(resources, r.List()[i].ToString()) + } + return strings.Join(resources, "\n") +} + +// Delete deletes RBAC resources. +func (r *RBACResources) Delete(client kubeclient.Interface) error { + var err error + for _, resource := range r.List() { + switch resource.Kind { + case "ClusterRole": + err = karmadautil.DeleteClusterRole(client, resource.Name) + case "ClusterRoleBinding": + err = karmadautil.DeleteClusterRoleBinding(client, resource.Name) + case "Role": + err = karmadautil.DeleteRole(client, resource.Namespace, resource.Name) + case "RoleBinding": + err = karmadautil.DeleteRoleBinding(client, resource.Namespace, resource.Name) + } + if err != nil { + return err + } + } + return nil +} + +// Obj defines the struct which contains the information of kind, name and namespace. +type Obj struct{ Kind, Name, Namespace string } + +// ToString returns a string that concatenates kind, name, and namespace using "/". +func (o *Obj) ToString() string { + if o.Namespace == "" { + return fmt.Sprintf("%s/%s", o.Kind, o.Name) + } + return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name) +} + +// GenerateClusterRole generates the clusterRole that karmada-agent needed. +func GenerateClusterRole(clusterName string) *rbacv1.ClusterRole { + clusterRoleName := fmt.Sprintf("system:karmada:%s:agent", clusterName) + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "ClusterRole", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterRoleName, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters"}, + ResourceNames: []string{clusterName}, + Verbs: []string{"get", "delete"}, + }, + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters"}, + Verbs: []string{"create", "list", "watch"}, + }, + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters/status"}, + ResourceNames: []string{clusterName}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"config.karmada.io"}, + Resources: []string{"resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"get", "create", "update"}, + }, + { + APIGroups: []string{"certificates.k8s.io"}, + Resources: []string{"certificatesigningrequests"}, + Verbs: []string{"get", "create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"patch", "create", "update"}, + }, + }, + } +} + +// GenerateClusterRoleBinding generates the clusterRoleBinding that karmada-agent needed. +func GenerateClusterRoleBinding(clusterName string) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("system:karmada:%s:agent", clusterName)}, + Subjects: []rbacv1.Subject{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "User", + Name: generateAgentUserName(clusterName), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: fmt.Sprintf("system:karmada:%s:agent", clusterName), + }, + } +} + +// GenerateSecretAccessRole generates the secret-related Role that karmada-agent needed. +func GenerateSecretAccessRole(clusterName, clusterNamespace string) *rbacv1.Role { + secretAccessRoleName := fmt.Sprintf("system:karmada:%s:agent-secret", clusterName) + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretAccessRoleName, + Namespace: clusterNamespace, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "patch"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + ResourceNames: []string{clusterName, clusterName + "-impersonator"}, + }, + { + Verbs: []string{"create"}, + APIGroups: []string{""}, + Resources: []string{"secrets"}, + }, + }, + } +} + +// GenerateSecretAccessRoleBinding generates the secret-related RoleBinding that karmada-agent needed. +func GenerateSecretAccessRoleBinding(clusterName, clusterNamespace string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("system:karmada:%s:agent-secret", clusterName), + Namespace: clusterNamespace, + }, + Subjects: []rbacv1.Subject{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "User", + Name: generateAgentUserName(clusterName), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: fmt.Sprintf("system:karmada:%s:agent-secret", clusterName), + }, + } +} + +// GenerateWorkAccessRole generates the work-related Role that karmada-agent needed. +func GenerateWorkAccessRole(clusterName string) *rbacv1.Role { + workAccessRoleName := fmt.Sprintf("system:karmada:%s:agent-work", clusterName) + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: workAccessRoleName, + Namespace: "karmada-es-" + clusterName, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "list", "watch", "update", "delete"}, + APIGroups: []string{"work.karmada.io"}, + Resources: []string{"works"}, + }, + { + Verbs: []string{"patch", "update"}, + APIGroups: []string{"work.karmada.io"}, + Resources: []string{"works/status"}, + }, + }, + } +} + +// GenerateWorkAccessRoleBinding generates the work-related RoleBinding that karmada-agent needed. +func GenerateWorkAccessRoleBinding(clusterName string) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("system:karmada:%s:agent-work", clusterName), + Namespace: "karmada-es-" + clusterName, + }, + Subjects: []rbacv1.Subject{ + { + APIGroup: "rbac.authorization.k8s.io", + Kind: "User", + Name: generateAgentUserName(clusterName), + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: fmt.Sprintf("system:karmada:%s:agent-work", clusterName), + }, + } +} + +func (o *CommandRegisterOption) constructKubeConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster, csrName, commonName string, organization []string) (*clientcmdapi.Config, error) { var cert []byte - pk, csr, err := generateKeyAndCSR(o.ClusterName) + pk, csr, err := generateKeyAndCSR(commonName, organization) if err != nil { return nil, err } @@ -519,8 +884,6 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub return nil, err } - csrName := o.ClusterName + "-" + k8srand.String(5) - certificateSigningRequest := &certificatesv1.CertificateSigningRequest{ ObjectMeta: metav1.ObjectMeta{ Name: csrName, @@ -547,35 +910,45 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub if err != nil { return nil, err } - - klog.V(1).Infof("Waiting for the client certificate to be issued") + klog.V(1).Infof(fmt.Sprintf("Waiting for the client certificate %s to be issued", csrName)) err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, o.Timeout, false, func(context.Context) (done bool, err error) { csrOK, err := bootstrapClient.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), csrName, metav1.GetOptions{}) if err != nil { - return false, fmt.Errorf("failed to get the cluster csr %s. err: %v", o.ClusterName, err) + return false, fmt.Errorf("failed to get the cluster csr %s. err: %v", csrName, err) } if csrOK.Status.Certificate != nil { - klog.V(1).Infof("Signing certificate successfully") + klog.V(1).Infof(fmt.Sprintf("Signing certificate of csr %s successfully", csrName)) cert = csrOK.Status.Certificate return true, nil } - klog.V(1).Infof("Waiting for the client certificate to be issued") + klog.V(1).Infof(fmt.Sprintf("Waiting for the client certificate of csr %s to be issued", csrName)) + klog.V(1).Infof("Approve the CSR %s manually by executing `kubectl certificate approve %s` on the control plane", csrName, csrName) return false, nil }) if err != nil { return nil, err } - karmadaAgentCfg := CreateWithCert( + return CreateWithCert( karmadaClusterInfo.Server, DefaultClusterName, o.ClusterName, karmadaClusterInfo.CertificateAuthorityData, cert, pkData, - ) + ), nil +} + +// constructKarmadaAgentConfig constructs the final kubeconfig used by karmada-agent +func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*clientcmdapi.Config, error) { + csrName := o.ClusterName + "-" + k8srand.String(5) + + karmadaAgentCfg, err := o.constructKubeConfig(bootstrapClient, karmadaClusterInfo, csrName, generateAgentUserName(o.ClusterName), []string{ClusterPermissionGroups}) + if err != nil { + return nil, err + } kubeConfigFile := filepath.Join(KarmadaDir, KarmadaAgentKubeConfigFileName) @@ -588,6 +961,11 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub return karmadaAgentCfg, nil } +// constructKarmadaAgentConfig constructs the kubeconfig to generate rbac config for karmada-agent. +func (o *CommandRegisterOption) constructAgentRBACGeneratorConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster, csrName string) (*clientcmdapi.Config, error) { + return o.constructKubeConfig(bootstrapClient, karmadaClusterInfo, csrName, AgentRBACGenerator, []string{ClusterPermissionGroups}) +} + // createSecretAndRBACInMemberCluster create required secrets and rbac in member cluster func (o *CommandRegisterOption) createSecretAndRBACInMemberCluster(karmadaAgentCfg *clientcmdapi.Config) error { configBytes, err := clientcmd.Write(*karmadaAgentCfg) @@ -781,7 +1159,7 @@ func (o *CommandRegisterOption) makeKarmadaAgentDeployment() *appsv1.Deployment } // generateKeyAndCSR generate private key and csr -func generateKeyAndCSR(clusterName string) (*rsa.PrivateKey, []byte, error) { +func generateKeyAndCSR(commonName string, organization []string) (*rsa.PrivateKey, []byte, error) { pk, err := rsa.GenerateKey(rand.Reader, 3072) if err != nil { return nil, nil, err @@ -789,8 +1167,8 @@ func generateKeyAndCSR(clusterName string) (*rsa.PrivateKey, []byte, error) { csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{ Subject: pkix.Name{ - CommonName: ClusterPermissionPrefix + clusterName, - Organization: []string{ClusterPermissionGroups}, + CommonName: commonName, + Organization: organization, }, }, pk) if err != nil { @@ -1070,3 +1448,7 @@ func ToKarmadaClient(config *clientcmdapi.Config) (*karmadaclientset.Clientset, return karmadaClient, nil } + +func generateAgentUserName(clusterName string) string { + return ClusterPermissionPrefix + clusterName +} diff --git a/pkg/karmadactl/unregister/unregister.go b/pkg/karmadactl/unregister/unregister.go index 90b7ccfaa98d..a3866a954637 100644 --- a/pkg/karmadactl/unregister/unregister.go +++ b/pkg/karmadactl/unregister/unregister.go @@ -135,6 +135,9 @@ type CommandUnregisterOption struct { // MemberClusterClient member cluster client set MemberClusterClient kubeclient.Interface + + // rbacResources contains RBAC resources that grant the necessary permissions for the unregistering cluster to access to Karmada control plane. + rbacResources *register.RBACResources } // AddFlags adds flags to the specified FlagSet. @@ -157,6 +160,8 @@ func (j *CommandUnregisterOption) Complete(args []string) error { if len(args) > 0 { j.ClusterName = args[0] } + + j.rbacResources = register.GenerateRBACResources(j.ClusterName, j.ClusterNamespace) return nil } @@ -303,30 +308,67 @@ func (j *CommandUnregisterOption) getKarmadaAgentConfig(agent *appsv1.Deployment return clientcmd.Load(agentConfigSecret.Data[fileName]) } -type obj struct{ Kind, Name, Namespace string } - -func (o *obj) ToString() string { - if o.Namespace == "" { - return fmt.Sprintf("%s/%s", o.Kind, o.Name) - } - return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name) -} - // RunUnregisterCluster unregister the pull mode cluster from karmada. func (j *CommandUnregisterOption) RunUnregisterCluster() error { if j.DryRun { return nil } - // 1. delete the cluster object from the Karmada control plane + start := time.Now() + // 1. delete the work object from the Karmada control plane + // When deleting a cluster, the deletion triggers the removal of executionSpace, which can lead to the deletion of RBAC roles related to work. + // Therefore, the deletion of work should be performed before deleting the cluster. + err := cmdutil.EnsureWorksDeleted(j.ControlPlaneClient, names.GenerateExecutionSpaceName(j.ClusterName), j.Wait) + if err != nil { + klog.Errorf("Failed to delete works object. cluster name: %s, error: %v", j.ClusterName, err) + return err + } + j.Wait = j.Wait - time.Since(start) + + // 2. delete the cluster object from the Karmada control plane //TODO: add flag --force to implement force deletion. - if err := cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil { + if err = cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil { klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) return err } klog.Infof("Successfully delete cluster object (%s) from control plane.", j.ClusterName) - // 2. delete resource created by karmada in member cluster + if j.KarmadaConfig != "" { + if err = j.rbacResources.Delete(j.ControlPlaneKubeClient); err != nil { + klog.Errorf("Failed to delete karmada-agent RBAC resources from control plane. cluster name: %s, error: %v", j.ClusterName, err) + return err + } + klog.Infof("Successfully delete karmada-agent RBAC resources from control plane. cluster name: %s", j.ClusterName) + } else { + klog.Warningf("The RBAC resources on the control plane need to be manually cleaned up, including the following resources:\n%s", j.rbacResources.ToString()) + } + + // 3. delete resource created by karmada in member cluster + if err = j.cleanupMemberClusterResources(); err != nil { + return err + } + + // 4. delete local obsolete files generated by karmadactl + localObsoleteFiles := []register.Obj{ + {Kind: "File", Name: filepath.Join(register.KarmadaDir, register.KarmadaAgentKubeConfigFileName)}, + {Kind: "File", Name: register.CACertPath}, + } + for _, obj := range localObsoleteFiles { + if err = os.Remove(obj.Name); err != nil { + if os.IsNotExist(err) { + continue + } + klog.Errorf("Failed to delete local file (%v) in current node: %+v.", obj.Name, err) + return err + } + klog.Infof("Successfully delete local file (%v) in current node.", obj.Name) + } + + return nil +} + +// cleanupMemberClusterResources clean up the resources which created by karmada in member cluster +func (j *CommandUnregisterOption) cleanupMemberClusterResources() error { var err error for _, resource := range j.listMemberClusterResources() { switch resource.Kind { @@ -350,29 +392,12 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error { } klog.Infof("Successfully delete resource (%v) from member cluster (%s).", resource, j.ClusterName) } - - // 3. delete local obsolete files generated by karmadactl - localObsoleteFiles := []obj{ - {Kind: "File", Name: filepath.Join(register.KarmadaDir, register.KarmadaAgentKubeConfigFileName)}, - {Kind: "File", Name: register.CACertPath}, - } - for _, obj := range localObsoleteFiles { - if err = os.Remove(obj.Name); err != nil { - if os.IsNotExist(err) { - continue - } - klog.Errorf("Failed to delete local file (%v) in current node: %+v.", obj.Name, err) - return err - } - klog.Infof("Successfully delete local file (%v) in current node.", obj.Name) - } - - return nil + return err } // listMemberClusterResources lists resources to be deleted which created by karmada in member cluster -func (j *CommandUnregisterOption) listMemberClusterResources() []obj { - return []obj{ +func (j *CommandUnregisterOption) listMemberClusterResources() []register.Obj { + return []register.Obj{ // the rbac resource prepared for karmada-controller-manager to access member cluster's kube-apiserver {Kind: "ServiceAccount", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName(j.ClusterName)}, {Kind: "ClusterRole", Name: names.GenerateRoleName(names.GenerateServiceAccountName(j.ClusterName))}, diff --git a/pkg/karmadactl/unregister/unregister_test.go b/pkg/karmadactl/unregister/unregister_test.go index ab3c97e9df63..875ddc83ba88 100644 --- a/pkg/karmadactl/unregister/unregister_test.go +++ b/pkg/karmadactl/unregister/unregister_test.go @@ -216,6 +216,7 @@ func TestCommandUnregisterOption_RunUnregisterCluster(t *testing.T) { } j.ControlPlaneClient = fakekarmadaclient.NewSimpleClientset(tt.clusterObject...) j.MemberClusterClient = fake.NewSimpleClientset(tt.clusterResources...) + j.rbacResources = register.GenerateRBACResources(j.ClusterName, j.ClusterNamespace) err := j.RunUnregisterCluster() if (err == nil && tt.wantErr) || (err != nil && !tt.wantErr) { t.Errorf("RunUnregisterCluster() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/karmadactl/util/work.go b/pkg/karmadactl/util/work.go new file mode 100644 index 000000000000..48fe759bb99d --- /dev/null +++ b/pkg/karmadactl/util/work.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// EnsureWorksDeleted ensures that all Work resources in the specified namespace are deleted. +func EnsureWorksDeleted(controlPlaneKarmadaClient karmadaclientset.Interface, namespace string, + timeout time.Duration) error { + // make sure the works object under the given namespace has been deleted. + err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(context.Context) (done bool, err error) { + list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("failed to list work in namespace %s", namespace) + } + + if len(list.Items) == 0 { + return true, nil + } + for i := range list.Items { + work := &list.Items[i] + err = controlPlaneKarmadaClient.WorkV1alpha1().Works(namespace).Delete(context.TODO(), work.GetName(), metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("failed to delete the work(%s/%s)", namespace, work.GetName()) + } + } + return false, nil + }) + + return err +} diff --git a/pkg/util/rbac.go b/pkg/util/rbac.go index 81496680ed49..93bf5379b7ea 100644 --- a/pkg/util/rbac.go +++ b/pkg/util/rbac.go @@ -191,3 +191,49 @@ func GenerateImpersonationRules(subjects []rbacv1.Subject) []rbacv1.PolicyRule { return rules } + +// CreateRole just try to create the Role. +func CreateRole(client kubeclient.Interface, roleObj *rbacv1.Role) (*rbacv1.Role, error) { + createdObj, err := client.RbacV1().Roles(roleObj.GetNamespace()).Create(context.TODO(), roleObj, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return roleObj, nil + } + + return nil, err + } + + return createdObj, nil +} + +// CreateRoleBinding just try to create the RoleBinding. +func CreateRoleBinding(client kubeclient.Interface, roleBindingObj *rbacv1.RoleBinding) (*rbacv1.RoleBinding, error) { + createdObj, err := client.RbacV1().RoleBindings(roleBindingObj.GetNamespace()).Create(context.TODO(), roleBindingObj, metav1.CreateOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return roleBindingObj, nil + } + + return nil, err + } + + return createdObj, nil +} + +// DeleteRole just try to delete the Role. +func DeleteRole(client kubeclient.Interface, namespace, name string) error { + err := client.RbacV1().Roles(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +// DeleteRoleBinding just try to delete the RoleBinding. +func DeleteRoleBinding(client kubeclient.Interface, namespace, name string) error { + err := client.RbacV1().RoleBindings(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +}