From 9ec1ffc521d08217d18bf25ac820ee49ead5cd66 Mon Sep 17 00:00:00 2001 From: ONE7live Date: Sat, 7 Oct 2023 13:41:26 +0800 Subject: [PATCH] feat: Kosmosctl supports knode join and unjoin Signed-off-by: ONE7live --- examples/knode-demo.yaml | 8 + pkg/kosmosctl/get/get.go | 27 ++- pkg/kosmosctl/install/install.go | 150 ++++++++----- pkg/kosmosctl/join/join.go | 209 ++++++++++++------ pkg/kosmosctl/kosmosctl.go | 4 +- .../manifest/manifest_clusterrolebindings.go | 12 +- .../manifest/manifest_clusterroles.go | 4 +- pkg/kosmosctl/manifest/manifest_crds.go | 2 +- .../manifest/manifest_deployments.go | 24 +- .../manifest/manifest_serviceaccounts.go | 8 +- pkg/kosmosctl/uninstall/uninstall.go | 23 +- pkg/kosmosctl/unjoin/unjoin.go | 132 +++++++---- pkg/kosmosctl/util/builder.go | 9 + pkg/kosmosctl/util/check.go | 53 +++++ 14 files changed, 466 insertions(+), 199 deletions(-) create mode 100644 examples/knode-demo.yaml diff --git a/examples/knode-demo.yaml b/examples/knode-demo.yaml new file mode 100644 index 000000000..45fbacca4 --- /dev/null +++ b/examples/knode-demo.yaml @@ -0,0 +1,8 @@ +apiVersion: kosmos.io/v1alpha1 +kind: Knode +metadata: + name: knode-worker1 +spec: + nodeName: knode-worker1 + type: k8s + kubeconfig: {{ .cluster-kubeconfig }} diff --git a/pkg/kosmosctl/get/get.go b/pkg/kosmosctl/get/get.go index 59aac6b93..134ea9af7 100644 --- a/pkg/kosmosctl/get/get.go +++ b/pkg/kosmosctl/get/get.go @@ -9,27 +9,32 @@ import ( ctlget "k8s.io/kubectl/pkg/cmd/get" ctlutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" + + "github.com/kosmos.io/kosmos/pkg/kosmosctl/util" ) const ( ClustersGroupResource = "clusters.kosmos.io" ClusterNodesGroupResource = "clusternodes.kosmos.io" + KnodesGroupResource = "knodes.kosmos.io" ) type CommandGetOptions struct { Cluster string ClusterNode string + Namespace string + GetOptions *ctlget.GetOptions } -// NewCmdGet Display resources from the Clusterlink control plane. +// NewCmdGet Display resources from the Kosmos control plane. func NewCmdGet(f ctlutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewCommandGetOptions(streams) cmd := &cobra.Command{ Use: fmt.Sprintf("get [(-o|--output=)%s] (TYPE[.VERSION][.GROUP] [NAME | -l label] | TYPE[.VERSION][.GROUP]/NAME ...) [flags]", strings.Join(o.GetOptions.PrintFlags.AllowedFormats(), "|")), - Short: i18n.T("Display resources from the Clusterlink control plane"), + Short: i18n.T("Display resources from the Kosmos control plane"), Long: "", Example: "", SilenceUsage: true, @@ -43,6 +48,8 @@ func NewCmdGet(f ctlutil.Factory, streams genericclioptions.IOStreams) *cobra.Co } o.GetOptions.PrintFlags.AddFlags(cmd) + flags := cmd.Flags() + flags.StringVarP(&o.Namespace, "namespace", "n", util.DefaultNamespace, "If present, the namespace scope for this CLI request.") return cmd } @@ -61,6 +68,8 @@ func (o *CommandGetOptions) Complete(f ctlutil.Factory, cmd *cobra.Command, args return fmt.Errorf("kosmosctl get complete error, options failed: %s", err) } + o.GetOptions.Namespace = o.Namespace + return nil } @@ -74,13 +83,13 @@ func (o *CommandGetOptions) Validate() error { } func (o *CommandGetOptions) Run(f ctlutil.Factory, cmd *cobra.Command, args []string) error { - for i := range args { - switch args[i] { - case "cluster", "clusters": - args[i] = ClustersGroupResource - case "clusternode", "clusternodes": - args[i] = ClusterNodesGroupResource - } + switch args[0] { + case "cluster", "clusters": + args[0] = ClustersGroupResource + case "clusternode", "clusternodes": + args[0] = ClusterNodesGroupResource + case "knode", "knodes": + args[0] = KnodesGroupResource } err := o.GetOptions.Run(f, cmd, args) diff --git a/pkg/kosmosctl/install/install.go b/pkg/kosmosctl/install/install.go index 0cad041d6..0e88584e8 100644 --- a/pkg/kosmosctl/install/install.go +++ b/pkg/kosmosctl/install/install.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" @@ -12,20 +13,33 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/homedir" + "k8s.io/klog/v2" ctlutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" + "k8s.io/kubectl/pkg/util/templates" "github.com/kosmos.io/kosmos/pkg/kosmosctl/manifest" "github.com/kosmos.io/kosmos/pkg/kosmosctl/util" "github.com/kosmos.io/kosmos/pkg/version" ) +var installExample = templates.Examples(i18n.T(` + # Install all module to Kosmos control plane, e.g: + kosmosctl install + + # Install clusterlink module to Kosmos control plane, e.g: + kosmosctl install -m clusterlink + + # Install clustertree module to Kosmos control plane, e.g: + kosmosctl install -m clustertree --host-kubeconfig=[host-kubeconfig] +`)) + type CommandInstallOptions struct { - Namespace string - ImageRegistry string - Version string - Module string - HostKubeConfig string + Namespace string + ImageRegistry string + Version string + Module string Client kubernetes.Interface ExtensionsClient extensionsclient.Interface @@ -39,7 +53,7 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command { Use: "install", Short: i18n.T("Install the Kosmos control plane in a Kubernetes cluster"), Long: "", - Example: "", + Example: installExample, SilenceUsage: true, DisableFlagsInUseLine: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -54,7 +68,6 @@ func NewCmdInstall(f ctlutil.Factory) *cobra.Command { flags.StringVarP(&o.Namespace, "namespace", "n", util.DefaultNamespace, "Kosmos namespace.") flags.StringVarP(&o.ImageRegistry, "private-image-registry", "", util.DefaultImageRepository, "Private image registry where pull images from. If set, all required images will be downloaded from it, it would be useful in offline installation scenarios. In addition, you still can use --kube-image-registry to specify the registry for Kubernetes's images.") flags.StringVarP(&o.Module, "module", "m", util.DefaultInstallModule, "Kosmos specify the module to install.") - flags.StringVar(&o.HostKubeConfig, "host-kubeconfig", "", "Absolute path to the host kubeconfig file.") return cmd } @@ -83,40 +96,41 @@ func (o *CommandInstallOptions) Validate() error { return fmt.Errorf("namespace must be specified") } - if o.Module != "clusterlink" && o.HostKubeConfig == "" { - return fmt.Errorf("host-kubeconfig must be specified") - } - return nil } func (o *CommandInstallOptions) Run() error { + klog.Info("Kosmos starts installing.") switch o.Module { case "clusterlink": err := o.runClusterlink() if err != nil { return err } - case "clusterrouter": - err := o.runClusterrouter() + util.CheckInstall("Clusterlink") + case "clustertree": + err := o.runClustertree() if err != nil { return err } + util.CheckInstall("Clustertree") case "all": err := o.runClusterlink() if err != nil { return err } - err = o.runClusterrouter() + err = o.runClustertree() if err != nil { return err } + util.CheckInstall("Clusterlink && Clustertree") } return nil } func (o *CommandInstallOptions) runClusterlink() error { + klog.Info("Start creating kosmos-clusterlink...") namespace := &corev1.Namespace{} namespace.Name = o.Namespace _, err := o.Client.CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{}) @@ -125,7 +139,9 @@ func (o *CommandInstallOptions) runClusterlink() error { return fmt.Errorf("kosmosctl install clusterlink run error, namespace options failed: %v", err) } } + klog.Info("Namespace kosmos-system has been created.") + klog.Info("Start creating kosmos-clusterlink ServiceAccount...") clusterlinkServiceAccount, err := util.GenerateServiceAccount(manifest.ClusterlinkNetworkManagerServiceAccount, manifest.ServiceAccountReplace{ Namespace: o.Namespace, }) @@ -138,7 +154,9 @@ func (o *CommandInstallOptions) runClusterlink() error { return fmt.Errorf("kosmosctl install clusterlink run error, serviceaccount options failed: %v", err) } } + klog.Info("ServiceAccount clusterlink-network-manager has been created.") + klog.Info("Start creating kosmos-clusterlink ClusterRole...") clusterlinkClusterRole, err := util.GenerateClusterRole(manifest.ClusterlinkNetworkManagerClusterRole, nil) if err != nil { return err @@ -149,7 +167,9 @@ func (o *CommandInstallOptions) runClusterlink() error { return fmt.Errorf("kosmosctl install clusterlink run error, clusterrole options failed: %v", err) } } + klog.Info("ClusterRole clusterlink-network-manager has been created.") + klog.Info("Start creating kosmos-clusterlink ClusterRoleBinding...") clusterlinkClusterRoleBinding, err := util.GenerateClusterRoleBinding(manifest.ClusterlinkNetworkManagerClusterRoleBinding, manifest.ClusterRoleBindingReplace{ Namespace: o.Namespace, }) @@ -162,8 +182,9 @@ func (o *CommandInstallOptions) runClusterlink() error { return fmt.Errorf("kosmosctl install clusterlink run error, clusterrolebinding options failed: %v", err) } } + klog.Info("ClusterRoleBinding clusterlink-network-manager has been created.") - //ToDo CRD manifest parameters are configurable, through obj + klog.Info("Attempting to create kosmos-clusterlink knode CRDs...") crds := apiextensionsv1.CustomResourceDefinitionList{} clusterlinkCluster, err := util.GenerateCustomResourceDefinition(manifest.ClusterlinkCluster, nil) if err != nil { @@ -185,9 +206,11 @@ func (o *CommandInstallOptions) runClusterlink() error { return fmt.Errorf("kosmosctl install clusterlink run error, crd options failed: %v", err) } } + klog.Info("Create CRD " + crds.Items[i].Name + " successful.") } - deployment, err := util.GenerateDeployment(manifest.ClusterlinkNetworkManagerDeployment, manifest.DeploymentReplace{ + klog.Info("Start creating kosmos-clusterlink Deployment...") + clusterlinkDeployment, err := util.GenerateDeployment(manifest.ClusterlinkNetworkManagerDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, ImageRepository: o.ImageRegistry, Version: version.GetReleaseVersion().PatchRelease(), @@ -195,95 +218,113 @@ func (o *CommandInstallOptions) runClusterlink() error { if err != nil { return err } - _, err = o.Client.AppsV1().Deployments(o.Namespace).Create(context.Background(), deployment, metav1.CreateOptions{}) + _, err = o.Client.AppsV1().Deployments(o.Namespace).Create(context.Background(), clusterlinkDeployment, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { return fmt.Errorf("kosmosctl install clusterlink run error, deployment options failed: %v", err) } } + if err = util.WaitDeploymentReady(o.Client, clusterlinkDeployment, 120); err != nil { + return fmt.Errorf("kosmosctl install clusterlink run error, deployment options failed: %v", err) + } else { + klog.Info("Deployment clusterlink-network-manager has been created.") + } return nil } -func (o *CommandInstallOptions) runClusterrouter() error { +func (o *CommandInstallOptions) runClustertree() error { + klog.Info("Start creating kosmos-clustertree...") namespace := &corev1.Namespace{} namespace.Name = o.Namespace _, err := o.Client.CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl install clusterrouter run error, namespace options failed: %v", err) + return fmt.Errorf("kosmosctl install clustertree run error, namespace options failed: %v", err) } } + klog.Info("Namespace kosmos-system has been created.") - hostKubeconfig, err := os.ReadFile(o.HostKubeConfig) + klog.Info("Start creating kosmos-clustertree ServiceAccount...") + clustertreeServiceAccount, err := util.GenerateServiceAccount(manifest.ClusterTreeKnodeManagerServiceAccount, manifest.ServiceAccountReplace{ + Namespace: o.Namespace, + }) if err != nil { - return fmt.Errorf("kosmosctl install clusterrouter run error, host-kubeconfig read failed: %v", err) - } - clusterRouterConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "host-kubeconfig", - Namespace: o.Namespace, - }, - Data: map[string]string{ - "kubeconfig": string(hostKubeconfig), - }, + return err } - _, err = o.Client.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), clusterRouterConfigMap, metav1.CreateOptions{}) + _, err = o.Client.CoreV1().ServiceAccounts(o.Namespace).Create(context.TODO(), clustertreeServiceAccount, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl install clusterrouter run error, configmap options failed: %v", err) + return fmt.Errorf("kosmosctl install clustertree run error, serviceaccount options failed: %v", err) } } + klog.Info("ServiceAccount clustertree-knode-manager has been created.") - clusterRouterServiceAccount, err := util.GenerateServiceAccount(manifest.ClusterRouterKnodeServiceAccount, manifest.ServiceAccountReplace{ - Namespace: o.Namespace, - }) + klog.Info("Start creating kosmos-clustertree ClusterRole...") + clustertreeClusterRole, err := util.GenerateClusterRole(manifest.ClusterTreeKnodeManagerClusterRole, nil) if err != nil { return err } - _, err = o.Client.CoreV1().ServiceAccounts(o.Namespace).Create(context.TODO(), clusterRouterServiceAccount, metav1.CreateOptions{}) + _, err = o.Client.RbacV1().ClusterRoles().Create(context.TODO(), clustertreeClusterRole, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl install clusterrouter run error, serviceaccount options failed: %v", err) + return fmt.Errorf("kosmosctl install clustertree run error, clusterrole options failed: %v", err) } } + klog.Info("ClusterRole clustertree-knode has been created.") - clusterRouterClusterRole, err := util.GenerateClusterRole(manifest.ClusterRouterKnodeClusterRole, nil) + klog.Info("Start creating kosmos-clustertree ClusterRoleBinding...") + clustertreeClusterRoleBinding, err := util.GenerateClusterRoleBinding(manifest.ClusterTreeKnodeManagerClusterRoleBinding, manifest.ClusterRoleBindingReplace{ + Namespace: o.Namespace, + }) if err != nil { return err } - _, err = o.Client.RbacV1().ClusterRoles().Create(context.TODO(), clusterRouterClusterRole, metav1.CreateOptions{}) + _, err = o.Client.RbacV1().ClusterRoleBindings().Create(context.TODO(), clustertreeClusterRoleBinding, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl install clusterrouter run error, clusterrole options failed: %v", err) + return fmt.Errorf("kosmosctl install clustertree run error, clusterrolebinding options failed: %v", err) } } + klog.Info("ClusterRoleBinding clustertree-knode has been created.") - clusterRouterClusterRoleBinding, err := util.GenerateClusterRoleBinding(manifest.ClusterRouterKnodeClusterRoleBinding, manifest.ClusterRoleBindingReplace{ - Namespace: o.Namespace, - }) + klog.Info("Attempting to create kosmos-clustertree knode CRDs...") + clustertreeKnode, err := util.GenerateCustomResourceDefinition(manifest.ClusterTreeKnode, nil) if err != nil { return err } - _, err = o.Client.RbacV1().ClusterRoleBindings().Create(context.TODO(), clusterRouterClusterRoleBinding, metav1.CreateOptions{}) + _, err = o.ExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), clustertreeKnode, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl install clusterrouter run error, clusterrolebinding options failed: %v", err) + return fmt.Errorf("kosmosctl install clustertree run error, crd options failed: %v", err) } } + klog.Info("Create CRD " + clustertreeKnode.Name + " successful.") - clusterRouterKnode, err := util.GenerateCustomResourceDefinition(manifest.ClusterRouterKnode, nil) + klog.Info("Start creating kosmos-clustertree ConfigMap...") + hostKubeconfig, err := os.ReadFile(filepath.Join(homedir.HomeDir(), ".kube", "config")) if err != nil { - return err + return fmt.Errorf("kosmosctl install clustertree run error, host-kubeconfig read failed: %v", err) + } + clustertreeConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "host-kubeconfig", + Namespace: o.Namespace, + }, + Data: map[string]string{ + "kubeconfig": string(hostKubeconfig), + }, } - _, err = o.ExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), clusterRouterKnode, metav1.CreateOptions{}) + _, err = o.Client.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), clustertreeConfigMap, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl install clusterrouter run error, crd options failed: %v", err) + return fmt.Errorf("kosmosctl install clustertree run error, configmap options failed: %v", err) } } + klog.Info("ConfigMap host-kubeconfig has been created.") - deployment, err := util.GenerateDeployment(manifest.ClusterRouterKnodeDeployment, manifest.DeploymentReplace{ + klog.Info("Start creating kosmos-clustertree Deployment...") + clustertreeDeployment, err := util.GenerateDeployment(manifest.ClusterTreeKnodeManagerDeployment, manifest.DeploymentReplace{ Namespace: o.Namespace, ImageRepository: o.ImageRegistry, Version: version.GetReleaseVersion().PatchRelease(), @@ -291,12 +332,17 @@ func (o *CommandInstallOptions) runClusterrouter() error { if err != nil { return err } - _, err = o.Client.AppsV1().Deployments(o.Namespace).Create(context.Background(), deployment, metav1.CreateOptions{}) + _, err = o.Client.AppsV1().Deployments(o.Namespace).Create(context.Background(), clustertreeDeployment, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("kosmosctl install clusterrouter run error, deployment options failed: %v", err) + return fmt.Errorf("kosmosctl install clustertree run error, deployment options failed: %v", err) } } + if err = util.WaitDeploymentReady(o.Client, clustertreeDeployment, 120); err != nil { + return fmt.Errorf("kosmosctl install clustertree run error, deployment options failed: %v", err) + } else { + klog.Info("Deployment clustertree-knode-manager has been created.") + } return nil } diff --git a/pkg/kosmosctl/join/join.go b/pkg/kosmosctl/join/join.go index 220ec580f..84bf50400 100644 --- a/pkg/kosmosctl/join/join.go +++ b/pkg/kosmosctl/join/join.go @@ -10,144 +10,208 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" k8syaml "k8s.io/apimachinery/pkg/runtime/serializer/yaml" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" ctlutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" "k8s.io/kubectl/pkg/util/templates" "github.com/kosmos.io/kosmos/pkg/kosmosctl/manifest" "github.com/kosmos.io/kosmos/pkg/kosmosctl/util" - "github.com/kosmos.io/kosmos/pkg/utils" "github.com/kosmos.io/kosmos/pkg/version" ) var joinExample = templates.Examples(i18n.T(` - # join member-cluster to master control plane - kosmosctl join -f member-cluster.yaml --master-kubeconfig=[master-kubeconfig] --cluster-kubeconfig=[member-kubeconfig] + # Join cluster resource from a directory containing cluster.yaml, e.g: + kosmosctl join -f cluster.yaml --master-kubeconfig=[master-kubeconfig] --cluster-kubeconfig=[cluster-kubeconfig] + + # Join cluster resource without master-kubeconfig, e.g: + kosmosctl join -f cluster.yaml --cluster-kubeconfig=[cluster-kubeconfig] + + # Join knode resource from a directory containing knode.yaml, e.g: + kosmosctl join -f knode.yaml --master-kubeconfig=[master-kubeconfig] --knode-kubeconfig=[knode-kubeconfig] + + # Join knode resource without master-kubeconfig, e.g: + kosmosctl join -f knode.yaml --knode-kubeconfig=[knode-kubeconfig] `)) type CommandJoinOptions struct { - File string + Module string + File string + Resource *unstructured.Unstructured + MasterKubeConfig string ClusterKubeConfig string + KnodeKubeConfig string - DynamicClient *dynamic.DynamicClient Client kubernetes.Interface + DynamicClient *dynamic.DynamicClient } -// NewCmdJoin join this to Clusterlink control plane +// NewCmdJoin join resource to Kosmos control plane. func NewCmdJoin(f ctlutil.Factory) *cobra.Command { o := &CommandJoinOptions{} cmd := &cobra.Command{ Use: "join", - Short: i18n.T("Join this to clusterlink control plane"), + Short: i18n.T("Join resource to Kosmos control plane"), Long: "", Example: joinExample, SilenceUsage: true, DisableFlagsInUseLine: true, RunE: func(cmd *cobra.Command, args []string) error { - ctlutil.CheckErr(o.Complete(f, cmd, args)) + ctlutil.CheckErr(o.Complete(f)) ctlutil.CheckErr(o.Validate()) - ctlutil.CheckErr(o.Run(f, cmd, args)) + ctlutil.CheckErr(o.Run()) return nil }, } - cmd.Flags().StringVarP(&o.File, "file", "f", "", "cluster.yaml") + flags := cmd.Flags() + flags.StringVarP(&o.File, "file", "f", "", "Absolute path to the resource file.") err := cmd.MarkFlagRequired("file") if err != nil { fmt.Printf("kosmosctl join cmd error, MarkFlagRequired failed: %s", err) } - cmd.Flags().StringVarP(&o.MasterKubeConfig, "master-kubeconfig", "", "", "master-kubeconfig") - err = cmd.MarkFlagRequired("master-kubeconfig") - if err != nil { - fmt.Printf("kosmosctl join cmd error, MarkFlagRequired failed: %s", err) - } - cmd.Flags().StringVarP(&o.ClusterKubeConfig, "cluster-kubeconfig", "", "", "cluster-kubeconfig") - err = cmd.MarkFlagRequired("cluster-kubeconfig") - if err != nil { - fmt.Printf("kosmosctl join cmd error, MarkFlagRequired failed: %s", err) - } + flags.StringVar(&o.MasterKubeConfig, "master-kubeconfig", "", "Absolute path to the master kubeconfig file.") + flags.StringVar(&o.ClusterKubeConfig, "cluster-kubeconfig", "", "Absolute path to the cluster kubeconfig file.") + flags.StringVar(&o.KnodeKubeConfig, "knode-kubeconfig", "", "Absolute path to the knode kubeconfig file.") return cmd } -func (o *CommandJoinOptions) Complete(f ctlutil.Factory, cmd *cobra.Command, args []string) error { +func (o *CommandJoinOptions) Complete(f ctlutil.Factory) error { var masterConfig *restclient.Config var err error - masterConfig, err = clientcmd.BuildConfigFromFlags("", o.MasterKubeConfig) - if err != nil { - return fmt.Errorf("kosmosctl join complete error, generate masterConfig failed: %s", err) + if len(o.MasterKubeConfig) > 0 { + masterConfig, err = clientcmd.BuildConfigFromFlags("", o.MasterKubeConfig) + if err != nil { + return fmt.Errorf("kosmosctl join complete error, generate masterConfig failed: %s", err) + } + } else { + masterConfig, err = f.ToRESTConfig() + if err != nil { + return fmt.Errorf("kosmosctl join complete error, generate masterConfig failed: %s", err) + } } - clusterConfig, err := clientcmd.BuildConfigFromFlags("", o.ClusterKubeConfig) + o.DynamicClient, err = dynamic.NewForConfig(masterConfig) if err != nil { - return fmt.Errorf("kosmosctl join complete error, generate memberConfig failed: %s", err) + return fmt.Errorf("kosmosctl join complete error, generate dynamic client failed: %s", err) } - o.Client, err = kubernetes.NewForConfig(clusterConfig) - if err != nil { - return fmt.Errorf("kosmosctl join complete error, generate basic client failed: %v", err) + if len(o.ClusterKubeConfig) > 0 { + clusterConfig, err := clientcmd.BuildConfigFromFlags("", o.ClusterKubeConfig) + if err != nil { + return fmt.Errorf("kosmosctl join complete error, generate clusterConfig failed: %s", err) + } + + o.Client, err = kubernetes.NewForConfig(clusterConfig) + if err != nil { + return fmt.Errorf("kosmosctl join complete error, generate basic client failed: %v", err) + } } - o.DynamicClient, err = dynamic.NewForConfig(masterConfig) - if err != nil { - return fmt.Errorf("kosmosctl join complete error, generate dynamic client failed: %s", err) + if len(o.KnodeKubeConfig) > 0 { + knodeConfig, err := clientcmd.BuildConfigFromFlags("", o.KnodeKubeConfig) + if err != nil { + return fmt.Errorf("kosmosctl join complete error, generate knodeConfig failed: %s", err) + } + + o.Client, err = kubernetes.NewForConfig(knodeConfig) + if err != nil { + return fmt.Errorf("kosmosctl join complete error, generate basic client failed: %v", err) + } } return nil } func (o *CommandJoinOptions) Validate() error { - return nil -} - -func (o *CommandJoinOptions) Run(f ctlutil.Factory, cmd *cobra.Command, args []string) error { - // 1. create cluster in master - cr, err := os.ReadFile(o.File) + yamlFileByte, err := os.ReadFile(o.File) if err != nil { - return fmt.Errorf("(cluster) kosmosctl join run warning, readfile failed: %s", err) + return fmt.Errorf("kosmosctl join validate warning, read yaml file failed: %s", err) } + decoder := k8syaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) obj := &unstructured.Unstructured{} - _, _, err = decoder.Decode(cr, nil, obj) + _, _, err = decoder.Decode(yamlFileByte, nil, obj) if err != nil { - return fmt.Errorf("(cluster) kosmosctl join run warning, decode failed: %s", err) + return fmt.Errorf("kosmosctl join validate warning, decode failed: %s", err) + } + + switch obj.GetKind() { + case "Cluster": + _, err = o.DynamicClient.Resource(util.ClusterGVR).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + if err != nil { + if apierrors.IsAlreadyExists(err) { + return fmt.Errorf("kosmosctl join validate warning, clsuter already exists: %s", err) + } + } + o.Module = "clusterlink" + o.Resource = obj + case "Knode": + _, err = o.DynamicClient.Resource(util.KnodeGVR).Get(context.TODO(), obj.GetName(), metav1.GetOptions{}) + if err != nil && apierrors.IsAlreadyExists(err) { + if apierrors.IsAlreadyExists(err) { + return fmt.Errorf("kosmosctl join validate warning, knode already exists: %s", err) + } + } + o.Module = "clustertree" + o.Resource = obj } - _, err = o.DynamicClient.Resource(schema.GroupVersionResource{ - Group: "kosmos.io", - Version: "v1alpha1", - Resource: "clusters", - }).Namespace("").Create(context.TODO(), obj, metav1.CreateOptions{}) + + return nil +} + +func (o *CommandJoinOptions) Run() error { + switch o.Module { + case "clusterlink": + err := o.runCluster() + if err != nil { + return err + } + case "clustertree": + err := o.runKnode() + if err != nil { + return err + } + } + + return nil +} + +func (o *CommandJoinOptions) runCluster() error { + klog.Info("Start registering cluster to kosmos control plane...") + // 1. create cluster in master + _, err := o.DynamicClient.Resource(util.ClusterGVR).Namespace("").Create(context.TODO(), o.Resource, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("(cluster) kosmosctl join run warning, create cluster failed: %s", err) } // 2. create namespace in member namespace := &corev1.Namespace{} - namespace.Name = utils.NamespaceClusterLinksystem + namespace.Name = util.DefaultNamespace _, err = o.Client.CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("(namespace) kosmosctl join run warning, create namespace failed: %s", err) + return fmt.Errorf("(cluster namespace) kosmosctl join run warning, create namespace failed: %s", err) } // 3. create secret in member masterKubeConfig, err := os.ReadFile(o.MasterKubeConfig) if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("(secret) kosmosctl join run warning, read masterconfig failed: %s", err) + return fmt.Errorf("(cluster secret) kosmosctl join run warning, read masterconfig failed: %s", err) } secret := &corev1.Secret{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ - Name: utils.ControlPanelSecretName, - Namespace: utils.NamespaceClusterLinksystem, + Name: util.ControlPanelSecretName, + Namespace: util.DefaultNamespace, }, Data: map[string][]byte{ "kubeconfig": masterKubeConfig, @@ -155,47 +219,64 @@ func (o *CommandJoinOptions) Run(f ctlutil.Factory, cmd *cobra.Command, args []s } _, err = o.Client.CoreV1().Secrets(secret.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("(secret) kosmosctl join run warning, create secret failed: %s", err) + return fmt.Errorf("(cluster secret) kosmosctl join run warning, create secret failed: %s", err) } // 4. create rbac in member clusterRole, err := util.GenerateClusterRole(manifest.ClusterlinkClusterRole, nil) if err != nil { - return fmt.Errorf("(rbac) kosmosctl join run warning, generate clusterrole failed: %s", err) + return fmt.Errorf("(cluster rbac) kosmosctl join run warning, generate clusterrole failed: %s", err) } _, err = o.Client.RbacV1().ClusterRoles().Create(context.TODO(), clusterRole, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("(rbac) kosmosctl join run warning, create clusterrole failed: %s", err) + return fmt.Errorf("(cluster rbac) kosmosctl join run warning, create clusterrole failed: %s", err) } clusterRoleBinding, err := util.GenerateClusterRoleBinding(manifest.ClusterlinkClusterRoleBinding, nil) if err != nil { - return fmt.Errorf("(rbac) kosmosctl join run warning, generate clusterrolebinding failed: %s", err) + return fmt.Errorf("(cluster rbac) kosmosctl join run warning, generate clusterrolebinding failed: %s", err) } _, err = o.Client.RbacV1().ClusterRoleBindings().Create(context.TODO(), clusterRoleBinding, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("(rbac) kosmosctl join run warning, create clusterrolebinding failed: %s", err) + return fmt.Errorf("(cluster rbac) kosmosctl join run warning, create clusterrolebinding failed: %s", err) } // 5. create operator in member - serviceAccount, err := util.GenerateServiceAccount(manifest.ClusterlinkServiceAccount, nil) + serviceAccount, err := util.GenerateServiceAccount(manifest.ClusterlinkOperatorServiceAccount, nil) if err != nil { - return fmt.Errorf("(operator) kosmosctl join run warning, generate serviceaccount failed: %s", err) + return fmt.Errorf("(cluster operator) kosmosctl join run warning, generate serviceaccount failed: %s", err) } _, err = o.Client.CoreV1().ServiceAccounts(serviceAccount.Namespace).Create(context.TODO(), serviceAccount, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("(operator) kosmosctl join run warning, create serviceaccount failed: %s", err) + return fmt.Errorf("(cluster operator) kosmosctl join run warning, create serviceaccount failed: %s", err) } - deployment, err := util.GenerateDeployment(manifest.ClusterlinkDeployment, manifest.ClusterlinkDeploymentReplace{ + + deployment, err := util.GenerateDeployment(manifest.ClusterlinkOperatorDeployment, manifest.ClusterlinkDeploymentReplace{ Version: version.GetReleaseVersion().PatchRelease(), - ClusterName: obj.GetName(), + ClusterName: o.Resource.GetName(), }) if err != nil { - return fmt.Errorf("(operator) kosmosctl join run warning, generate deployment failed: %s", err) + return fmt.Errorf("(cluster operator) kosmosctl join run warning, generate deployment failed: %s", err) } _, err = o.Client.AppsV1().Deployments(deployment.Namespace).Create(context.TODO(), deployment, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("(operator) kosmosctl join run warning, create deployment failed: %s", err) + return fmt.Errorf("(cluster operator) kosmosctl join run warning, create deployment failed: %s", err) + } + if err = util.WaitDeploymentReady(o.Client, deployment, 120); err != nil { + return fmt.Errorf("(cluster operator) kosmosctl join run warning, create deployment failed: %s", err) + } else { + klog.Info("Cluster registration successful.") + } + + return nil +} + +func (o *CommandJoinOptions) runKnode() error { + klog.Info("Start registering knode to kosmos control plane...") + _, err := o.DynamicClient.Resource(util.KnodeGVR).Namespace("").Create(context.TODO(), o.Resource, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("(knode) kosmosctl join run warning, create knode failed: %s", err) } + klog.Info("Knode registration successful.") return nil } diff --git a/pkg/kosmosctl/kosmosctl.go b/pkg/kosmosctl/kosmosctl.go index df2c84dbd..87b1d58cb 100644 --- a/pkg/kosmosctl/kosmosctl.go +++ b/pkg/kosmosctl/kosmosctl.go @@ -28,8 +28,8 @@ var DefaultConfigFlags = genericclioptions.NewConfigFlags(true).WithDeprecatedPa func NewKosmosCtlCommand() *cobra.Command { cmds := &cobra.Command{ Use: "kosmosctl", - Short: i18n.T("kosmosctl controls a Kubernetes cluster network"), - Long: templates.LongDesc(`kosmosctl controls a Kubernetes cluster network.`), + Short: i18n.T("kosmosctl controls the Kosmos cluster manager"), + Long: templates.LongDesc(`kosmosctl controls the Kosmos cluster manager.`), RunE: runHelp, } diff --git a/pkg/kosmosctl/manifest/manifest_clusterrolebindings.go b/pkg/kosmosctl/manifest/manifest_clusterrolebindings.go index 201e29e19..3724f93bf 100644 --- a/pkg/kosmosctl/manifest/manifest_clusterrolebindings.go +++ b/pkg/kosmosctl/manifest/manifest_clusterrolebindings.go @@ -42,25 +42,25 @@ roleRef: subjects: - kind: ServiceAccount name: clusterlink-controller-manager - namespace: clusterlink-system + namespace: {{ .Namespace }} - kind: ServiceAccount name: clusterlink-operator - namespace: clusterlink-system + namespace: {{ .Namespace }} ` - ClusterRouterKnodeClusterRoleBinding = ` + ClusterTreeKnodeManagerClusterRoleBinding = ` --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: clusterrouter-knode + name: clustertree-knode-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: clusterrouter-knode + name: clustertree-knode-manager subjects: - kind: ServiceAccount - name: clusterrouter-knode + name: clustertree-knode-manager namespace: {{ .Namespace }} ` ) diff --git a/pkg/kosmosctl/manifest/manifest_clusterroles.go b/pkg/kosmosctl/manifest/manifest_clusterroles.go index 886058b5c..00d6a74a9 100644 --- a/pkg/kosmosctl/manifest/manifest_clusterroles.go +++ b/pkg/kosmosctl/manifest/manifest_clusterroles.go @@ -40,12 +40,12 @@ rules: verbs: ["get"] ` - ClusterRouterKnodeClusterRole = ` + ClusterTreeKnodeManagerClusterRole = ` --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: clusterrouter-knode + name: clustertree-knode-manager rules: - apiGroups: ['*'] resources: ['*'] diff --git a/pkg/kosmosctl/manifest/manifest_crds.go b/pkg/kosmosctl/manifest/manifest_crds.go index 169c1e5f4..36c4fb17c 100644 --- a/pkg/kosmosctl/manifest/manifest_crds.go +++ b/pkg/kosmosctl/manifest/manifest_crds.go @@ -353,7 +353,7 @@ spec: status: {} ` - ClusterRouterKnode = `--- + ClusterTreeKnode = `--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: diff --git a/pkg/kosmosctl/manifest/manifest_deployments.go b/pkg/kosmosctl/manifest/manifest_deployments.go index d32e56b10..f8c949588 100644 --- a/pkg/kosmosctl/manifest/manifest_deployments.go +++ b/pkg/kosmosctl/manifest/manifest_deployments.go @@ -36,12 +36,12 @@ spec: memory: 500Mi ` - ClusterlinkDeployment = ` + ClusterlinkOperatorDeployment = ` apiVersion: apps/v1 kind: Deployment metadata: name: clusterlink-operator - namespace: clusterlink-system + namespace: {{ .Namespace }} labels: app: operator spec: @@ -99,31 +99,31 @@ spec: ` - ClusterRouterKnodeDeployment = `--- + ClusterTreeKnodeManagerDeployment = `--- apiVersion: apps/v1 kind: Deployment metadata: - name: knode + name: clustertree-knode-manager namespace: {{ .Namespace }} labels: - app: clusterrouter-controller-manager + app: clustertree-knode-manager spec: replicas: 1 selector: matchLabels: - app: knode + app: clustertree-knode-manager template: metadata: labels: - app: knode + app: clustertree-knode-manager spec: - serviceAccountName: clusterrouter-knode + serviceAccountName: clustertree-knode-manager containers: - name: manager - image: {{ .ImageRepository }}/knode:v{{ .Version }} + image: {{ .ImageRepository }}/clustertree-knode-manager:v{{ .Version }} imagePullPolicy: Always command: - - /clusterrouter + - clustertree-knode-manager - --kube-api-qps=500 - --kube-api-burst=1000 - --kubeconfig=/etc/kube/config @@ -136,8 +136,8 @@ spec: - configMap: defaultMode: 420 items: - - key: Kubeconfig - path: Kubeconfig + - key: kubeconfig + path: kubeconfig name: host-kubeconfig name: config-volume ` diff --git a/pkg/kosmosctl/manifest/manifest_serviceaccounts.go b/pkg/kosmosctl/manifest/manifest_serviceaccounts.go index b24dbe39c..de4ef986f 100644 --- a/pkg/kosmosctl/manifest/manifest_serviceaccounts.go +++ b/pkg/kosmosctl/manifest/manifest_serviceaccounts.go @@ -17,19 +17,19 @@ metadata: namespace: {{ .Namespace }} ` - ClusterlinkServiceAccount = ` + ClusterlinkOperatorServiceAccount = ` apiVersion: v1 kind: ServiceAccount metadata: name: clusterlink-operator - namespace: clusterlink-system + namespace: {{ .Namespace }} ` - ClusterRouterKnodeServiceAccount = ` + ClusterTreeKnodeManagerServiceAccount = ` apiVersion: v1 kind: ServiceAccount metadata: - name: clusterrouter-knode + name: clustertree-knode-manager namespace: {{ .Namespace }} ` ) diff --git a/pkg/kosmosctl/uninstall/uninstall.go b/pkg/kosmosctl/uninstall/uninstall.go index 9465c741c..3999a5c76 100644 --- a/pkg/kosmosctl/uninstall/uninstall.go +++ b/pkg/kosmosctl/uninstall/uninstall.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" ctlutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" @@ -14,9 +15,8 @@ import ( ) const ( - clusterlinkNetworkManager = "clusterlink-network-manager" - clusterrouterKnode = "knode" - clusterrouterKnodeResource = "clusterrouter-knode" + clusterlinkNetworkManager = "clusterlink-network-manager" + clustertreeKnodeManager = "clustertree-knode-manager" ) type CommandUninstallOptions struct { @@ -75,14 +75,15 @@ func (o *CommandUninstallOptions) Validate() error { } func (o *CommandUninstallOptions) Run() error { + klog.Info("Kosmos starts uninstalling.") switch o.Module { case "clusterlink": err := o.runClusterlink() if err != nil { return err } - case "clusterrouter": - err := o.runClusterrouter() + case "clustertree": + err := o.runClustertree() if err != nil { return err } @@ -91,7 +92,7 @@ func (o *CommandUninstallOptions) Run() error { if err != nil { return err } - err = o.runClusterrouter() + err = o.runClustertree() if err != nil { return err } @@ -105,6 +106,7 @@ func (o *CommandUninstallOptions) Run() error { } func (o *CommandUninstallOptions) runClusterlink() error { + klog.Info("Start uninstalling clusterlink from kosmos control plane...") err := o.Client.AppsV1().Deployments(util.DefaultNamespace).Delete(context.TODO(), clusterlinkNetworkManager, metav1.DeleteOptions{}) if err != nil { return fmt.Errorf("kosmosctl uninstall clusterlink run error, deployment options failed: %v", err) @@ -115,19 +117,22 @@ func (o *CommandUninstallOptions) runClusterlink() error { return fmt.Errorf("kosmosctl uninstall clusterlink run error, serviceaccount options failed: %v", err) } + klog.Info("Clusterlink was uninstalled.") return nil } -func (o *CommandUninstallOptions) runClusterrouter() error { - err := o.Client.AppsV1().Deployments(util.DefaultNamespace).Delete(context.TODO(), clusterrouterKnode, metav1.DeleteOptions{}) +func (o *CommandUninstallOptions) runClustertree() error { + klog.Info("Start uninstalling clustertree from kosmos control plane...") + err := o.Client.AppsV1().Deployments(util.DefaultNamespace).Delete(context.TODO(), clustertreeKnodeManager, metav1.DeleteOptions{}) if err != nil { return fmt.Errorf("kosmosctl uninstall clusterrouter run error, deployment options failed: %v", err) } - err = o.Client.CoreV1().ServiceAccounts(util.DefaultNamespace).Delete(context.TODO(), clusterrouterKnodeResource, metav1.DeleteOptions{}) + err = o.Client.CoreV1().ServiceAccounts(util.DefaultNamespace).Delete(context.TODO(), clustertreeKnodeManager, metav1.DeleteOptions{}) if err != nil { return fmt.Errorf("kosmosctl uninstall clusterrouter run error, serviceaccount options failed: %v", err) } + klog.Info("Clustertree was uninstalled.") return nil } diff --git a/pkg/kosmosctl/unjoin/unjoin.go b/pkg/kosmosctl/unjoin/unjoin.go index 1c7bafe62..eee0f8cef 100644 --- a/pkg/kosmosctl/unjoin/unjoin.go +++ b/pkg/kosmosctl/unjoin/unjoin.go @@ -8,29 +8,34 @@ import ( "github.com/spf13/cobra" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" ctlutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/i18n" "k8s.io/kubectl/pkg/util/templates" "github.com/kosmos.io/kosmos/pkg/kosmosctl/util" - "github.com/kosmos.io/kosmos/pkg/utils" ) const ( - ClusterlinkOperator = "clusterlink-operator" + clusterlinkOperator = "clusterlink-operator" ) var unjoinExample = templates.Examples(i18n.T(` - # unjoin member1-cluster in master control plane - kosmosctl unjoin member-cluster --cluster-kubeconfig=[member-kubeconfig] --master-kubeconfig=[master-kubeconfig] + # Unjoin cluster from Kosmos control plane in any cluster, e.g: + kosmosctl unjoin cluster [cluster-name] --cluster-kubeconfig=[member-kubeconfig] --master-kubeconfig=[master-kubeconfig] - # unjoin member1-cluster in current master control plane - kosmosctl unjoin member-cluster --cluster-kubeconfig=[member-kubeconfig] + # Unjoin cluster from Kosmos control plane in master cluster, e.g: + kosmosctl unjoin cluster [cluster-name] --cluster-kubeconfig=[member-kubeconfig] + + # Unjoin knode from Kosmos control plane in any cluster, e.g: + kosmosctl unjoin knode [knode-name] --master-kubeconfig=[master-kubeconfig] + + # Unjoin knode from Kosmos control plane in master cluster, e.g: + kosmosctl unjoin knode [knode-name] `)) type CommandUnJoinOptions struct { @@ -41,34 +46,34 @@ type CommandUnJoinOptions struct { DynamicClient *dynamic.DynamicClient } -// NewCmdUnJoin Delete this in Clusterlink control plane +// NewCmdUnJoin Delete resource in Kosmos control plane. func NewCmdUnJoin(f ctlutil.Factory) *cobra.Command { o := &CommandUnJoinOptions{} cmd := &cobra.Command{ Use: "unjoin", - Short: i18n.T("Unjoin this in clusterlink control plane"), + Short: i18n.T("Unjoin resource in kosmos control plane"), Long: "", Example: unjoinExample, SilenceUsage: true, DisableFlagsInUseLine: true, RunE: func(cmd *cobra.Command, args []string) error { - ctlutil.CheckErr(o.Complete(f, cmd, args)) - ctlutil.CheckErr(o.Validate()) - ctlutil.CheckErr(o.Run(f, cmd, args)) + ctlutil.CheckErr(o.Complete(f)) + ctlutil.CheckErr(o.Validate(args)) + ctlutil.CheckErr(o.Run(args)) return nil }, } - cmd.Flags().StringVarP(&o.MasterKubeConfig, "master-kubeconfig", "", "", "master-kubeconfig") - cmd.Flags().StringVarP(&o.ClusterKubeConfig, "cluster-kubeconfig", "", "", "cluster-kubeconfig") + cmd.Flags().StringVarP(&o.MasterKubeConfig, "master-kubeconfig", "", "", "Absolute path to the master kubeconfig file.") + cmd.Flags().StringVarP(&o.ClusterKubeConfig, "cluster-kubeconfig", "", "", "Absolute path to the cluster kubeconfig file.") err := cmd.MarkFlagRequired("cluster-kubeconfig") if err != nil { - fmt.Printf("kosmosctl join cmd error, MarkFlagRequired failed: %s", err) + fmt.Printf("kosmosctl unjoin cmd error, MarkFlagRequired failed: %s", err) } return cmd } -func (o *CommandUnJoinOptions) Complete(f ctlutil.Factory, cmd *cobra.Command, args []string) error { +func (o *CommandUnJoinOptions) Complete(f ctlutil.Factory) error { var masterConfig *restclient.Config var err error @@ -102,18 +107,51 @@ func (o *CommandUnJoinOptions) Complete(f ctlutil.Factory, cmd *cobra.Command, a return nil } -func (o *CommandUnJoinOptions) Validate() error { +func (o *CommandUnJoinOptions) Validate(args []string) error { + switch args[0] { + case "cluster": + _, err := o.DynamicClient.Resource(util.ClusterGVR).Get(context.TODO(), args[1], metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl unjoin validate warning, clsuter is not found: %s", err) + } + return fmt.Errorf("kosmosctl unjoin validate error, get cluster failed: %s", err) + } + case "knode": + _, err := o.DynamicClient.Resource(util.KnodeGVR).Get(context.TODO(), args[1], metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return fmt.Errorf("kosmosctl unjoin validate warning, knode is not found: %s", err) + } + return fmt.Errorf("kosmosctl unjoin validate error, get knode failed: %s", err) + } + } + + return nil +} + +func (o *CommandUnJoinOptions) Run(args []string) error { + switch args[0] { + case "cluster": + err := o.runCluster(args[1]) + if err != nil { + return err + } + case "knode": + err := o.runKnode(args[1]) + if err != nil { + return err + } + } + return nil } -func (o *CommandUnJoinOptions) Run(f ctlutil.Factory, cmd *cobra.Command, args []string) error { - //delete cluster +func (o *CommandUnJoinOptions) runCluster(clusterName string) error { + klog.Info("Start removing cluster from kosmos control plane...") + // 1. delete cluster for { - err := o.DynamicClient.Resource(schema.GroupVersionResource{ - Group: "kosmos.io", - Version: "v1alpha1", - Resource: "clusters", - }).Namespace("").Delete(context.TODO(), args[0], metav1.DeleteOptions{}) + err := o.DynamicClient.Resource(util.ClusterGVR).Namespace("").Delete(context.TODO(), clusterName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { break @@ -123,37 +161,55 @@ func (o *CommandUnJoinOptions) Run(f ctlutil.Factory, cmd *cobra.Command, args [ time.Sleep(3 * time.Second) } - // delete operator - err := o.Client.CoreV1().ServiceAccounts(util.DefaultNamespace).Delete(context.TODO(), ClusterlinkOperator, metav1.DeleteOptions{}) + // 2. delete operator + err := o.Client.CoreV1().ServiceAccounts(util.DefaultNamespace).Delete(context.TODO(), clusterlinkOperator, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("(operator) kosmosctl unjoin run error, delete serviceaccout failed: %s", err) } - err = o.Client.AppsV1().Deployments(util.DefaultNamespace).Delete(context.TODO(), ClusterlinkOperator, metav1.DeleteOptions{}) + err = o.Client.AppsV1().Deployments(util.DefaultNamespace).Delete(context.TODO(), clusterlinkOperator, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("(operator) kosmosctl unjoin run error, delete deployment failed: %s", err) } - //delete secret - err = o.Client.CoreV1().Secrets(util.DefaultNamespace).Delete(context.TODO(), utils.ControlPanelSecretName, metav1.DeleteOptions{}) + // 3. delete secret + err = o.Client.CoreV1().Secrets(util.DefaultNamespace).Delete(context.TODO(), util.ControlPanelSecretName, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("(secret) kosmosctl unjoin run error, delete secret failed: %s", err) } - //delete namespace - err = o.Client.CoreV1().Namespaces().Delete(context.TODO(), util.DefaultNamespace, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("(namespace) kosmosctl unjoin run error, delete namespace failed: %s", err) - } - - //delete rbac - err = o.Client.RbacV1().ClusterRoleBindings().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) + // 4. delete rbac + err = o.Client.RbacV1().ClusterRoleBindings().Delete(context.TODO(), util.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("(rbac) kosmosctl unjoin run error, delete clusterrolebinding failed: %s", err) } - err = o.Client.RbacV1().ClusterRoles().Delete(context.TODO(), utils.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) + err = o.Client.RbacV1().ClusterRoles().Delete(context.TODO(), util.ExternalIPPoolNamePrefix, metav1.DeleteOptions{}) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("(rbac) kosmosctl unjoin run error, delete clusterrole failed: %s", err) } + // 5. delete namespace + err = o.Client.CoreV1().Namespaces().Delete(context.TODO(), util.DefaultNamespace, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("(namespace) kosmosctl unjoin run error, delete namespace failed: %s", err) + } + + klog.Info("Cluster [" + clusterName + "] was removed.") + return nil +} + +func (o *CommandUnJoinOptions) runKnode(knodeName string) error { + klog.Info("Start removing knode from kosmos control plane...") + for { + err := o.DynamicClient.Resource(util.KnodeGVR).Namespace("").Delete(context.TODO(), knodeName, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + break + } + return fmt.Errorf("(knode) kosmosctl unjoin run error, delete knode failed: %s", err) + } + time.Sleep(3 * time.Second) + } + + klog.Info("Knode [" + knodeName + "] was removed.") return nil } diff --git a/pkg/kosmosctl/util/builder.go b/pkg/kosmosctl/util/builder.go index 60b2c96ab..bb0122ca6 100644 --- a/pkg/kosmosctl/util/builder.go +++ b/pkg/kosmosctl/util/builder.go @@ -10,6 +10,7 @@ import ( rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" ) @@ -17,6 +18,14 @@ const ( DefaultNamespace = "kosmos-system" DefaultImageRepository = "ghcr.io/kosmos-io" DefaultInstallModule = "all" + + ExternalIPPoolNamePrefix = "clusterlink" + ControlPanelSecretName = "controlpanel-config" +) + +var ( + ClusterGVR = schema.GroupVersionResource{Group: "kosmos.io", Version: "v1alpha1", Resource: "clusters"} + KnodeGVR = schema.GroupVersionResource{Group: "kosmos.io", Version: "v1alpha1", Resource: "knodes"} ) func GenerateDeployment(deployTemplate string, obj interface{}) (*appsv1.Deployment, error) { diff --git a/pkg/kosmosctl/util/check.go b/pkg/kosmosctl/util/check.go index b8b4fd328..42354bc5e 100644 --- a/pkg/kosmosctl/util/check.go +++ b/pkg/kosmosctl/util/check.go @@ -7,6 +7,7 @@ import ( "strings" "time" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -89,6 +90,41 @@ func isPodReady(c kubernetes.Interface, n, p string) wait.ConditionFunc { } } +// WaitDeploymentReady wait deployment ready or timeout. +func WaitDeploymentReady(c kubernetes.Interface, d *appsv1.Deployment, timeoutSeconds int) error { + var lastErr error + + pollError := wait.PollImmediate(time.Second, time.Duration(timeoutSeconds)*time.Second, func() (bool, error) { + deploy, err := c.AppsV1().Deployments(d.GetNamespace()).Get(context.TODO(), d.GetName(), metav1.GetOptions{}) + if err != nil { + lastErr = err + return false, nil + } + if deploy.Generation != deploy.Status.ObservedGeneration { + lastErr = fmt.Errorf("current generation %d, observed generation %d", + deploy.Generation, deploy.Status.ObservedGeneration) + return false, nil + } + if (deploy.Spec.Replicas != nil) && (deploy.Status.UpdatedReplicas < *d.Spec.Replicas) { + lastErr = fmt.Errorf("the number of pods targeted by the deployment (%d pods) is different "+ + "from the number of pods targeted by the deployment that have the desired template spec (%d pods)", + *deploy.Spec.Replicas, deploy.Status.UpdatedReplicas) + return false, nil + } + if deploy.Status.AvailableReplicas < deploy.Status.UpdatedReplicas { + lastErr = fmt.Errorf("expected %d replicas, got %d available replicas", + deploy.Status.UpdatedReplicas, deploy.Status.AvailableReplicas) + return false, nil + } + return true, nil + }) + if pollError != nil { + return fmt.Errorf("wait for Deployment(%s/%s) ready: %v: %v", d.GetNamespace(), d.GetName(), pollError, lastErr) + } + + return nil +} + // MapToString labels to string. func MapToString(labels map[string]string) string { v := new(bytes.Buffer) @@ -100,3 +136,20 @@ func MapToString(labels map[string]string) string { } return strings.TrimRight(v.String(), ",") } + +func CheckInstall(modules string) { + fmt.Printf(` +-------------------------------------------------------------------------------------- + █████ ████ ███████ █████████ ██████ ██████ ███████ █████████ +░░███ ███░ ███░░░░░███ ███░░░░░███░░██████ ██████ ███░░░░░███ ███░░░░░███ + ░███ ███ ███ ░░███░███ ░░░ ░███░█████░███ ███ ░░███░███ ░░░ + ░███████ ░███ ░███░░█████████ ░███░░███ ░███ ░███ ░███░░█████████ + ░███░░███ ░███ ░███ ░░░░░░░░███ ░███ ░░░ ░███ ░███ ░███ ░░░░░░░░███ + ░███ ░░███ ░░███ ███ ███ ░███ ░███ ░███ ░░███ ███ ███ ░███ + █████ ░░████ ░░░███████░ ░░█████████ █████ █████ ░░░███████░ ░░█████████ +░░░░░ ░░░░ ░░░░░░░ ░░░░░░░░░ ░░░░░ ░░░░░ ░░░░░░░ ░░░░░░░░░ +--------------------------------------------------------------------------------------- +Kosmos has been installed successfully. The module %[1]s is installed. + +`, modules) +}