Skip to content

Commit db81e54

Browse files
ONE7liveONE7live
committed
fix: update kosmosctl to support new version
Signed-off-by: ONE7live <[email protected]>
1 parent ff2019f commit db81e54

11 files changed

+717
-209
lines changed

‎pkg/kosmosctl/floater/analysis.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,8 @@ type PrintAnalysisData struct {
5656
}
5757

5858
func NewCmdAnalysis(f ctlutil.Factory) *cobra.Command {
59-
o := &CommandAnalysisOptions{
60-
Version: version.GetReleaseVersion().PatchRelease(),
61-
}
59+
o := &CommandAnalysisOptions{}
60+
6261
cmd := &cobra.Command{
6362
Use: "analysis",
6463
Short: i18n.T("Analysis network connectivity between Kosmos clusters"),
@@ -98,6 +97,10 @@ func (o *CommandAnalysisOptions) Complete(f ctlutil.Factory) error {
9897
return fmt.Errorf("kosmosctl analysis complete error, generate dynamic client failed: %s", err)
9998
}
10099

100+
if len(o.Version) == 0 {
101+
o.Version = version.GetReleaseVersion().PatchRelease()
102+
}
103+
101104
af := NewAnalysisFloater(o)
102105
if err = af.completeFromKubeConfigPath(o.KubeConfig, o.Context); err != nil {
103106
return err

‎pkg/kosmosctl/floater/check.go

+66-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package floater
22

33
import (
4+
"context"
45
"fmt"
6+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
7+
"k8s.io/client-go/kubernetes"
58
"os"
69
"strconv"
710

@@ -59,9 +62,8 @@ type PrintCheckData struct {
5962
}
6063

6164
func NewCmdCheck() *cobra.Command {
62-
o := &CommandCheckOptions{
63-
Version: version.GetReleaseVersion().PatchRelease(),
64-
}
65+
o := &CommandCheckOptions{}
66+
6567
cmd := &cobra.Command{
6668
Use: "check",
6769
Short: i18n.T("Check network connectivity between Kosmos clusters"),
@@ -89,6 +91,7 @@ func NewCmdCheck() *cobra.Command {
8991
flags.StringVarP(&o.Namespace, "namespace", "n", utils.DefaultNamespace, "Kosmos namespace.")
9092
flags.StringVarP(&o.ImageRepository, "image-repository", "r", utils.DefaultImageRepository, "Image repository.")
9193
flags.StringVarP(&o.DstImageRepository, "dst-image-repository", "", "", "Destination cluster image repository.")
94+
flags.StringVar(&o.Version, "version", "", "image version for pull images")
9295
flags.StringVar(&o.KubeConfig, "kubeconfig", "", "Absolute path to the host kubeconfig file.")
9396
flags.StringVar(&o.Context, "context", "", "The name of the kubeconfig context.")
9497
flags.StringVar(&o.SrcKubeConfig, "src-kubeconfig", "", "Absolute path to the source cluster kubeconfig file.")
@@ -104,6 +107,10 @@ func NewCmdCheck() *cobra.Command {
104107
}
105108

106109
func (o *CommandCheckOptions) Complete() error {
110+
if len(o.Version) == 0 {
111+
o.Version = version.GetReleaseVersion().PatchRelease()
112+
}
113+
107114
if len(o.DstImageRepository) == 0 {
108115
o.DstImageRepository = o.ImageRepository
109116
}
@@ -127,7 +134,62 @@ func (o *CommandCheckOptions) Complete() error {
127134

128135
func (o *CommandCheckOptions) Validate() error {
129136
if len(o.Namespace) == 0 {
130-
return fmt.Errorf("namespace must be specified")
137+
return fmt.Errorf("kosmosctl check validate error, namespace must be specified")
138+
}
139+
140+
nodeRoleLabel := utils.NodeRoleLabel + "=" + utils.NodeRoleValue
141+
srcConfig, err := utils.RestConfig(o.SrcKubeConfig, o.Context)
142+
if err != nil {
143+
return fmt.Errorf("kosmosctl check validate error, exception in kubeconfig verification of source cluster")
144+
}
145+
srcK8sClient, err := kubernetes.NewForConfig(srcConfig)
146+
if err != nil {
147+
return fmt.Errorf("kosmosctl check validate error, generate source K8s basic client failed: %v", err)
148+
}
149+
srcNodes, err := srcK8sClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
150+
LabelSelector: nodeRoleLabel,
151+
})
152+
for _, node := range srcNodes.Items {
153+
labels := node.Labels
154+
if value, exists := labels[utils.KosmosExcludeNodeLabel]; !exists || value != utils.KosmosExcludeNodeValue {
155+
if node.Labels == nil {
156+
node.Labels = make(map[string]string)
157+
}
158+
node.Labels[utils.KosmosExcludeNodeLabel] = utils.KosmosExcludeNodeValue
159+
160+
_, err = srcK8sClient.CoreV1().Nodes().Update(context.TODO(), &node, metav1.UpdateOptions{})
161+
if err != nil {
162+
return fmt.Errorf("kosmosctl check validate error, source cluster agent node label update exception: %v", err)
163+
}
164+
}
165+
}
166+
167+
if o.DstFloater != nil {
168+
dstConfig, err := utils.RestConfig(o.DstKubeConfig, o.Context)
169+
if err != nil {
170+
return fmt.Errorf("kosmosctl check verify error, kubeconfig verification of the target cluster failed")
171+
}
172+
dstK8sClient, err := kubernetes.NewForConfig(dstConfig)
173+
if err != nil {
174+
return fmt.Errorf("kosmosctl check validate error, generate target K8s basic client failed: %v", err)
175+
}
176+
dstNodes, err := dstK8sClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
177+
LabelSelector: nodeRoleLabel,
178+
})
179+
for _, node := range dstNodes.Items {
180+
labels := node.Labels
181+
if value, exists := labels[utils.KosmosExcludeNodeLabel]; !exists || value != utils.KosmosExcludeNodeValue {
182+
if node.Labels == nil {
183+
node.Labels = make(map[string]string)
184+
}
185+
node.Labels[utils.KosmosExcludeNodeLabel] = utils.KosmosExcludeNodeValue
186+
187+
_, err = dstK8sClient.CoreV1().Nodes().Update(context.TODO(), &node, metav1.UpdateOptions{})
188+
if err != nil {
189+
return fmt.Errorf("kosmosctl check validate error, target cluster agent node label update exception: %v", err)
190+
}
191+
}
192+
}
131193
}
132194

133195
return nil

‎pkg/kosmosctl/install/install.go

+32-28
Original file line numberDiff line numberDiff line change
@@ -292,11 +292,11 @@ func (o *CommandInstallOptions) runClusterlink() error {
292292
if err != nil {
293293
return err
294294
}
295-
clusterlinkClusterNode, err := util.GenerateCustomResourceDefinition(manifest.ClusterNode, nil)
295+
clusterlinkClusterNode, err := util.GenerateCustomResourceDefinition(manifest.ClusterlinkClusterNode, nil)
296296
if err != nil {
297297
return err
298298
}
299-
clusterlinkNodeConfig, err := util.GenerateCustomResourceDefinition(manifest.NodeConfig, nil)
299+
clusterlinkNodeConfig, err := util.GenerateCustomResourceDefinition(manifest.ClusterlinkNodeConfig, nil)
300300
if err != nil {
301301
return err
302302
}
@@ -334,7 +334,7 @@ func (o *CommandInstallOptions) runClusterlink() error {
334334
}
335335
klog.Info("Deployment " + networkManagerDeploy.Name + " has been created.")
336336

337-
operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{
337+
operatorDeploy, err := util.GenerateDeployment(manifest.ClusterlinkOperatorDeployment, manifest.DeploymentReplace{
338338
Namespace: o.Namespace,
339339
Version: o.Version,
340340
UseProxy: o.UseProxy,
@@ -489,27 +489,6 @@ func (o *CommandInstallOptions) runClustertree() error {
489489
}
490490
klog.Info("Deployment clustertree-cluster-manager has been created.")
491491

492-
operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{
493-
Namespace: o.Namespace,
494-
Version: o.Version,
495-
UseProxy: o.UseProxy,
496-
ImageRepository: o.ImageRegistry,
497-
})
498-
if err != nil {
499-
return fmt.Errorf("kosmosctl install operator run error, operator generate deployment failed: %s", err)
500-
}
501-
_, err = o.K8sClient.AppsV1().Deployments(operatorDeploy.Namespace).Get(context.TODO(), operatorDeploy.Name, metav1.GetOptions{})
502-
if err != nil {
503-
if apierrors.IsNotFound(err) {
504-
err = o.createOperator()
505-
if err != nil {
506-
return err
507-
}
508-
} else {
509-
return fmt.Errorf("kosmosctl install operator run error, operator get deployment failed: %s", err)
510-
}
511-
}
512-
513492
return nil
514493
}
515494

@@ -583,7 +562,7 @@ func (o *CommandInstallOptions) runScheduler() error {
583562
}
584563
klog.Infof("ConfigMap %s has been created.", scheduleConfigFile.Name)
585564

586-
hostkubeConfigMap := &corev1.ConfigMap{
565+
hostKubeConfigMap := &corev1.ConfigMap{
587566
ObjectMeta: metav1.ObjectMeta{
588567
Name: utils.HostKubeConfigName,
589568
Namespace: o.Namespace,
@@ -597,14 +576,39 @@ func (o *CommandInstallOptions) runScheduler() error {
597576
}()),
598577
},
599578
}
600-
_, err = o.K8sClient.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), hostkubeConfigMap, metav1.CreateOptions{})
579+
_, err = o.K8sClient.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), hostKubeConfigMap, metav1.CreateOptions{})
601580
if err != nil {
602581
if !apierrors.IsAlreadyExists(err) {
603582
return fmt.Errorf("kosmosctl install scheduler run error, configmap options failed: %v", err)
604583
}
605584
}
606585
klog.Info("ConfigMap host-kubeconfig has been created.")
607586

587+
klog.Info("Attempting to create Kosmos-Scheduler CRDs...")
588+
crds := apiextensionsv1.CustomResourceDefinitionList{}
589+
schedulerCDP, err := util.GenerateCustomResourceDefinition(manifest.SchedulerClusterDistributionPolicies, manifest.CRDReplace{
590+
Namespace: o.Namespace,
591+
})
592+
if err != nil {
593+
return err
594+
}
595+
schedulerDP, err := util.GenerateCustomResourceDefinition(manifest.SchedulerDistributionPolicies, nil)
596+
if err != nil {
597+
return err
598+
}
599+
crds.Items = append(crds.Items, *schedulerCDP, *schedulerDP)
600+
for i := range crds.Items {
601+
_, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), &crds.Items[i], metav1.CreateOptions{})
602+
if err != nil {
603+
if apierrors.IsAlreadyExists(err) {
604+
klog.Warningf("CRD %v is existed, creation process will skip", &crds.Items[i].Name)
605+
continue
606+
}
607+
return fmt.Errorf("kosmosctl install scheduler run error, crd options failed: %v", err)
608+
}
609+
klog.Info("Create CRD " + crds.Items[i].Name + " successful.")
610+
}
611+
608612
klog.Info("Start creating kosmos-scheduler Deployment...")
609613
schedulerDeploy, err := util.GenerateDeployment(manifest.SchedulerDeployment, manifest.DeploymentReplace{
610614
Namespace: o.Namespace,
@@ -631,7 +635,7 @@ func (o *CommandInstallOptions) runScheduler() error {
631635

632636
func (o *CommandInstallOptions) createOperator() error {
633637
klog.Info("Start creating ClusterLink-Operator...")
634-
operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{
638+
operatorDeploy, err := util.GenerateDeployment(manifest.ClusterlinkOperatorDeployment, manifest.DeploymentReplace{
635639
Namespace: o.Namespace,
636640
Version: o.Version,
637641
UseProxy: o.UseProxy,
@@ -685,7 +689,7 @@ func (o *CommandInstallOptions) createOperator() error {
685689
return fmt.Errorf("kosmosctl install operator run error, operator options clusterrolebinding failed: %s", err)
686690
}
687691

688-
operatorSA, err := util.GenerateServiceAccount(manifest.KosmosOperatorServiceAccount, manifest.ServiceAccountReplace{
692+
operatorSA, err := util.GenerateServiceAccount(manifest.ClusterlinkOperatorServiceAccount, manifest.ServiceAccountReplace{
689693
Namespace: o.Namespace,
690694
})
691695
if err != nil {

‎pkg/kosmosctl/join/join.go

+20-45
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,13 @@ func (o *CommandJoinOptions) CreateTreeRelatedCRDs() error {
301301

302302
func (o *CommandJoinOptions) runCluster() error {
303303
klog.Info("Start registering cluster to kosmos control plane...")
304+
// create ns if it does not exist
305+
kosmosNS := &corev1.Namespace{}
306+
kosmosNS.Name = o.Namespace
307+
_, err := o.K8sClient.CoreV1().Namespaces().Create(context.TODO(), kosmosNS, metav1.CreateOptions{})
308+
if err != nil && !apierrors.IsAlreadyExists(err) {
309+
return fmt.Errorf("kosmosctl join run error, create namespace failed: %s", err)
310+
}
304311

305312
// create cluster in control panel
306313
cluster := v1alpha1.Cluster{
@@ -354,38 +361,26 @@ func (o *CommandJoinOptions) runCluster() error {
354361

355362
cluster.Spec.ClusterLinkOptions.DefaultNICName = o.DefaultNICName
356363
cluster.Spec.ClusterLinkOptions.CNI = o.CNI
357-
}
358-
359-
if o.EnableTree {
360-
// create ClusterTree related crds
361-
err := o.CreateTreeRelatedCRDs()
362-
if err != nil {
363-
return err
364-
}
365364

366-
clusterPodConvert, err := util.GenerateCustomResourceDefinition(manifest.ClusterPodConvert, nil)
365+
clusterlinkOperatorSA, err := util.GenerateServiceAccount(manifest.ClusterlinkOperatorServiceAccount, manifest.ServiceAccountReplace{
366+
Namespace: o.Namespace,
367+
})
367368
if err != nil {
368-
return err
369+
return fmt.Errorf("kosmosctl join run error, generate serviceaccount failed: %s", err)
369370
}
370-
_, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), clusterPodConvert, metav1.CreateOptions{})
371-
if err != nil {
372-
if !apierrors.IsAlreadyExists(err) {
373-
return fmt.Errorf("kosmosctl join run error, crd options failed: %v", err)
374-
}
371+
_, err = o.K8sClient.CoreV1().ServiceAccounts(clusterlinkOperatorSA.Namespace).Create(context.TODO(), clusterlinkOperatorSA, metav1.CreateOptions{})
372+
if err != nil && !apierrors.IsAlreadyExists(err) {
373+
return fmt.Errorf("kosmosctl join run error, create serviceaccount failed: %s", err)
375374
}
376-
klog.Info("Create CRD " + clusterPodConvert.Name + " successful.")
375+
klog.Info("ServiceAccount " + clusterlinkOperatorSA.Name + " has been created.")
376+
}
377377

378-
podConvert, err := util.GenerateCustomResourceDefinition(manifest.PodConvert, nil)
378+
if o.EnableTree {
379+
// create ClusterTree related crds
380+
err := o.CreateTreeRelatedCRDs()
379381
if err != nil {
380382
return err
381383
}
382-
_, err = o.K8sExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), podConvert, metav1.CreateOptions{})
383-
if err != nil {
384-
if !apierrors.IsAlreadyExists(err) {
385-
return fmt.Errorf("kosmosctl join run error, crd options failed: %v", err)
386-
}
387-
}
388-
klog.Info("Create CRD " + podConvert.Name + " successful.")
389384

390385
if len(o.LeafModel) > 0 {
391386
switch o.LeafModel {
@@ -425,20 +420,12 @@ func (o *CommandJoinOptions) runCluster() error {
425420
}
426421
}
427422

428-
_, err := o.KosmosClient.KosmosV1alpha1().Clusters().Create(context.TODO(), &cluster, metav1.CreateOptions{})
423+
_, err = o.KosmosClient.KosmosV1alpha1().Clusters().Create(context.TODO(), &cluster, metav1.CreateOptions{})
429424
if err != nil {
430425
return fmt.Errorf("kosmosctl join run error, create cluster failed: %s", err)
431426
}
432427
klog.Info("Cluster " + o.Name + " has been created.")
433428

434-
// create ns if it does not exist
435-
kosmosNS := &corev1.Namespace{}
436-
kosmosNS.Name = o.Namespace
437-
_, err = o.K8sClient.CoreV1().Namespaces().Create(context.TODO(), kosmosNS, metav1.CreateOptions{})
438-
if err != nil && !apierrors.IsAlreadyExists(err) {
439-
return fmt.Errorf("kosmosctl join run error, create namespace failed: %s", err)
440-
}
441-
442429
// create rbac
443430
kosmosControlSA, err := util.GenerateServiceAccount(manifest.KosmosControlServiceAccount, manifest.ServiceAccountReplace{
444431
Namespace: o.Namespace,
@@ -490,18 +477,6 @@ func (o *CommandJoinOptions) runCluster() error {
490477
}
491478
klog.Info("ClusterRoleBinding " + kosmosCRB.Name + " has been created.")
492479

493-
kosmosOperatorSA, err := util.GenerateServiceAccount(manifest.KosmosOperatorServiceAccount, manifest.ServiceAccountReplace{
494-
Namespace: o.Namespace,
495-
})
496-
if err != nil {
497-
return fmt.Errorf("kosmosctl join run error, generate serviceaccount failed: %s", err)
498-
}
499-
_, err = o.K8sClient.CoreV1().ServiceAccounts(kosmosOperatorSA.Namespace).Create(context.TODO(), kosmosOperatorSA, metav1.CreateOptions{})
500-
if err != nil && !apierrors.IsAlreadyExists(err) {
501-
return fmt.Errorf("kosmosctl join run error, create serviceaccount failed: %s", err)
502-
}
503-
klog.Info("ServiceAccount " + kosmosOperatorSA.Name + " has been created.")
504-
505480
// ToDo Wait for all services to be running
506481

507482
klog.Info("Cluster [" + o.Name + "] registration successful.")

‎pkg/kosmosctl/manifest/manifest_configmaps.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,10 @@ data:
6767
disabled:
6868
- name: "VolumeBinding"
6969
- name: "TaintToleration"
70+
- name: "LeafNodeDistribution"
7071
enabled:
7172
- name: "LeafNodeTaintToleration"
7273
- name: "LeafNodeVolumeBinding"
73-
- name: "LeafNodeDistribution"
7474
score:
7575
disabled:
7676
- name: "VolumeBinding"

0 commit comments

Comments
 (0)