Added test (virtual cluster creation, with pod) and small kubeconfig …

…refactor (#211)

* added virtual cluster and pod test

* moved ClusterCreate

* match patch k8s host version

cli/cmds/cluster/create.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/rancher/k3k/cli/cmds"
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
-	"github.com/rancher/k3k/pkg/controller"
 	k3kcluster "github.com/rancher/k3k/pkg/controller/cluster"
 	"github.com/rancher/k3k/pkg/controller/cluster/server"
 	"github.com/rancher/k3k/pkg/controller/kubeconfig"
@@ -22,9 +21,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/authentication/user"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -190,11 +189,6 @@ func create(clx *cli.Context) error {
 	}
 
 	logrus.Infof("Extracting Kubeconfig for [%s] cluster", name)
-	cfg := &kubeconfig.KubeConfig{
-		CN:         controller.AdminCommonName,
-		ORG:        []string{user.SystemPrivilegedGroup},
-		ExpiryDate: 0,
-	}
 
 	logrus.Infof("waiting for cluster to be available..")
 
@@ -205,7 +199,9 @@ func create(clx *cli.Context) error {
 		Steps:    25,
 	}
 
-	var kubeconfig []byte
+	cfg := kubeconfig.New()
+
+	var kubeconfig *clientcmdapi.Config
 	if err := retry.OnError(availableBackoff, apierrors.IsNotFound, func() error {
 		kubeconfig, err = cfg.Extract(ctx, ctrlClient, cluster, host[0])
 		return err
@@ -224,7 +220,12 @@ func create(clx *cli.Context) error {
 	kubectl cluster-info
 	`, filepath.Join(pwd, cluster.Name+"-kubeconfig.yaml"))
 
-	return os.WriteFile(cluster.Name+"-kubeconfig.yaml", kubeconfig, 0644)
+	kubeconfigData, err := clientcmd.Write(*kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(cluster.Name+"-kubeconfig.yaml", kubeconfigData, 0644)
 }
 
 func validateCreateFlags() error {

cli/cmds/kubeconfig/kubeconfig.go

@@ -21,6 +21,7 @@ import (
 	"k8s.io/apiserver/pkg/authentication/user"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/util/retry"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -142,23 +143,24 @@ func generate(clx *cli.Context) error {
 	if orgs == nil {
 		orgs = []string{user.SystemPrivilegedGroup}
 	}
+
 	cfg := kubeconfig.KubeConfig{
 		CN:         cn,
 		ORG:        orgs,
 		ExpiryDate: time.Hour * 24 * time.Duration(expirationDays),
 		AltNames:   certAltNames,
 	}
+
 	logrus.Infof("waiting for cluster to be available..")
-	var kubeconfig []byte
+
+	var kubeconfig *clientcmdapi.Config
 	if err := retry.OnError(controller.Backoff, apierrors.IsNotFound, func() error {
 		kubeconfig, err = cfg.Extract(ctx, ctrlClient, &cluster, host[0])
-		if err != nil {
-			return err
-		}
-		return nil
+		return err
 	}); err != nil {
 		return err
 	}
+
 	pwd, err := os.Getwd()
 	if err != nil {
 		return err
@@ -174,5 +176,10 @@ func generate(clx *cli.Context) error {
 	kubectl cluster-info
 	`, filepath.Join(pwd, configName))
 
-	return os.WriteFile(configName, kubeconfig, 0644)
+	kubeconfigData, err := clientcmd.Write(*kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(configName, kubeconfigData, 0644)
 }

pkg/controller/cluster/cluster.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"strings"
 	"time"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
@@ -98,7 +99,8 @@ func (c *ClusterReconciler) Reconcile(ctx context.Context, req reconcile.Request
 		}
 
 		// update Status HostVersion
-		cluster.Status.HostVersion = fmt.Sprintf("v%s.%s.0-k3s1", hostVersion.Major, hostVersion.Minor)
+		k8sVersion := strings.Split(hostVersion.GitVersion, "+")[0]
+		cluster.Status.HostVersion = k8sVersion + "-k3s1"
 		if err := c.Client.Status().Update(ctx, &cluster); err != nil {
 			return reconcile.Result{}, err
 		}

pkg/controller/cluster/cluster_test.go

@@ -51,7 +51,7 @@ var _ = Describe("Cluster Controller", func() {
 
 				serverVersion, err := k8s.DiscoveryClient.ServerVersion()
 				Expect(err).To(Not(HaveOccurred()))
-				expectedHostVersion := fmt.Sprintf("v%s.%s.0-k3s1", serverVersion.Major, serverVersion.Minor)
+				expectedHostVersion := fmt.Sprintf("%s-k3s1", serverVersion.GitVersion)
 
 				Eventually(func() string {
 					err := k8sClient.Get(ctx, client.ObjectKeyFromObject(cluster), cluster)

pkg/controller/cluster/server/bootstrap/bootstrap.go

@@ -5,14 +5,17 @@ import (
 	"crypto/tls"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"net/http"
 	"time"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type ControlRuntimeBootstrap struct {
@@ -171,3 +174,24 @@ func DecodedBootstrap(token, ip string) (*ControlRuntimeBootstrap, error) {
 
 	return bootstrap, nil
 }
+
+func GetFromSecret(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster) (*ControlRuntimeBootstrap, error) {
+	key := types.NamespacedName{
+		Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
+		Namespace: cluster.Namespace,
+	}
+
+	var bootstrapSecret v1.Secret
+	if err := client.Get(ctx, key, &bootstrapSecret); err != nil {
+		return nil, err
+	}
+
+	bootstrapData := bootstrapSecret.Data["bootstrap"]
+	if bootstrapData == nil {
+		return nil, errors.New("empty bootstrap")
+	}
+
+	var bootstrap ControlRuntimeBootstrap
+	err := json.Unmarshal(bootstrapData, &bootstrap)
+	return &bootstrap, err
+}

pkg/controller/kubeconfig/kubeconfig.go

@@ -3,8 +3,6 @@ package kubeconfig
 import (
 	"context"
 	"crypto/x509"
-	"encoding/json"
-	"errors"
 	"fmt"
 	"time"
 
@@ -16,7 +14,7 @@ import (
 	"github.com/rancher/k3k/pkg/controller/cluster/server/bootstrap"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/apiserver/pkg/authentication/user"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -28,60 +26,45 @@ type KubeConfig struct {
 	ExpiryDate time.Duration
 }
 
-func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) ([]byte, error) {
-	nn := types.NamespacedName{
-		Name:      controller.SafeConcatNameWithPrefix(cluster.Name, "bootstrap"),
-		Namespace: cluster.Namespace,
-	}
-
-	var bootstrapSecret v1.Secret
-	if err := client.Get(ctx, nn, &bootstrapSecret); err != nil {
-		return nil, err
-	}
-
-	bootstrapData := bootstrapSecret.Data["bootstrap"]
-	if bootstrapData == nil {
-		return nil, errors.New("empty bootstrap")
+func New() *KubeConfig {
+	return &KubeConfig{
+		CN:         controller.AdminCommonName,
+		ORG:        []string{user.SystemPrivilegedGroup},
+		ExpiryDate: 0,
 	}
+}
 
-	var bootstrap bootstrap.ControlRuntimeBootstrap
-	if err := json.Unmarshal(bootstrapData, &bootstrap); err != nil {
+func (k *KubeConfig) Extract(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) (*clientcmdapi.Config, error) {
+	bootstrapData, err := bootstrap.GetFromSecret(ctx, client, cluster)
+	if err != nil {
 		return nil, err
 	}
+	serverCACert := []byte(bootstrapData.ServerCA.Content)
 
 	adminCert, adminKey, err := certs.CreateClientCertKey(
-		k.CN, k.ORG,
-		&k.AltNames, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, k.ExpiryDate,
-		bootstrap.ClientCA.Content,
-		bootstrap.ClientCAKey.Content)
+		k.CN,
+		k.ORG,
+		&k.AltNames,
+		[]x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		k.ExpiryDate,
+		bootstrapData.ClientCA.Content,
+		bootstrapData.ClientCAKey.Content,
+	)
 	if err != nil {
 		return nil, err
 	}
-	// get the server service to extract the right IP
-	nn = types.NamespacedName{
-		Name:      server.ServiceName(cluster.Name),
-		Namespace: cluster.Namespace,
-	}
-
-	var k3kService v1.Service
-	if err := client.Get(ctx, nn, &k3kService); err != nil {
-		return nil, err
-	}
 
-	url := fmt.Sprintf("https://%s:%d", k3kService.Spec.ClusterIP, server.ServerPort)
-	if k3kService.Spec.Type == v1.ServiceTypeNodePort {
-		nodePort := k3kService.Spec.Ports[0].NodePort
-		url = fmt.Sprintf("https://%s:%d", hostServerIP, nodePort)
-	}
-	kubeconfigData, err := kubeconfig(url, []byte(bootstrap.ServerCA.Content), adminCert, adminKey)
+	url, err := getURLFromService(ctx, client, cluster, hostServerIP)
 	if err != nil {
 		return nil, err
 	}
 
-	return kubeconfigData, nil
+	config := NewConfig(url, serverCACert, adminCert, adminKey)
+
+	return config, nil
 }
 
-func kubeconfig(url string, serverCA, clientCert, clientKey []byte) ([]byte, error) {
+func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi.Config {
 	config := clientcmdapi.NewConfig()
 
 	cluster := clientcmdapi.NewCluster()
@@ -101,10 +84,27 @@ func kubeconfig(url string, serverCA, clientCert, clientKey []byte) ([]byte, err
 	config.Contexts["default"] = context
 	config.CurrentContext = "default"
 
-	kubeconfig, err := clientcmd.Write(*config)
-	if err != nil {
-		return nil, err
+	return config
+}
+
+func getURLFromService(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, hostServerIP string) (string, error) {
+	// get the server service to extract the right IP
+	key := types.NamespacedName{
+		Name:      server.ServiceName(cluster.Name),
+		Namespace: cluster.Namespace,
+	}
+
+	var k3kService v1.Service
+	if err := client.Get(ctx, key, &k3kService); err != nil {
+		return "", err
+	}
+
+	url := fmt.Sprintf("https://%s:%d", k3kService.Spec.ClusterIP, server.ServerPort)
+
+	if k3kService.Spec.Type == v1.ServiceTypeNodePort {
+		nodePort := k3kService.Spec.Ports[0].NodePort
+		url = fmt.Sprintf("https://%s:%d", hostServerIP, nodePort)
 	}
 
-	return kubeconfig, nil
+	return url, nil
 }