Add networkpolicy to clustersets

Signed-off-by: galal-hussein <[email protected]>

.gitignore

@@ -6,3 +6,4 @@
 .idea
 .vscode/
 __debug*
+*-kubeconfig.yaml

charts/k3k/crds/k3k.io_clusters.yaml

@@ -113,9 +113,6 @@ spec:
                         type: boolean
                       ingressClassName:
                         type: string
-                    required:
-                    - enabled
-                    - ingressClassName
                     type: object
                   loadbalancer:
                     properties:
@@ -131,10 +128,6 @@ spec:
                     required:
                     - enabled
                     type: object
-                required:
-                - ingress
-                - loadbalancer
-                - nodePort
                 type: object
               nodeSelector:
                 additionalProperties:

charts/k3k/templates/deployment.yaml

@@ -15,23 +15,18 @@ spec:
       labels:
         {{- include "k3k.selectorLabels" . | nindent 8 }}
     spec:
-      volumes:
-        - name: webhook-serving
-          secret:
-            secretName: webhook-secret
       containers:
         - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           name: {{ .Chart.Name }}
+          environment:
+          - name: CLUSTER_CIDR
+            value: {{ .Values.host.clusterCIDR }}
           ports:
           - containerPort: 8080
             name: https
             protocol: TCP
           - containerPort: 9443
             name: https-webhook
             protocol: TCP
-          volumeMounts:
-            - name: webhook-serving
-              readOnly: true
-              mountPath: "/tmp/k8s-webhook-server/serving-certs"
       serviceAccountName: {{ include "k3k.serviceAccountName" . }}

charts/k3k/values.yaml

@@ -11,6 +11,10 @@ imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
 
+host:
+# clusterCIDR specifies the clusterCIDR that will be added to the default networkpolicy for clustersets
+  clusterCIDR: ""
+
 serviceAccount:
   # Specifies whether a service account should be created
   create: true

main.go

@@ -4,6 +4,7 @@ package main
 import (
 	"context"
 	"flag"
+	"os"
 
 	"github.com/rancher/k3k/pkg/apis/k3k.io/v1alpha1"
 	"github.com/rancher/k3k/pkg/controller/cluster"
@@ -13,24 +14,35 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
-	ctrlconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
 
-var Scheme = runtime.NewScheme()
+const (
+	clusterCIDRFlagName = "cluster-cidr"
+	clusterCIDREnvVar   = "CLUSTER_CIDR"
+	KubeconfigFlagName  = "kubeconfig"
+)
+
+var (
+	Scheme      = runtime.NewScheme()
+	clusterCIDR string
+	kubeconfig  string
+)
 
 func init() {
 	_ = clientgoscheme.AddToScheme(Scheme)
 	_ = v1alpha1.AddToScheme(Scheme)
 }
 
 func main() {
-	ctrlconfig.RegisterFlags(nil)
-	flag.Parse()
-
+	fs := addFlags()
+	fs.Parse(os.Args[1:])
 	ctx := context.Background()
 
-	kubeconfig := flag.Lookup("kubeconfig").Value.String()
+	if clusterCIDR == "" {
+		clusterCIDR = os.Getenv(clusterCIDREnvVar)
+	}
+
 	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
 	if err != nil {
 		klog.Fatalf("Failed to create config from kubeconfig file: %v", err)
@@ -52,12 +64,13 @@ func main() {
 		klog.Fatalf("Failed to add the new cluster controller: %v", err)
 	}
 	klog.Info("adding clusterset controller")
-	if err := clusterset.Add(ctx, mgr); err != nil {
+	if err := clusterset.Add(ctx, mgr, clusterCIDR); err != nil {
 		klog.Fatalf("Failed to add the clusterset controller: %v", err)
 	}
 
-	if err := cluster.AddWebhookHandler(ctx, mgr); err != nil {
-		klog.Fatalf("failed to add a webhook for the cluster type: %v", err)
+	klog.Info("adding networkpolicy node controller")
+	if err := clusterset.AddNodeController(ctx, mgr, clusterCIDR); err != nil {
+		klog.Fatalf("Failed to add the clusterset controller: %v", err)
 	}
 
 	if err := cluster.AddPodController(ctx, mgr); err != nil {
@@ -68,3 +81,10 @@ func main() {
 		klog.Fatalf("Failed to start the manager: %v", err)
 	}
 }
+
+func addFlags() *flag.FlagSet {
+	fs := flag.NewFlagSet("k3k", flag.ExitOnError)
+	fs.StringVar(&clusterCIDR, clusterCIDRFlagName, "", "The host's cluster CIDR")
+	fs.StringVar(&kubeconfig, KubeconfigFlagName, "", "Paths to a kubeconfig. Only required if out-of-cluster.")
+	return fs
+}

pkg/apis/k3k.io/v1alpha1/types.go

@@ -56,6 +56,7 @@ type ClusterSpec struct {
 	// Persistence contains options controlling how the etcd data of the virtual cluster is persisted. By default, no data
 	// persistence is guaranteed, so restart of a virtual cluster pod may result in data loss without this field.
 	Persistence *PersistenceConfig `json:"persistence,omitempty"`
+	// +optional
 	// Expose contains options for exposing the apiserver inside/outside of the cluster. By default, this is only exposed as a
 	// clusterIP which is relatively secure, but difficult to access outside of the cluster.
 	Expose *ExposeConfig `json:"expose,omitempty"`
@@ -91,14 +92,17 @@ type PersistenceConfig struct {
 }
 
 type ExposeConfig struct {
-	Ingress      *IngressConfig      `json:"ingress"`
-	LoadBalancer *LoadBalancerConfig `json:"loadbalancer"`
-	NodePort     *NodePortConfig     `json:"nodePort"`
+	// +optional
+	Ingress *IngressConfig `json:"ingress,omitempty"`
+	// +optional
+	LoadBalancer *LoadBalancerConfig `json:"loadbalancer,omitempty"`
+	// +optional
+	NodePort *NodePortConfig `json:"nodePort,omitempty"`
 }
 
 type IngressConfig struct {
-	Enabled          bool   `json:"enabled"`
-	IngressClassName string `json:"ingressClassName"`
+	Enabled          bool   `json:"enabled,omitempty"`
+	IngressClassName string `json:"ingressClassName,omitempty"`
 }
 
 type LoadBalancerConfig struct {

pkg/controller/cluster/pod.go

@@ -33,6 +33,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
+const (
+	podController = "k3k-pod-controller"
+)
+
 type PodReconciler struct {
 	Client ctrlruntimeclient.Client
 	Scheme *runtime.Scheme