title: Add ipsec support

Signed-off-by: GreatLazyMan <[email protected]>

Author: GreatLazyMan

cluster/images/Dockerfile

@@ -4,6 +4,6 @@ ARG BINARY
 
 RUN apk add --no-cache ca-certificates
 RUN apk update && apk upgrade
-RUN apk add ip6tables iptables curl tcpdump busybox-extras
+RUN apk add ip6tables iptables ipset curl tcpdump busybox-extras
 
 COPY ${BINARY} /bin/${BINARY}

deploy/crds/kosmos.io_clusters.yaml

@@ -56,6 +56,10 @@ spec:
                     - ip
                     - ip6
                     type: object
+                  clusterpodCIDRs:
+                    items:
+                      type: string
+                    type: array
                   cni:
                     default: calico
                     type: string
@@ -114,6 +118,8 @@ spec:
                   useIPPool:
                     default: false
                     type: boolean
+                  useexternalapiserver:
+                    type: boolean
                 type: object
               clusterTreeOptions:
                 properties:

deploy/crds/kosmos.io_nodeconfigs.yaml

@@ -92,6 +92,18 @@ spec:
                   - mac
                   type: object
                 type: array
+              ipsetsavoidmasq:
+                items:
+                  properties:
+                    cidr:
+                      type: string
+                    name:
+                      type: string
+                  required:
+                  - cidr
+                  - name
+                  type: object
+                type: array
               iptables:
                 items:
                   properties:

pkg/apis/kosmos/v1alpha1/cluster_types.go

@@ -95,6 +95,10 @@ type ClusterLinkOptions struct {
 	// NodeElasticIPMap presents mapping between nodename in kubernetes and elasticIP
 	// +optional
 	NodeElasticIPMap map[string]string `json:"nodeElasticIPMap,omitempty"`
+	// +optional
+	ClusterPodCIDRs []string `json:"clusterpodCIDRs,omitempty"`
+	// +optional
+	UseExternalApiserver bool `json:"useexternalapiserver,omitempty"`
 }
 
 type ClusterTreeOptions struct {

pkg/apis/kosmos/v1alpha1/nodeconfig_types.go

@@ -21,13 +21,14 @@ type NodeConfig struct {
 }
 
 type NodeConfigSpec struct {
-	Devices      []Device     `json:"devices,omitempty"`
-	Routes       []Route      `json:"routes,omitempty"`
-	Iptables     []Iptables   `json:"iptables,omitempty"`
-	Fdbs         []Fdb        `json:"fdbs,omitempty"`
-	Arps         []Arp        `json:"arps,omitempty"`
-	XfrmPolicies []XfrmPolicy `json:"xfrmpolicies,omitempty"`
-	XfrmStates   []XfrmState  `json:"xfrmstates,omitempty"`
+	Devices          []Device     `json:"devices,omitempty"`
+	Routes           []Route      `json:"routes,omitempty"`
+	Iptables         []Iptables   `json:"iptables,omitempty"`
+	Fdbs             []Fdb        `json:"fdbs,omitempty"`
+	Arps             []Arp        `json:"arps,omitempty"`
+	XfrmPolicies     []XfrmPolicy `json:"xfrmpolicies,omitempty"`
+	XfrmStates       []XfrmState  `json:"xfrmstates,omitempty"`
+	IPsetsAvoidMasqs []IPset      `json:"ipsetsavoidmasq,omitempty"`
 }
 
 type NodeConfigStatus struct {
@@ -137,6 +138,16 @@ func (a *XfrmState) Compare(v XfrmState) bool {
 		a.SPI == v.SPI
 }
 
+type IPset struct {
+	CIDR string `json:"cidr"`
+	Name string `json:"name"`
+}
+
+func (a *IPset) Compare(v IPset) bool {
+	return a.CIDR == v.CIDR &&
+		a.Name == v.Name
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 type NodeConfigList struct {

pkg/clusterlink/agent-manager/network-manager/network_manager.go

@@ -127,6 +127,14 @@ func (e *NetworkManager) Diff(oldConfig, newConfig *clusterlinkv1alpha1.NodeConf
 		createConfig.XfrmStates = createRecord
 		isSame = false
 	}
+	//ipset
+	if flag, deleteRecord, createRecord := compareFunc(oldConfig.IPsetsAvoidMasqs, newConfig.IPsetsAvoidMasqs, func(a, b clusterlinkv1alpha1.IPset) bool {
+		return a.Compare(b)
+	}); !flag {
+		deleteConfig.IPsetsAvoidMasqs = deleteRecord
+		createConfig.IPsetsAvoidMasqs = createRecord
+		isSame = false
+	}
 	// iptables:
 	if flag, deleteRecord, createRecord := compareFunc(oldConfig.Iptables, newConfig.Iptables, func(a, b clusterlinkv1alpha1.Iptables) bool {
 		return a.Compare(b)
@@ -215,6 +223,12 @@ func (e *NetworkManager) WriteSys(configDiff *ConfigDiff) error {
 				errs = errors.Wrap(err, fmt.Sprint(errs))
 			}
 		}
+		if config.IPsetsAvoidMasqs != nil {
+			if err := e.NetworkInterface.DeleteIPsetsAvoidMasq(config.IPsetsAvoidMasqs); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
 	}
 
 	if configDiff.createConfig != nil {
@@ -262,6 +276,12 @@ func (e *NetworkManager) WriteSys(configDiff *ConfigDiff) error {
 				errs = errors.Wrap(err, fmt.Sprint(errs))
 			}
 		}
+		if config.IPsetsAvoidMasqs != nil {
+			if err := e.NetworkInterface.AddIPsetsAvoidMasq(config.IPsetsAvoidMasqs); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
 	}
 
 	return errs
@@ -300,6 +320,7 @@ func printNodeConfig(data *clusterlinkv1alpha1.NodeConfigSpec) {
 	klog.Infof("Routes: %v", data.Routes)
 	klog.Infof("XfrmPolicys: %v", data.XfrmPolicies)
 	klog.Infof("XfrmStates: %v", data.XfrmStates)
+	klog.Infof("IPsetsAvoidMasqs: %v", data.IPsetsAvoidMasqs)
 }
 
 func (e *NetworkManager) UpdateSync() NodeConfigSyncStatus {

pkg/clusterlink/controllers/calicoippool/calicoippool_controller.go

@@ -339,6 +339,10 @@ func (c *Controller) Reconcile(key utils.QueueKey) error {
 	}
 
 	klog.Infof("start reconcile cluster %s", cluster.Name)
+	if cluster.Name == c.clusterName && cluster.Spec.ClusterLinkOptions.CNI != utils.CNITypeCalico {
+		klog.Infof("cluster %s cni type is %s skip reconcile", cluster.Name, cluster.Spec.ClusterLinkOptions.CNI)
+		return nil
+	}
 	for ipPool := range c.globalExtIPPoolSet {
 		if ipPool.cluster == cluster.Name {
 			delete(c.globalExtIPPoolSet, ipPool)
@@ -371,10 +375,6 @@ func (c *Controller) Reconcile(key utils.QueueKey) error {
 		c.globalExtIPPoolSet[extIPPool] = struct{}{}
 	}
 	klog.Infof("now has %d globalIPPools", len(c.globalExtIPPoolSet))
-	if cluster.Spec.ClusterLinkOptions.CNI != utils.CNITypeCalico {
-		klog.Infof("cluster %s cni type is %s skip reconcile", cluster.Name, cluster.Spec.ClusterLinkOptions.CNI)
-		return nil
-	}
 	if c.iPPoolClient == nil {
 		if cluster.Name == c.clusterName {
 			ipPoolClient, err := c.createIPPoolClient(cluster)

pkg/clusterlink/controllers/cluster/cluster_controller.go

@@ -42,6 +42,7 @@ import (
 // KubeFlannelNetworkConfig
 const (
 	FlannelCNI             = "flannel"
+	GlobalRouterCNI        = "globalrouter"
 	KubeFlannelConfigMap   = "kube-flannel-cfg"
 	KubeFlannelNetworkConf = "net-conf.json"
 	KubeFlannelIPPool      = "Network"
@@ -122,16 +123,23 @@ func (c *Controller) Start(ctx context.Context) error {
 	factory := informers.NewSharedInformerFactory(c.kubeClient, 0)
 	informer := factory.Core().V1().Pods().Informer()
 	c.podLister = factory.Core().V1().Pods().Lister()
-	podFilterFunc := func(pod *corev1.Pod) bool {
-		//TODO 确认这个写法是否正确
-		return pod.Labels["component"] == "kube-apiserver"
-	}
 
 	cluster, err := c.clusterLinkClient.KosmosV1alpha1().Clusters().Get(ctx, c.clusterName, metav1.GetOptions{})
 	if err != nil {
 		klog.Errorf("can not find local cluster %s, err: %v", c.clusterName, err)
 		return err
 	}
+	var podFilterFunc func(pod *corev1.Pod) bool
+	if cluster.Spec.ClusterLinkOptions.UseExternalApiserver {
+		podFilterFunc = func(pod *corev1.Pod) bool {
+			return pod.Labels["k8s-app"] == "kube-proxy" || pod.Labels["app"] == "clusterlink-controller-manager"
+		}
+	} else {
+		podFilterFunc = func(pod *corev1.Pod) bool {
+			//TODO 确认这个写法是否正确
+			return pod.Labels["component"] == "kube-apiserver"
+		}
+	}
 	_, err = informer.AddEventHandler(cache.FilteringResourceEventHandler{
 		FilterFunc: func(obj interface{}) bool {
 			pod := obj.(*corev1.Pod)
@@ -157,6 +165,15 @@ func (c *Controller) Start(ctx context.Context) error {
 			klog.Errorf("cluster %s initCalicoInformer err: %v", err)
 			return err
 		}
+	} else if cluster.Spec.ClusterLinkOptions.CNI == GlobalRouterCNI {
+		c.setClusterPodCIDRFun = func(cluster *clusterlinkv1alpha1.Cluster) error {
+			if len(cluster.Spec.ClusterLinkOptions.ClusterPodCIDRs) == 0 {
+				klog.Errorf("Please define ClusterPodCIDRs for cni %s", GlobalRouterCNI)
+				return fmt.Errorf("clusterpodcidrs is not defined for cni %s", GlobalRouterCNI)
+			}
+			cluster.Status.ClusterLinkStatus.PodCIDRs = cluster.Spec.ClusterLinkOptions.ClusterPodCIDRs
+			return nil
+		}
 	} else {
 		isEtcd := CheckIsEtcd(cluster)
 		if !isEtcd {
@@ -188,6 +205,7 @@ func (c *Controller) Reconcile(key utils.QueueKey) error {
 		klog.Error("invalid key")
 		return fmt.Errorf("invalid key")
 	}
+	klog.Info("cluster controller start reconcile")
 	namespacedName := types.NamespacedName{
 		Name:      clusterWideKey.Name,
 		Namespace: clusterWideKey.Namespace,

pkg/clusterlink/network-manager/handlers/cni_support.go

@@ -0,0 +1,38 @@
+package handlers
+
+import (
+	"k8s.io/klog"
+
+	"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
+	"github.com/kosmos.io/kosmos/pkg/clusterlink/controllers/cluster"
+	"github.com/kosmos.io/kosmos/pkg/clusterlink/network"
+)
+
+type CNISupport struct {
+	Next
+}
+
+func (h *CNISupport) Do(c *Context) (err error) {
+	flannelClusters, otherClusters := c.Filter.GetClusterByCNI([]string{cluster.GlobalRouterCNI, cluster.FlannelCNI})
+	allClustes := append(flannelClusters, otherClusters...)
+	for _, flanflannelCluster := range flannelClusters {
+		var targetIPset []v1alpha1.IPset
+		for _, otherClusters := range allClustes {
+			if otherClusters.Name != flanflannelCluster.Name {
+				for _, cidr := range otherClusters.Status.ClusterLinkStatus.PodCIDRs {
+					targetIPset = append(targetIPset, v1alpha1.IPset{
+						Name: network.KosmosIPsetVoidMasq,
+						CIDR: cidr,
+					})
+				}
+			}
+		}
+		klog.Infof("flannel cluster name: %s, ipset: %v", flanflannelCluster.Name, targetIPset)
+		targetNodes := c.Filter.GetAllNodesByClusterName(flanflannelCluster.Name)
+		for _, node := range targetNodes {
+			c.Results[node.Name].IPsetsAvoidMasq = targetIPset
+		}
+	}
+
+	return nil
+}

pkg/clusterlink/network-manager/handlers/nodeconfig.go

@@ -12,13 +12,14 @@ import (
 
 // NodeConfig network configuration of the node
 type NodeConfig struct {
-	Devices      []v1alpha1.Device     `json:"devices,omitempty"`
-	Routes       []v1alpha1.Route      `json:"routes,omitempty"`
-	Iptables     []v1alpha1.Iptables   `json:"iptables,omitempty"`
-	Fdbs         []v1alpha1.Fdb        `json:"fdbs,omitempty"`
-	Arps         []v1alpha1.Arp        `json:"arps,omitempty"`
-	XfrmPolicies []v1alpha1.XfrmPolicy `json:"xfrmpolicies,omitempty"`
-	XfrmStates   []v1alpha1.XfrmState  `json:"xfrmstates,omitempty"`
+	Devices         []v1alpha1.Device     `json:"devices,omitempty"`
+	Routes          []v1alpha1.Route      `json:"routes,omitempty"`
+	Iptables        []v1alpha1.Iptables   `json:"iptables,omitempty"`
+	Fdbs            []v1alpha1.Fdb        `json:"fdbs,omitempty"`
+	Arps            []v1alpha1.Arp        `json:"arps,omitempty"`
+	XfrmPolicies    []v1alpha1.XfrmPolicy `json:"xfrmpolicies,omitempty"`
+	XfrmStates      []v1alpha1.XfrmState  `json:"xfrmstates,omitempty"`
+	IPsetsAvoidMasq []v1alpha1.IPset      `json:"ipsetsavoidmasq,omitempty"`
 }
 
 func (c *NodeConfig) ToString() string {
@@ -35,13 +36,14 @@ func (c *NodeConfig) ToJson() ([]byte, error) {
 
 func (c *NodeConfig) ConvertToNodeConfigSpec() v1alpha1.NodeConfigSpec {
 	return v1alpha1.NodeConfigSpec{
-		Devices:      c.Devices,
-		Routes:       c.Routes,
-		Iptables:     c.Iptables,
-		Fdbs:         c.Fdbs,
-		Arps:         c.Arps,
-		XfrmStates:   c.XfrmStates,
-		XfrmPolicies: c.XfrmPolicies,
+		Devices:          c.Devices,
+		Routes:           c.Routes,
+		Iptables:         c.Iptables,
+		Fdbs:             c.Fdbs,
+		Arps:             c.Arps,
+		XfrmStates:       c.XfrmStates,
+		XfrmPolicies:     c.XfrmPolicies,
+		IPsetsAvoidMasqs: c.IPsetsAvoidMasq,
 	}
 }