From 3c369ab6422098d854727b25383d1522b44a5b14 Mon Sep 17 00:00:00 2001 From: baoyinghai Date: Thu, 31 Aug 2023 17:04:53 +0800 Subject: [PATCH] feat: print log during create vxlan device Signed-off-by: baoyinghai --- Makefile | 2 +- cmd/agent/app/agent.go | 2 + pkg/agent/controller.go | 13 +++++- pkg/agent/network-manager/network_manager.go | 4 ++ pkg/network/adapter.go | 24 ++++++----- pkg/network/constant.go | 35 ++++++++++------ pkg/network/device.go | 44 ++++++++++++-------- pkg/network/interface.go | 2 + 8 files changed, 85 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 95a2545df..a4001b416 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,7 @@ lint-fix: golangci-lint golangci-lint: ifeq (, $(shell which golangci-lint)) - GO111MODULE=on go install github.com/golangci/golangci-lint/cmd/golangci-lint@1.54.1 + GO111MODULE=on go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2 GOLANGLINT_BIN=$(shell go env GOPATH)/bin/golangci-lint else GOLANGLINT_BIN=$(shell which golangci-lint) diff --git a/cmd/agent/app/agent.go b/cmd/agent/app/agent.go index aa3b85271..8de607ae1 100644 --- a/cmd/agent/app/agent.go +++ b/cmd/agent/app/agent.go @@ -114,10 +114,12 @@ func run(ctx context.Context, opts *options.Options) error { factory := clusterlinkinformer.NewSharedInformerFactory(clusterlinkClientset, 0) nodeConfigLister := factory.Clusterlink().V1alpha1().NodeConfigs().Lister() + clusterLister := factory.Clusterlink().V1alpha1().Clusters().Lister() clusterNodeController := agent.Reconciler{ Scheme: mgr.GetScheme(), NodeConfigLister: nodeConfigLister, + ClusterLister: clusterLister, NodeName: os.Getenv(utils.EnvNodeName), ClusterName: os.Getenv(utils.EnvClusterName), NetworkManager: agent.NetworkManager(), diff --git a/pkg/agent/controller.go b/pkg/agent/controller.go index f07d8749d..447560b16 100644 --- a/pkg/agent/controller.go +++ b/pkg/agent/controller.go @@ -35,6 +35,7 @@ type Reconciler struct { ClusterName string NetworkManager *networkmanager.NetworkManager NodeConfigLister clusterlinkv1alpha1lister.NodeConfigLister + ClusterLister clusterlinkv1alpha1lister.ClusterLister DebounceFunc func(f func()) } @@ -87,7 +88,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( if err := r.Get(ctx, request.NamespacedName, &reconcileNode); err != nil { // The resource no longer exists if apierrors.IsNotFound(err) { - // TODO: cleanup + nodeConfigSyncStatus := r.NetworkManager.UpdateFromCRD(&clusterlinkv1alpha1.NodeConfig{ + Spec: clusterlinkv1alpha1.NodeConfigSpec{}}) + r.logResult(nodeConfigSyncStatus) return reconcile.Result{}, nil } klog.Errorf("get clusternode %s error: %v", request.NamespacedName, err) @@ -100,6 +103,14 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( return reconcile.Result{}, nil } + localCluster, err := r.ClusterLister.Get(r.ClusterName) + if err != nil { + klog.Errorf("could not get local cluster, clusterNode: %s, err: %v", r.NodeName, err) + return reconcile.Result{}, nil + } + + r.NetworkManager.UpdateConfig(localCluster) + r.DebounceFunc(func() { nodeConfigSyncStatus := r.NetworkManager.UpdateFromCRD(&reconcileNode) r.logResult(nodeConfigSyncStatus) diff --git a/pkg/agent/network-manager/network_manager.go b/pkg/agent/network-manager/network_manager.go index a41d192a8..379de4464 100644 --- a/pkg/agent/network-manager/network_manager.go +++ b/pkg/agent/network-manager/network_manager.go @@ -296,3 +296,7 @@ func (e *NetworkManager) UpdateSync() NodeConfigSyncStatus { return NodeConfigSyncSuccess } + +func (e *NetworkManager) UpdateConfig(cluster *clusterlinkv1alpha1.Cluster) { + e.NetworkInterface.UpdateCidrConfig(cluster) +} diff --git a/pkg/network/adapter.go b/pkg/network/adapter.go index e92b9927b..e12daea7c 100644 --- a/pkg/network/adapter.go +++ b/pkg/network/adapter.go @@ -57,7 +57,7 @@ func (n *DefaultNetWork) LoadSysConfig() (*clusterlinkv1alpha1.NodeConfigSpec, e func (n *DefaultNetWork) DeleteArps(arps []clusterlinkv1alpha1.Arp) error { var errs error for _, arp := range arps { - if err := deleteArp(arp); err == nil { + if err := deleteArp(arp); err != nil { errs = errors.Wrap(err, fmt.Sprintf("delete arp error, fdb: %v", arp)) } } @@ -67,7 +67,7 @@ func (n *DefaultNetWork) DeleteArps(arps []clusterlinkv1alpha1.Arp) error { func (n *DefaultNetWork) DeleteFdbs(fdbs []clusterlinkv1alpha1.Fdb) error { var errs error for _, fdb := range fdbs { - if err := deleteFdb(fdb); err == nil { + if err := deleteFdb(fdb); err != nil { errs = errors.Wrap(err, fmt.Sprintf("delete fdb error, fdb: %v", fdb)) } } @@ -77,7 +77,7 @@ func (n *DefaultNetWork) DeleteFdbs(fdbs []clusterlinkv1alpha1.Fdb) error { func (n *DefaultNetWork) DeleteIptables(records []clusterlinkv1alpha1.Iptables) error { var errs error for _, iptable := range records { - if err := deleteIptable(iptable); err == nil { + if err := deleteIptable(iptable); err != nil { errs = errors.Wrap(err, fmt.Sprintf("delete iptable error, deivce name: %v", iptable)) } } @@ -87,7 +87,7 @@ func (n *DefaultNetWork) DeleteIptables(records []clusterlinkv1alpha1.Iptables) func (n *DefaultNetWork) DeleteRoutes(routes []clusterlinkv1alpha1.Route) error { var errs error for _, route := range routes { - if err := deleteRoute(route); err == nil { + if err := deleteRoute(route); err != nil { errs = errors.Wrap(err, fmt.Sprintf("delete route error, deivce name: %v", route)) } } @@ -97,7 +97,7 @@ func (n *DefaultNetWork) DeleteRoutes(routes []clusterlinkv1alpha1.Route) error func (n *DefaultNetWork) DeleteDevices(devices []clusterlinkv1alpha1.Device) error { var errs error for _, device := range devices { - if err := deleteDevice(device); err == nil { + if err := deleteDevice(device); err != nil { errs = errors.Wrap(err, fmt.Sprintf("delete device error, deivce name: %s", device.Name)) } } @@ -127,7 +127,7 @@ func (n *DefaultNetWork) UpdateDevices([]clusterlinkv1alpha1.Device) error { func (n *DefaultNetWork) AddArps(arps []clusterlinkv1alpha1.Arp) error { var errs error for _, arp := range arps { - if err := addArp(arp); err == nil { + if err := addArp(arp); err != nil { errs = errors.Wrap(err, fmt.Sprintf("create arp error : %v", arp)) } } @@ -137,7 +137,7 @@ func (n *DefaultNetWork) AddArps(arps []clusterlinkv1alpha1.Arp) error { func (n *DefaultNetWork) AddFdbs(fdbs []clusterlinkv1alpha1.Fdb) error { var errs error for _, fdb := range fdbs { - if err := addFdb(fdb); err == nil { + if err := addFdb(fdb); err != nil { errs = errors.Wrap(err, fmt.Sprintf("create fdb error, deivce name: %v", fdb)) } } @@ -147,7 +147,7 @@ func (n *DefaultNetWork) AddFdbs(fdbs []clusterlinkv1alpha1.Fdb) error { func (n *DefaultNetWork) AddIptables(iptabless []clusterlinkv1alpha1.Iptables) error { var errs error for _, ipts := range iptabless { - if err := addIptables(ipts); err == nil { + if err := addIptables(ipts); err != nil { errs = errors.Wrap(err, fmt.Sprintf("create iptable error, deivce name: %v", ipts)) } } @@ -157,7 +157,7 @@ func (n *DefaultNetWork) AddIptables(iptabless []clusterlinkv1alpha1.Iptables) e func (n *DefaultNetWork) AddRoutes(routes []clusterlinkv1alpha1.Route) error { var errs error for _, route := range routes { - if err := addRoute(route); err == nil { + if err := addRoute(route); err != nil { errs = errors.Wrap(err, fmt.Sprintf("create route error, deivce name: %v", route)) } } @@ -167,7 +167,7 @@ func (n *DefaultNetWork) AddRoutes(routes []clusterlinkv1alpha1.Route) error { func (n *DefaultNetWork) AddDevices(devices []clusterlinkv1alpha1.Device) error { var errs error for _, device := range devices { - if err := addDevice(device); err == nil { + if err := addDevice(device); err != nil { errs = errors.Wrap(err, fmt.Sprintf("create device error, deivce name: %s", device.Name)) } } @@ -179,3 +179,7 @@ func (n *DefaultNetWork) InitSys() { klog.Warning(err) } } + +func (n *DefaultNetWork) UpdateCidrConfig(cluster *clusterlinkv1alpha1.Cluster) { + UpdateCidr(cluster.Spec.BridgeCIDRs.IP, cluster.Spec.BridgeCIDRs.IP6, cluster.Spec.LocalCIDRs.IP, cluster.Spec.LocalCIDRs.IP6) +} diff --git a/pkg/network/constant.go b/pkg/network/constant.go index bc2d2e3e3..b08dd8b3f 100644 --- a/pkg/network/constant.go +++ b/pkg/network/constant.go @@ -4,6 +4,7 @@ import ( "net" "github.com/vishvananda/netlink" + "k8s.io/klog/v2" ) const vxlanOverhead = 50 @@ -41,18 +42,6 @@ const ( ALL_ADDR_IPV4 = "0.0.0.0/0" ALL_ADDR_IPV6 = "0.0.0.0.0.0.0.0/0" - // VXLAN_ADDR_PREFIX_BRIDGE_IPv4 int = 220 - // VXLAN_ADDR_PREFIX_LOCAL_IPv4 int = 210 - - // VXLAN_ADDR_PREFIX_BRIDGE_IPv6 int = 9480 // 0x2508 - // VXLAN_ADDR_PREFIX_LOCAL_IPv6 int = 9470 // 0x24fe - - VXLAN_BRIDGE_NET_IPV4 = "220.0.0.0/8" - VXLAN_BRIDGE_NET_IPV6 = "9480::0/16" - - VXLAN_LOCAL_NET_IPV4 = "210.0.0.0/8" - VXLAN_LOCAL_NET_IPV6 = "9470::0/16" - VXLAN_BRIDGE_ID = 54 VXLAN_BRIDGE_PORT = 4876 @@ -85,6 +74,14 @@ type vxlanAttributes struct { family int } +var ( + VXLAN_BRIDGE_NET_IPV4 = "220.0.0.0/8" + VXLAN_BRIDGE_NET_IPV6 = "9480::0/16" + + VXLAN_LOCAL_NET_IPV4 = "210.0.0.0/8" + VXLAN_LOCAL_NET_IPV6 = "9470::0/16" +) + var VXLAN_BRIDGE = &vxlanAttributes{ name: VXLAN_BRIDGE_NAME, vxlanID: VXLAN_BRIDGE_ID, @@ -130,3 +127,17 @@ var VXLAN_LOCAL_6 = &vxlanAttributes{ } var ALL_DEVICES = []*vxlanAttributes{VXLAN_BRIDGE, VXLAN_LOCAL, VXLAN_BRIDGE_6, VXLAN_LOCAL_6} + +func UpdateCidr(bridge4, bridge6, local4, local6 string) { + VXLAN_BRIDGE_NET_IPV4 = bridge4 + VXLAN_BRIDGE_NET_IPV6 = bridge6 + VXLAN_LOCAL_NET_IPV4 = local4 + VXLAN_LOCAL_NET_IPV6 = local6 + + VXLAN_BRIDGE.cidr = VXLAN_BRIDGE_NET_IPV4 + VXLAN_LOCAL.cidr = VXLAN_LOCAL_NET_IPV4 + VXLAN_BRIDGE_6.cidr = VXLAN_BRIDGE_NET_IPV6 + VXLAN_LOCAL_6.cidr = VXLAN_LOCAL_NET_IPV6 + + klog.Infof("update cidr, bridge_v4: %s, bridge_v6: %s, local_v4: %s, local_v: %s", VXLAN_BRIDGE_NET_IPV4, VXLAN_BRIDGE_NET_IPV6, VXLAN_LOCAL_NET_IPV4, VXLAN_LOCAL_NET_IPV6) +} diff --git a/pkg/network/device.go b/pkg/network/device.go index 3f8ce08c9..49e5f8fbb 100644 --- a/pkg/network/device.go +++ b/pkg/network/device.go @@ -157,7 +157,13 @@ func loadDevices() ([]clusterlinkv1alpha1.Device, error) { } } - routes, err := netlink.RouteList(nil, d.family) + if vxlanNet == nil { + msg := fmt.Sprintf("Cannot get ip of device: %s", d.name) + klog.Error(msg) + return nil, fmt.Errorf(msg) + } + + addrListAll, err := netlink.AddrList(nil, d.family) if err != nil { return nil, err @@ -166,25 +172,29 @@ func loadDevices() ([]clusterlinkv1alpha1.Device, error) { interfaceIndex := -1 vxlanIPAddr := vxlanNet.IP.String() - for _, r := range routes { - internalIP := r.Src.String() - if ip, err := CIDRIPGenerator(d.cidr, internalIP); err == nil { - if ip.String() == vxlanIPAddr { - interfaceIndex = r.LinkIndex - break + for _, r := range addrListAll { + if r.LinkIndex != vxlanIface.Attrs().Index { + if ip, err := CIDRIPGenerator(d.cidr, r.IP.String()); err == nil { + if ip.String() == vxlanIPAddr { + interfaceIndex = r.LinkIndex + break + } } } } - if interfaceIndex == -1 { - klog.Warning("can not find the dev for vxlan, name: %s", d.name) - continue - } + bindDev := "" - defaultIface, err := netlink.LinkByIndex(interfaceIndex) - if err != nil { - klog.Errorf("When we get device by linkinx, get error : %v", err) - return nil, err + if interfaceIndex == -1 { + klog.Warningf("can not find the phys_dev for vxlan, name: %s", d.name) + } else { + defaultIface, err := netlink.LinkByIndex(interfaceIndex) + if err != nil { + klog.Errorf("When we get device by linkinx, get error : %v", err) + return nil, err + } else { + bindDev = defaultIface.Attrs().Name + } } ret = append(ret, clusterlinkv1alpha1.Device{ @@ -192,7 +202,7 @@ func loadDevices() ([]clusterlinkv1alpha1.Device, error) { Name: vxlan.LinkAttrs.Name, Addr: vxlanNet.String(), Mac: vxlan.LinkAttrs.HardwareAddr.String(), - BindDev: defaultIface.Attrs().Name, + BindDev: bindDev, ID: int32(vxlan.VxlanId), Port: int32(vxlan.Port), }) @@ -324,7 +334,7 @@ func UpdateDefaultIptablesAndKernalConfig(name string, ipFamily int) error { } } - klog.Infof("Get default interface %s, mtu %d, ipv4 %s, ipv6: %s", name) + klog.Infof("Get default interface %s", name) return nil } diff --git a/pkg/network/interface.go b/pkg/network/interface.go index 791f95428..75f58f28c 100644 --- a/pkg/network/interface.go +++ b/pkg/network/interface.go @@ -33,6 +33,8 @@ type NetWork interface { AddDevices([]clusterlinkv1alpha1.Device) error InitSys() + + UpdateCidrConfig(cluster *clusterlinkv1alpha1.Cluster) } func NewNetWork() NetWork {