diff --git a/plugins/meta/sbr/main.go b/plugins/meta/sbr/main.go index 7379a01ea..c89fdeda6 100644 --- a/plugins/meta/sbr/main.go +++ b/plugins/meta/sbr/main.go @@ -23,6 +23,7 @@ import ( "github.com/alexflint/go-filemutex" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" @@ -49,6 +50,16 @@ type PluginConf struct { // Add plugin-specific flags here Table *int `json:"table,omitempty"` + // Gateways allows specifying static/hardcoded gateway IP addresses + // If set, these will be used instead of the gateway from prevResult + // Supports dual-stack: provide one IPv4 and/or one IPv6 gateway + // Note: Currently applies the same gateway to all IPs of the same family. + // Per-subnet gateway mapping is not yet supported. + Gateways []string `json:"gateways,omitempty"` + // AddSourceHints adds source IP hints to subnet routes in the main table, + // enabling destination-based routing when no explicit source IP is specified. + // This allows both source-based and destination-based routing to work together. + AddSourceHints bool `json:"addSourceHints,omitempty"` } // Wrapper that does a lock before and unlock after operations to serialise @@ -168,7 +179,7 @@ func cmdAdd(args *skel.CmdArgs) error { if conf.Table != nil { return doRoutesWithTable(ipCfgs, *conf.Table) } - return doRoutes(ipCfgs, args.IfName) + return doRoutes(ipCfgs, args.IfName, conf.Gateways, conf.AddSourceHints) }) if err != nil { return err @@ -207,7 +218,7 @@ func getNextTableID(rules []netlink.Rule, routes []netlink.Route, candidateID in } // doRoutes does all the work to set up routes and rules during an add. -func doRoutes(ipCfgs []*current.IPConfig, iface string) error { +func doRoutes(ipCfgs []*current.IPConfig, iface string, staticGateways []string, addSourceHints bool) error { // Get a list of rules and routes ready. rules, err := netlinksafe.RuleList(netlink.FAMILY_ALL) if err != nil { @@ -238,6 +249,33 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string) error { return fmt.Errorf("Unable to list routes: %v", err) } + // Parse static gateways if provided (supports dual-stack: one IPv4, one IPv6) + var staticGwV4, staticGwV6 net.IP + if len(staticGateways) > 2 { + return fmt.Errorf("gateways list cannot contain more than 2 entries (one IPv4, one IPv6)") + } + for _, gw := range staticGateways { + ip := net.ParseIP(gw) + if ip == nil { + return fmt.Errorf("invalid static gateway IP address: %s", gw) + } + if ip.To4() != nil { + if staticGwV4 != nil { + // We already have an IPv4 static gateway + return fmt.Errorf("gateways list contains multiple IPv4 addresses: %s and %s", staticGwV4, gw) + } + staticGwV4 = ip + log.Printf("Using static IPv4 gateway: %s", ip) + } else { + if staticGwV6 != nil { + // We already have an IPv6 static gateway + return fmt.Errorf("gateways list contains multiple IPv6 addresses: %s and %s", staticGwV6, gw) + } + staticGwV6 = ip + log.Printf("Using static IPv6 gateway: %s", ip) + } + } + // Loop through setting up source based rules and default routes. for _, ipCfg := range ipCfgs { log.Printf("Set rule for source %s", ipCfg.String()) @@ -260,10 +298,24 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string) error { return fmt.Errorf("Failed to add rule: %v", err) } + // Determine which gateway to use: static gateway takes precedence + // Match gateway by IP family (IPv4 vs IPv6) + var gateway net.IP + isIPv4 := ipCfg.Address.IP.To4() != nil + + switch { + case isIPv4 && staticGwV4 != nil: + gateway = staticGwV4 + case !isIPv4 && staticGwV6 != nil: + gateway = staticGwV6 + case ipCfg.Gateway != nil: + gateway = ipCfg.Gateway + } + // Add a default route, since this may have been removed by previous // plugin. - if ipCfg.Gateway != nil { - log.Printf("Adding default route to gateway %s", ipCfg.Gateway.String()) + if gateway != nil { + log.Printf("Adding default route to gateway %s", gateway.String()) var dest net.IPNet if ipCfg.Address.IP.To4() != nil { @@ -276,7 +328,7 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string) error { route := netlink.Route{ Dst: &dest, - Gw: ipCfg.Gateway, + Gw: gateway, Table: table, LinkIndex: linkIndex, } @@ -284,7 +336,7 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string) error { err = netlink.RouteAdd(&route) if err != nil { return fmt.Errorf("Failed to add default route to %s: %v", - ipCfg.Gateway.String(), + gateway.String(), err) } } @@ -320,15 +372,59 @@ func doRoutes(ipCfgs []*current.IPConfig, iface string) error { table = getNextTableID(rules, routes, table) } - // Delete all the interface routes in the default routing table, which were - // copied to source based routing tables. - // Not deleting them while copying to accommodate for multiple ipCfgs from - // the same subnet. Else, (error for network is unreachable while adding gateway) - for _, route := range routes { - log.Printf("Deleting route %s from table %d", route.String(), route.Table) - err := netlink.RouteDel(&route) - if err != nil { - return fmt.Errorf("Failed to delete route: %v", err) + // Handle routes in the default routing table + if addSourceHints { + // Keep or re-add subnet routes in main table with source IP hints + // for destination-based routing when no explicit source IP is specified + log.Printf("Adding source IP hints to subnet routes in main table") + + for _, ipCfg := range ipCfgs { + // Find the subnet route for this IP + subnet := &net.IPNet{ + IP: ipCfg.Address.IP.Mask(ipCfg.Address.Mask), + Mask: ipCfg.Address.Mask, + } + + log.Printf("Adding/replacing route for subnet %s with src hint %s in main table", + subnet.String(), ipCfg.Address.IP.String()) + + // Add route to main table with source IP hint + route := netlink.Route{ + LinkIndex: linkIndex, + Dst: subnet, + Src: ipCfg.Address.IP, + Table: unix.RT_TABLE_MAIN, + Scope: netlink.SCOPE_LINK, + } + + // Use RouteReplace to update if it exists + err := netlink.RouteReplace(&route) + if err != nil { + log.Printf("Warning: Failed to add subnet route to main table: %v", err) + // Don't fail completely, just warn + } + } + + // Delete non-subnet routes from main table (gateway routes, etc.) + for _, route := range routes { + // Skip subnet routes (scope link), only delete other routes + if route.Scope != netlink.SCOPE_LINK { + log.Printf("Deleting non-subnet route %s from table %d", route.String(), route.Table) + err := netlink.RouteDel(&route) + if err != nil { + log.Printf("Warning: Failed to delete route: %v", err) + } + } + } + } else { + // Not deleting them while copying to accommodate for multiple ipCfgs from + // the same subnet. Else, (error for network is unreachable while adding gateway) + for _, route := range routes { + log.Printf("Deleting route %s from table %d", route.String(), route.Table) + err := netlink.RouteDel(&route) + if err != nil { + return fmt.Errorf("Failed to delete route: %v", err) + } } } diff --git a/plugins/meta/sbr/sbr_linux_test.go b/plugins/meta/sbr/sbr_linux_test.go index d0f872672..1a6b41bbc 100644 --- a/plugins/meta/sbr/sbr_linux_test.go +++ b/plugins/meta/sbr/sbr_linux_test.go @@ -628,4 +628,350 @@ var _ = Describe("sbr test", func() { Expect(rules[1].Table).To(Equal(tableID)) Expect(rules[1].Src.String()).To(Equal("192.168.1.209/32")) }) + + It("Works with static IPv4 gateway configuration", func() { + ifname := "net1" + // Configure with a static IPv4 gateway (192.168.1.254) that differs from + // the gateway in prevResult (192.168.1.1) + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-sbr-test", + "type": "sbr", + "gateways": ["192.168.1.254"], + "prevResult": { + "cniVersion": "0.3.0", + "interfaces": [ + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "192.168.1.209/24", + "gateway": "192.168.1.1", + "interface": 0 + } + ], + "routes": [] + } +}` + conf = fmt.Sprintf(conf, ifname, targetNs.Path()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNs.Path(), + IfName: ifname, + StdinData: []byte(conf), + } + + err := setup(targetNs, createDefaultStatus()) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred()) + + newStatus, err := readback(targetNs, []string{"net1", "eth0"}) + Expect(err).NotTo(HaveOccurred()) + + // Check that the static gateway (192.168.1.254) is used instead of + // the prevResult gateway (192.168.1.1) + Expect(newStatus.Rules).To(HaveLen(1)) + devNet1 := newStatus.Devices[0] + + // Find the default route in table 100 + var foundDefaultRoute bool + for _, route := range devNet1.Routes { + if route.Table == 100 && route.Dst != nil && route.Dst.IP.Equal(net.IPv4zero) { + // This is the default route, check the gateway + Expect(route.Gw.String()).To(Equal("192.168.1.254")) + foundDefaultRoute = true + break + } + } + Expect(foundDefaultRoute).To(BeTrue(), "Expected to find default route with static gateway") + }) + + It("Works with static IPv6 gateway configuration", func() { + ifname := "net1" + // Configure with a static IPv6 gateway + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-sbr-test", + "type": "sbr", + "gateways": ["fd00::1"], + "prevResult": { + "cniVersion": "0.3.0", + "interfaces": [ + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "6", + "address": "fd00::10/64", + "gateway": "fd00::ffff", + "interface": 0 + } + ], + "routes": [] + } +}` + conf = fmt.Sprintf(conf, ifname, targetNs.Path()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNs.Path(), + IfName: ifname, + StdinData: []byte(conf), + } + + preStatus := createDefaultStatus() + // Replace IPv4 address with IPv6 on net1 + preStatus.Devices[1].Addrs = []net.IPNet{ + { + IP: net.ParseIP("fd00::10"), + Mask: net.CIDRMask(64, 128), + }, + } + // Replace IPv4 routes with empty (IPv6 routes are auto-created) + preStatus.Devices[1].Routes = []netlink.Route{} + + err := setup(targetNs, preStatus) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred()) + + newStatus, err := readback(targetNs, []string{"net1", "eth0"}) + Expect(err).NotTo(HaveOccurred()) + + // Check that the static gateway (fd00::1) is used instead of + // the prevResult gateway (fd00::ffff) + Expect(newStatus.Rules).To(HaveLen(1)) + devNet1 := newStatus.Devices[0] + + // Find the default route in table 100 + var foundDefaultRoute bool + for _, route := range devNet1.Routes { + if route.Table == 100 && route.Dst != nil && route.Dst.IP.Equal(net.IPv6zero) { + // This is the default route, check the gateway + Expect(route.Gw.String()).To(Equal("fd00::1")) + foundDefaultRoute = true + break + } + } + Expect(foundDefaultRoute).To(BeTrue(), "Expected to find IPv6 default route with static gateway") + }) + + It("Works with dual-stack static gateway configuration", func() { + ifname := "net1" + // Configure with both IPv4 and IPv6 static gateways + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-sbr-test", + "type": "sbr", + "gateways": ["192.168.1.254", "fd00::1"], + "prevResult": { + "cniVersion": "0.3.0", + "interfaces": [ + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "192.168.1.209/24", + "gateway": "192.168.1.1", + "interface": 0 + }, + { + "version": "6", + "address": "fd00::10/64", + "gateway": "fd00::ffff", + "interface": 0 + } + ], + "routes": [] + } +}` + conf = fmt.Sprintf(conf, ifname, targetNs.Path()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNs.Path(), + IfName: ifname, + StdinData: []byte(conf), + } + + preStatus := createDefaultStatus() + // Add IPv6 address alongside IPv4 on net1 + preStatus.Devices[1].Addrs = append(preStatus.Devices[1].Addrs, + net.IPNet{ + IP: net.ParseIP("fd00::10"), + Mask: net.CIDRMask(64, 128), + }) + + err := setup(targetNs, preStatus) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred()) + + newStatus, err := readback(targetNs, []string{"net1", "eth0"}) + Expect(err).NotTo(HaveOccurred()) + + // Check that both IPv4 and IPv6 rules were created + Expect(newStatus.Rules).To(HaveLen(2)) + devNet1 := newStatus.Devices[0] + + // Find both default routes + var foundIPv4Route, foundIPv6Route bool + for _, route := range devNet1.Routes { + if route.Table >= 100 && route.Dst != nil { + if route.Dst.IP.Equal(net.IPv4zero) && route.Gw != nil { + Expect(route.Gw.String()).To(Equal("192.168.1.254")) + foundIPv4Route = true + } + if route.Dst.IP.Equal(net.IPv6zero) && route.Gw != nil { + Expect(route.Gw.String()).To(Equal("fd00::1")) + foundIPv6Route = true + } + } + } + Expect(foundIPv4Route).To(BeTrue(), "Expected to find IPv4 default route with static gateway") + Expect(foundIPv6Route).To(BeTrue(), "Expected to find IPv6 default route with static gateway") + }) + + It("Rejects multiple IPv4 gateways", func() { + ifname := "net1" + // Configure with two IPv4 gateways (should fail) + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-sbr-test", + "type": "sbr", + "gateways": ["192.168.1.1", "192.168.1.254"], + "prevResult": { + "cniVersion": "0.3.0", + "interfaces": [ + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "192.168.1.209/24", + "gateway": "192.168.1.1", + "interface": 0 + } + ], + "routes": [] + } +}` + conf = fmt.Sprintf(conf, ifname, targetNs.Path()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNs.Path(), + IfName: ifname, + StdinData: []byte(conf), + } + + preStatus := createDefaultStatus() + err := setup(targetNs, preStatus) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("multiple IPv4 addresses")) + }) + + It("Works with addSourceHints enabled", func() { + ifname := "net1" + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-sbr-test", + "type": "sbr", + "addSourceHints": true, + "prevResult": { + "cniVersion": "0.3.0", + "interfaces": [ + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "192.168.1.209/24", + "gateway": "192.168.1.1", + "interface": 0 + } + ], + "routes": [] + } +}` + conf = fmt.Sprintf(conf, ifname, targetNs.Path()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: targetNs.Path(), + IfName: ifname, + StdinData: []byte(conf), + } + + err := setup(targetNs, createDefaultStatus()) + Expect(err).NotTo(HaveOccurred()) + + _, _, err = testutils.CmdAddWithArgs(args, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred()) + + newStatus, err := readback(targetNs, []string{"net1", "eth0"}) + Expect(err).NotTo(HaveOccurred()) + + // Check that source-based routing rule was created + Expect(newStatus.Rules).To(HaveLen(1)) + Expect(newStatus.Rules[0].Table).To(Equal(100)) + Expect(newStatus.Rules[0].Src.String()).To(Equal("192.168.1.209/32")) + + devNet1 := newStatus.Devices[0] + + // Check that routes exist in both table 100 (source-based) and + // table 254/main (for destination-based routing) + var foundSubnetRouteInMain bool + var foundSubnetRouteInTable100 bool + var foundDefaultRouteInTable100 bool + + for _, route := range devNet1.Routes { + // Look for subnet route (192.168.1.0/24) in main table with source hint + if route.Dst != nil && route.Dst.IP.Equal(net.IPv4(192, 168, 1, 0)) && + route.Dst.Mask.String() == "ffffff00" { + if route.Table == 254 || route.Table == 0 { // 0 is treated as main + Expect(route.Src.String()).To(Equal("192.168.1.209"), + "Subnet route in main table should have source IP hint") + Expect(route.Scope).To(Equal(netlink.SCOPE_LINK)) + foundSubnetRouteInMain = true + } + if route.Table == 100 { + foundSubnetRouteInTable100 = true + } + } + + // Look for default route in table 100 + if route.Table == 100 && route.Dst != nil && route.Dst.IP.Equal(net.IPv4zero) { + Expect(route.Gw.String()).To(Equal("192.168.1.1")) + foundDefaultRouteInTable100 = true + } + } + + Expect(foundSubnetRouteInMain).To(BeTrue(), + "Expected subnet route in main table with source hint") + Expect(foundSubnetRouteInTable100).To(BeTrue(), + "Expected subnet route in table 100 for source-based routing") + Expect(foundDefaultRouteInTable100).To(BeTrue(), + "Expected default route in table 100") + }) })