diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go index 9623d1d0a8b..79ccbc80bff 100644 --- a/mocks/pkg/ovs/interface.go +++ b/mocks/pkg/ovs/interface.go @@ -2058,6 +2058,20 @@ func (mr *MockAddressSetMockRecorder) AddressSetUpdateAddress(asName any, addres return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddressSetUpdateAddress", reflect.TypeOf((*MockAddressSet)(nil).AddressSetUpdateAddress), varargs...) } +// BatchDeleteAddressSetByNames mocks base method. +func (m *MockAddressSet) BatchDeleteAddressSetByNames(asNames []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BatchDeleteAddressSetByNames", asNames) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchDeleteAddressSetByNames indicates an expected call of BatchDeleteAddressSetByNames. +func (mr *MockAddressSetMockRecorder) BatchDeleteAddressSetByNames(asNames any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchDeleteAddressSetByNames", reflect.TypeOf((*MockAddressSet)(nil).BatchDeleteAddressSetByNames), asNames) +} + // CreateAddressSet mocks base method. func (m *MockAddressSet) CreateAddressSet(asName string, externalIDs map[string]string) error { m.ctrl.T.Helper() @@ -2162,6 +2176,20 @@ func (mr *MockLogicalRouterStaticRouteMockRecorder) AddLogicalRouterStaticRoute( return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterStaticRoute", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).AddLogicalRouterStaticRoute), varargs...) } +// BatchDeleteLogicalRouterStaticRoute mocks base method. +func (m *MockLogicalRouterStaticRoute) BatchDeleteLogicalRouterStaticRoute(lrName string, staticRoutes []*ovnnb.LogicalRouterStaticRoute) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BatchDeleteLogicalRouterStaticRoute", lrName, staticRoutes) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchDeleteLogicalRouterStaticRoute indicates an expected call of BatchDeleteLogicalRouterStaticRoute. +func (mr *MockLogicalRouterStaticRouteMockRecorder) BatchDeleteLogicalRouterStaticRoute(lrName, staticRoutes any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchDeleteLogicalRouterStaticRoute", reflect.TypeOf((*MockLogicalRouterStaticRoute)(nil).BatchDeleteLogicalRouterStaticRoute), lrName, staticRoutes) +} + // ClearLogicalRouterStaticRoute mocks base method. func (m *MockLogicalRouterStaticRoute) ClearLogicalRouterStaticRoute(lrName string) error { m.ctrl.T.Helper() @@ -2320,6 +2348,58 @@ func (mr *MockLogicalRouterPolicyMockRecorder) AddLogicalRouterPolicy(lrName, pr return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLogicalRouterPolicy", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).AddLogicalRouterPolicy), lrName, priority, match, action, nextHops, bfdSessions, externalIDs) } +// BatchAddLogicalRouterPolicy mocks base method. +func (m *MockLogicalRouterPolicy) BatchAddLogicalRouterPolicy(lrName string, policies ...*ovnnb.LogicalRouterPolicy) error { + m.ctrl.T.Helper() + varargs := []any{lrName} + for _, a := range policies { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "BatchAddLogicalRouterPolicy", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchAddLogicalRouterPolicy indicates an expected call of BatchAddLogicalRouterPolicy. +func (mr *MockLogicalRouterPolicyMockRecorder) BatchAddLogicalRouterPolicy(lrName any, policies ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{lrName}, policies...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchAddLogicalRouterPolicy", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).BatchAddLogicalRouterPolicy), varargs...) +} + +// BatchDeleteLogicalRouterPolicy mocks base method. +func (m *MockLogicalRouterPolicy) BatchDeleteLogicalRouterPolicy(lrName string, logicalRouteRolicies []*ovnnb.LogicalRouterPolicy) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BatchDeleteLogicalRouterPolicy", lrName, logicalRouteRolicies) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchDeleteLogicalRouterPolicy indicates an expected call of BatchDeleteLogicalRouterPolicy. +func (mr *MockLogicalRouterPolicyMockRecorder) BatchDeleteLogicalRouterPolicy(lrName, logicalRouteRolicies any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchDeleteLogicalRouterPolicy", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).BatchDeleteLogicalRouterPolicy), lrName, logicalRouteRolicies) +} + +// BatchDeleteLogicalRouterPolicyByUUID mocks base method. +func (m *MockLogicalRouterPolicy) BatchDeleteLogicalRouterPolicyByUUID(lrName string, uuidList ...string) error { + m.ctrl.T.Helper() + varargs := []any{lrName} + for _, a := range uuidList { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "BatchDeleteLogicalRouterPolicyByUUID", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchDeleteLogicalRouterPolicyByUUID indicates an expected call of BatchDeleteLogicalRouterPolicyByUUID. +func (mr *MockLogicalRouterPolicyMockRecorder) BatchDeleteLogicalRouterPolicyByUUID(lrName any, uuidList ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{lrName}, uuidList...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchDeleteLogicalRouterPolicyByUUID", reflect.TypeOf((*MockLogicalRouterPolicy)(nil).BatchDeleteLogicalRouterPolicyByUUID), varargs...) +} + // ClearLogicalRouterPolicy mocks base method. func (m *MockLogicalRouterPolicy) ClearLogicalRouterPolicy(lrName string) error { m.ctrl.T.Helper() @@ -2783,6 +2863,86 @@ func (mr *MockNbClientMockRecorder) AddressSetUpdateAddress(asName any, addresse return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddressSetUpdateAddress", reflect.TypeOf((*MockNbClient)(nil).AddressSetUpdateAddress), varargs...) } +// BatchAddLogicalRouterPolicy mocks base method. +func (m *MockNbClient) BatchAddLogicalRouterPolicy(lrName string, policies ...*ovnnb.LogicalRouterPolicy) error { + m.ctrl.T.Helper() + varargs := []any{lrName} + for _, a := range policies { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "BatchAddLogicalRouterPolicy", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchAddLogicalRouterPolicy indicates an expected call of BatchAddLogicalRouterPolicy. +func (mr *MockNbClientMockRecorder) BatchAddLogicalRouterPolicy(lrName any, policies ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{lrName}, policies...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchAddLogicalRouterPolicy", reflect.TypeOf((*MockNbClient)(nil).BatchAddLogicalRouterPolicy), varargs...) +} + +// BatchDeleteAddressSetByNames mocks base method. +func (m *MockNbClient) BatchDeleteAddressSetByNames(asNames []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BatchDeleteAddressSetByNames", asNames) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchDeleteAddressSetByNames indicates an expected call of BatchDeleteAddressSetByNames. +func (mr *MockNbClientMockRecorder) BatchDeleteAddressSetByNames(asNames any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchDeleteAddressSetByNames", reflect.TypeOf((*MockNbClient)(nil).BatchDeleteAddressSetByNames), asNames) +} + +// BatchDeleteLogicalRouterPolicy mocks base method. +func (m *MockNbClient) BatchDeleteLogicalRouterPolicy(lrName string, logicalRouteRolicies []*ovnnb.LogicalRouterPolicy) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BatchDeleteLogicalRouterPolicy", lrName, logicalRouteRolicies) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchDeleteLogicalRouterPolicy indicates an expected call of BatchDeleteLogicalRouterPolicy. +func (mr *MockNbClientMockRecorder) BatchDeleteLogicalRouterPolicy(lrName, logicalRouteRolicies any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchDeleteLogicalRouterPolicy", reflect.TypeOf((*MockNbClient)(nil).BatchDeleteLogicalRouterPolicy), lrName, logicalRouteRolicies) +} + +// BatchDeleteLogicalRouterPolicyByUUID mocks base method. +func (m *MockNbClient) BatchDeleteLogicalRouterPolicyByUUID(lrName string, uuidList ...string) error { + m.ctrl.T.Helper() + varargs := []any{lrName} + for _, a := range uuidList { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "BatchDeleteLogicalRouterPolicyByUUID", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchDeleteLogicalRouterPolicyByUUID indicates an expected call of BatchDeleteLogicalRouterPolicyByUUID. +func (mr *MockNbClientMockRecorder) BatchDeleteLogicalRouterPolicyByUUID(lrName any, uuidList ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{lrName}, uuidList...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchDeleteLogicalRouterPolicyByUUID", reflect.TypeOf((*MockNbClient)(nil).BatchDeleteLogicalRouterPolicyByUUID), varargs...) +} + +// BatchDeleteLogicalRouterStaticRoute mocks base method. +func (m *MockNbClient) BatchDeleteLogicalRouterStaticRoute(lrName string, staticRoutes []*ovnnb.LogicalRouterStaticRoute) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BatchDeleteLogicalRouterStaticRoute", lrName, staticRoutes) + ret0, _ := ret[0].(error) + return ret0 +} + +// BatchDeleteLogicalRouterStaticRoute indicates an expected call of BatchDeleteLogicalRouterStaticRoute. +func (mr *MockNbClientMockRecorder) BatchDeleteLogicalRouterStaticRoute(lrName, staticRoutes any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchDeleteLogicalRouterStaticRoute", reflect.TypeOf((*MockNbClient)(nil).BatchDeleteLogicalRouterStaticRoute), lrName, staticRoutes) +} + // CleanLogicalSwitchPortMigrateOptions mocks base method. func (m *MockNbClient) CleanLogicalSwitchPortMigrateOptions(lspName string) error { m.ctrl.T.Helper() diff --git a/pkg/controller/init.go b/pkg/controller/init.go index d278212a900..50170d97c0f 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -775,81 +775,91 @@ func (c *Controller) syncVlanCR() error { return nil } -func (c *Controller) migrateNodeRoute(af int, node, ip, nexthop string) error { - // default vpc use static route in old version, migrate to policy route - var ( - match = fmt.Sprintf("ip%d.dst == %s", af, ip) - action = kubeovnv1.PolicyRouteActionReroute - externalIDs = map[string]string{ - "vendor": util.CniTypeName, - "node": node, +func (c *Controller) batchMigrateNodeRoute(nodes []*v1.Node) error { + start := time.Now() + addPolicies := make([]*kubeovnv1.PolicyRoute, 0) + delPolicies := make([]*kubeovnv1.PolicyRoute, 0) + staticRoutes := make([]*kubeovnv1.StaticRoute, 0) + externalIDsMap := make(map[string]map[string]string) + delAsNames := make([]string, 0) + for _, node := range nodes { + if node.Annotations[util.AllocatedAnnotation] != "true" { + continue + } + nodeName := node.Name + nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(*node) + joinAddrV4, joinAddrV6 := util.SplitStringIP(node.Annotations[util.IPAddressAnnotation]) + if nodeIPv4 != "" && joinAddrV4 != "" { + buildNodeRoute(4, nodeName, joinAddrV4, nodeIPv4, &addPolicies, &delPolicies, &staticRoutes, externalIDsMap, &delAsNames) + } + if nodeIPv6 != "" && joinAddrV6 != "" { + buildNodeRoute(6, nodeName, joinAddrV6, nodeIPv6, &addPolicies, &delPolicies, &staticRoutes, externalIDsMap, &delAsNames) } - ) - klog.V(3).Infof("add policy route for router: %s, priority: %d, match %s, action %s, nexthop %s, externalID %v", - c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, action, nexthop, externalIDs) - if err := c.addPolicyRouteToVpc( - c.config.ClusterRouter, - &kubeovnv1.PolicyRoute{ - Priority: util.NodeRouterPolicyPriority, - Match: match, - Action: action, - NextHopIP: nexthop, - }, - externalIDs, - ); err != nil { - klog.Errorf("failed to add logical router policy for node %s: %v", node, err) - return err } - if err := c.deleteStaticRouteFromVpc( - c.config.ClusterRouter, - util.MainRouteTable, - ip, - "", - kubeovnv1.PolicyDst, - ); err != nil { - klog.Errorf("failed to delete obsolete static route for node %s: %v", node, err) + if err := c.batchAddPolicyRouteToVpc(c.config.ClusterRouter, addPolicies, externalIDsMap); err != nil { + klog.Errorf("failed to batch add logical router policy for lr %s nodes %d: %v", c.config.ClusterRouter, len(nodes), err) return err } - - asName := nodeUnderlayAddressSetName(node, af) - obsoleteMatch := fmt.Sprintf("ip%d.dst == %s && ip%d.src != $%s", af, ip, af, asName) - klog.V(3).Infof("delete policy route for router: %s, priority: %d, match %s", c.config.ClusterRouter, util.NodeRouterPolicyPriority, obsoleteMatch) - if err := c.deletePolicyRouteFromVpc(c.config.ClusterRouter, util.NodeRouterPolicyPriority, obsoleteMatch); err != nil { - klog.Errorf("failed to delete obsolete logical router policy for node %s: %v", node, err) + if err := c.batchDeleteStaticRouteFromVpc(c.config.ClusterRouter, staticRoutes); err != nil { + klog.Errorf("failed to batch delete obsolete logical router static route for lr %s nodes %d: %v", c.config.ClusterRouter, len(nodes), err) return err } - - if err := c.OVNNbClient.DeleteAddressSet(asName); err != nil { - klog.Errorf("delete obsolete address set %s for node %s: %v", asName, node, err) + if err := c.batchDeletePolicyRouteFromVpc(c.config.ClusterRouter, delPolicies); err != nil { + klog.Errorf("failed to batch delete obsolete logical router policy for lr %s nodes %d: %v", c.config.ClusterRouter, len(nodes), err) return err } + if err := c.OVNNbClient.BatchDeleteAddressSetByNames(delAsNames); err != nil { + klog.Errorf("failed to batch delete obsolete address set for asNames %v nodes %d: %v", delAsNames, len(nodes), err) + return err + } + klog.V(3).Infof("take to %v batch migrate node route for router: %s priority: %d add policy len: %d extrenalID len: %d del policy len: %d del address set len: %d", + time.Since(start), c.config.ClusterRouter, util.NodeRouterPolicyPriority, len(addPolicies), len(externalIDsMap), len(delPolicies), len(delAsNames)) return nil } +func buildNodeRoute(af int, nodeName, nexthop, ip string, addPolicies, delPolicies *[]*kubeovnv1.PolicyRoute, staticRoutes *[]*kubeovnv1.StaticRoute, externalIDsMap map[string]map[string]string, delAsNames *[]string) { + var ( + match = fmt.Sprintf("ip%d.dst == %s", af, ip) + action = kubeovnv1.PolicyRouteActionReroute + externalIDs = map[string]string{ + "vendor": util.CniTypeName, + "node": nodeName, + } + ) + *addPolicies = append(*addPolicies, &kubeovnv1.PolicyRoute{ + Priority: util.NodeRouterPolicyPriority, + Match: match, + Action: action, + NextHopIP: nexthop, + }) + externalIDsMap[buildExternalIDsMapKey(match, string(action), util.NodeRouterPolicyPriority)] = externalIDs + *staticRoutes = append(*staticRoutes, &kubeovnv1.StaticRoute{ + Policy: kubeovnv1.PolicyDst, + RouteTable: util.MainRouteTable, + NextHopIP: "", + CIDR: ip, + }) + asName := nodeUnderlayAddressSetName(nodeName, af) + obsoleteMatch := fmt.Sprintf("ip%d.dst == %s && ip%d.src != $%s", af, ip, af, asName) + *delPolicies = append(*delPolicies, &kubeovnv1.PolicyRoute{ + Match: obsoleteMatch, + Priority: util.NodeRouterPolicyPriority, + }) + *delAsNames = append(*delAsNames, asName) +} + func (c *Controller) syncNodeRoutes() error { nodes, err := c.nodesLister.List(labels.Everything()) if err != nil { klog.Errorf("failed to list nodes: %v", err) return err } - for _, node := range nodes { - if node.Annotations[util.AllocatedAnnotation] != "true" { - continue - } - nodeIPv4, nodeIPv6 := util.GetNodeInternalIP(*node) - joinAddrV4, joinAddrV6 := util.SplitStringIP(node.Annotations[util.IPAddressAnnotation]) - if nodeIPv4 != "" && joinAddrV4 != "" { - if err = c.migrateNodeRoute(4, node.Name, nodeIPv4, joinAddrV4); err != nil { - klog.Errorf("failed to migrate IPv4 route for node %s: %v", node.Name, err) - } - } - if nodeIPv6 != "" && joinAddrV6 != "" { - if err = c.migrateNodeRoute(6, node.Name, nodeIPv6, joinAddrV6); err != nil { - klog.Errorf("failed to migrate IPv6 route for node %s: %v", node.Name, err) - } - } + + if err := c.batchMigrateNodeRoute(nodes); err != nil { + klog.Errorf("failed to batch migrate node routes: %v", err) + return err } if err := c.addNodeGatewayStaticRoute(); err != nil { diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index d12f8cecb7f..499fea79a08 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -11,6 +11,7 @@ import ( "slices" "sort" "strings" + "time" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -761,6 +762,38 @@ func (c *Controller) addPolicyRouteToVpc(vpcName string, policy *kubeovnv1.Polic return nil } +func buildExternalIDsMapKey(match, action string, priority int) string { + return fmt.Sprintf("%s-%s-%d", match, action, priority) +} + +func (c *Controller) batchAddPolicyRouteToVpc(name string, policies []*kubeovnv1.PolicyRoute, externalIDs map[string]map[string]string) error { + if len(policies) == 0 { + return nil + } + start := time.Now() + routerPolicies := make([]*ovnnb.LogicalRouterPolicy, 0, len(policies)) + for _, policy := range policies { + var nextHops []string + if policy.NextHopIP != "" { + nextHops = strings.Split(policy.NextHopIP, ",") + } + routerPolicies = append(routerPolicies, &ovnnb.LogicalRouterPolicy{ + Priority: policy.Priority, + Nexthops: nextHops, + Action: string(policy.Action), + Match: policy.Match, + ExternalIDs: externalIDs[buildExternalIDsMapKey(policy.Match, string(policy.Action), policy.Priority)], + }) + } + + if err := c.OVNNbClient.BatchAddLogicalRouterPolicy(name, routerPolicies...); err != nil { + klog.Errorf("batch add policy route to vpc %s failed, %v", name, err) + return err + } + klog.Infof("take to %v batch add policy route to vpc %s policies %d", time.Since(start), name, len(policies)) + return nil +} + func (c *Controller) deletePolicyRouteFromVpc(vpcName string, priority int, match string) error { var ( vpc, cachedVpc *kubeovnv1.Vpc @@ -790,6 +823,44 @@ func (c *Controller) deletePolicyRouteFromVpc(vpcName string, priority int, matc return nil } +func (c *Controller) batchDeletePolicyRouteFromVpc(name string, policies []*kubeovnv1.PolicyRoute) error { + var ( + vpc, cachedVpc *kubeovnv1.Vpc + err error + ) + + start := time.Now() + routerPolicies := make([]*ovnnb.LogicalRouterPolicy, 0, len(policies)) + for _, policy := range policies { + routerPolicies = append(routerPolicies, &ovnnb.LogicalRouterPolicy{ + Priority: policy.Priority, + Match: policy.Match, + }) + } + + if err = c.OVNNbClient.BatchDeleteLogicalRouterPolicy(name, routerPolicies); err != nil { + return err + } + klog.V(3).Infof("take to %v batch delete policy route from vpc %s policies %d", time.Since(start), name, len(policies)) + + cachedVpc, err = c.vpcsLister.Get(name) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + klog.Error(err) + return err + } + vpc = cachedVpc.DeepCopy() + // make sure custom policies not be deleted + _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}) + if err != nil { + klog.Error(err) + return err + } + return nil +} + func (c *Controller) addStaticRouteToVpc(name string, route *kubeovnv1.StaticRoute) error { if route.BfdID != "" { klog.Infof("vpc %s add static ecmp route: %+v", name, route) @@ -826,6 +897,48 @@ func (c *Controller) deleteStaticRouteFromVpc(name, table, cidr, nextHop string, return nil } +func (c *Controller) batchDeleteStaticRouteFromVpc(name string, staticRoutes []*kubeovnv1.StaticRoute) error { + var ( + vpc, cachedVpc *kubeovnv1.Vpc + err error + ) + start := time.Now() + routeCount := len(staticRoutes) + delRoutes := make([]*ovnnb.LogicalRouterStaticRoute, 0, routeCount) + for _, sr := range staticRoutes { + policyStr := convertPolicy(sr.Policy) + newRoute := &ovnnb.LogicalRouterStaticRoute{ + RouteTable: sr.RouteTable, + Nexthop: sr.NextHopIP, + Policy: &policyStr, + IPPrefix: sr.CIDR, + } + delRoutes = append(delRoutes, newRoute) + } + if err = c.OVNNbClient.BatchDeleteLogicalRouterStaticRoute(name, delRoutes); err != nil { + klog.Errorf("batch del vpc %s static route %d failed, %v", name, routeCount, err) + return err + } + klog.V(3).Infof("take to %v batch delete static route from vpc %s static routes %d", time.Since(start), name, len(delRoutes)) + + cachedVpc, err = c.vpcsLister.Get(name) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + klog.Error(err) + return err + } + vpc = cachedVpc.DeepCopy() + // make sure custom policies not be deleted + _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}) + if err != nil { + klog.Error(err) + return err + } + return nil +} + func diffPolicyRouteWithExisted(exists, target []*kubeovnv1.PolicyRoute) ([]*kubeovnv1.PolicyRoute, []*kubeovnv1.PolicyRoute) { var ( dels, adds []*kubeovnv1.PolicyRoute diff --git a/pkg/ovs/interface.go b/pkg/ovs/interface.go index d07636d4069..7bd5337762d 100644 --- a/pkg/ovs/interface.go +++ b/pkg/ovs/interface.go @@ -173,6 +173,7 @@ type AddressSet interface { DeleteAddressSet(asName ...string) error DeleteAddressSets(externalIDs map[string]string) error ListAddressSets(externalIDs map[string]string) ([]ovnnb.AddressSet, error) + BatchDeleteAddressSetByNames(asNames []string) error } type LogicalRouterStaticRoute interface { @@ -185,6 +186,7 @@ type LogicalRouterStaticRoute interface { ListLogicalRouterStaticRoutesByOption(lrName, routeTable, key, value string) ([]*ovnnb.LogicalRouterStaticRoute, error) ListLogicalRouterStaticRoutes(lrName string, routeTable, policy *string, ipPrefix string, externalIDs map[string]string) ([]*ovnnb.LogicalRouterStaticRoute, error) LogicalRouterStaticRouteExists(lrName, routeTable, policy, ipPrefix, nexthop string) (bool, error) + BatchDeleteLogicalRouterStaticRoute(lrName string, staticRoutes []*ovnnb.LogicalRouterStaticRoute) error } type LogicalRouterPolicy interface { @@ -198,6 +200,9 @@ type LogicalRouterPolicy interface { GetLogicalRouterPolicy(lrName string, priority int, match string, ignoreNotFound bool) ([]*ovnnb.LogicalRouterPolicy, error) GetLogicalRouterPoliciesByExtID(lrName, key, value string) ([]*ovnnb.LogicalRouterPolicy, error) UpdateLogicalRouterPolicy(policy *ovnnb.LogicalRouterPolicy, fields ...interface{}) error + BatchAddLogicalRouterPolicy(lrName string, policies ...*ovnnb.LogicalRouterPolicy) error + BatchDeleteLogicalRouterPolicyByUUID(lrName string, uuidList ...string) error + BatchDeleteLogicalRouterPolicy(lrName string, logicalRouteRolicies []*ovnnb.LogicalRouterPolicy) error } type NAT interface { diff --git a/pkg/ovs/ovn-nb-address_set.go b/pkg/ovs/ovn-nb-address_set.go index 235a0056cc7..d1871a5deff 100644 --- a/pkg/ovs/ovn-nb-address_set.go +++ b/pkg/ovs/ovn-nb-address_set.go @@ -142,6 +142,46 @@ func (c *OVNNbClient) DeleteAddressSet(asName ...string) error { return nil } +// BatchDeleteAddressSetByAsName batch delete address set by names +func (c *OVNNbClient) BatchDeleteAddressSetByNames(asNames []string) error { + asNameMap := make(map[string]struct{}, len(asNames)) + for _, name := range asNames { + asNameMap[name] = struct{}{} + } + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) + defer cancel() + + asList := make([]ovnnb.AddressSet, 0) + if err := c.ovsDbClient.WhereCache(func(as *ovnnb.AddressSet) bool { + _, exist := asNameMap[as.Name] + return exist + }).List(ctx, &asList); err != nil { + klog.Error(err) + return fmt.Errorf("batch delete address set %d list failed: %w", len(asNames), err) + } + + // not found, skip + if len(asList) == 0 { + return nil + } + + var modelList []model.Model = make([]model.Model, 0, len(asList)) + for _, as := range asList { + modelList = append(modelList, &as) + } + op, err := c.Where(modelList...).Delete() + if err != nil { + klog.Error(err) + return fmt.Errorf("batch delete address set %d op failed: %w", len(asList), err) + } + + if err := c.Transact("as-del", op); err != nil { + return fmt.Errorf("batch delete address set %d failed: %w", len(asList), err) + } + + return nil +} + // DeleteAddressSets delete several address set once func (c *OVNNbClient) DeleteAddressSets(externalIDs map[string]string) error { // it's dangerous when externalIDs is empty, it will delete all address set diff --git a/pkg/ovs/ovn-nb-address_set_test.go b/pkg/ovs/ovn-nb-address_set_test.go index a003033931e..6bb0df652bb 100644 --- a/pkg/ovs/ovn-nb-address_set_test.go +++ b/pkg/ovs/ovn-nb-address_set_test.go @@ -347,3 +347,30 @@ func (suite *OvnClientTestSuite) testUpdateAddressSet() { require.Contains(t, err.Error(), "address_set is nil") }) } + +func (suite *OvnClientTestSuite) testBatchDeleteAddressSetByNames() { + t := suite.T() + t.Parallel() + + nbClient := suite.ovnNBClient + asName := "test_batch_delete_as" + + t.Run("no err when delete existent address set", func(t *testing.T) { + t.Parallel() + + err := nbClient.CreateAddressSet(asName, nil) + require.NoError(t, err) + + err = nbClient.BatchDeleteAddressSetByNames([]string{asName}) + require.NoError(t, err) + + _, err = nbClient.GetAddressSet(asName, false) + require.ErrorContains(t, err, "object not found") + }) + + t.Run("no err when delete non-existent address set", func(t *testing.T) { + t.Parallel() + err := nbClient.BatchDeleteAddressSetByNames([]string{"test-batch-delete-as-non-existent"}) + require.NoError(t, err) + }) +} diff --git a/pkg/ovs/ovn-nb-logical_router_policy.go b/pkg/ovs/ovn-nb-logical_router_policy.go index c8427bb4fdb..d9d3185e94b 100644 --- a/pkg/ovs/ovn-nb-logical_router_policy.go +++ b/pkg/ovs/ovn-nb-logical_router_policy.go @@ -6,6 +6,7 @@ import ( "fmt" "maps" "slices" + "time" "github.com/ovn-org/libovsdb/model" "github.com/ovn-org/libovsdb/ovsdb" @@ -78,6 +79,58 @@ func (c *OVNNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, return nil } +// BatchAddLogicalRouterPolicy batch add a policy route to logical router +func (c *OVNNbClient) BatchAddLogicalRouterPolicy(lrName string, policies ...*ovnnb.LogicalRouterPolicy) error { + if len(policies) == 0 { + return nil + } + + start := time.Now() + var ( + needDelete []string + needCreatePolicy []*ovnnb.LogicalRouterPolicy + needUpdatePolicy = make(map[*ovnnb.LogicalRouterPolicy]*ovnnb.LogicalRouterPolicy) + ) + policyListMap, err := c.batchListLogicalRouterPoliciesByFilter(lrName, policies...) + if err != nil { + return fmt.Errorf("batch list logical router %s policies %d: %w", lrName, len(policies), err) + } + + for policy, policyList := range policyListMap { + if len(policyList) == 0 { + needCreatePolicy = append(needCreatePolicy, policy) + continue + } + duplicate, policyFound := c.matchLogicalRouterPolicies(policy, policyList) + if policyFound == nil { + needCreatePolicy = append(needCreatePolicy, policy) + } else if !maps.Equal(policyFound.ExternalIDs, policy.ExternalIDs) { + needUpdatePolicy[policy] = policyFound + } + if len(duplicate) > 0 { + needDelete = append(needDelete, duplicate...) + } + } + klog.Infof("take to %vms batch add logical router %s list policy del %d create %d update %d", time.Since(start).Milliseconds(), lrName, len(needDelete), len(needCreatePolicy), len(needUpdatePolicy)) + if len(needDelete) > 0 { + if err := c.BatchDeleteLogicalRouterPolicyByUUID(lrName, needDelete...); err != nil { + return err + } + } + if len(needCreatePolicy) > 0 { + if err := c.batchCreateLogicalRouterPolicies(lrName, needCreatePolicy); err != nil { + return err + } + } + if len(needUpdatePolicy) > 0 { + if err := c.batchUpdatetLogicalRouterPolicies(needUpdatePolicy); err != nil { + return err + } + } + klog.Infof("take to %vms batch add logical router %s policy %d", time.Since(start).Milliseconds(), lrName, len(policies)) + return nil +} + // CreateLogicalRouterPolicies create several logical router policy once func (c *OVNNbClient) CreateLogicalRouterPolicies(lrName string, policies ...*ovnnb.LogicalRouterPolicy) error { if len(policies) == 0 { @@ -135,6 +188,41 @@ func (c *OVNNbClient) DeleteLogicalRouterPolicy(lrName string, priority int, mat return nil } +// DeleteLogicalRouterPolicy delete policy from logical router +func (c *OVNNbClient) BatchDeleteLogicalRouterPolicy(lrName string, logicalRouteRolicies []*ovnnb.LogicalRouterPolicy) error { + if len(logicalRouteRolicies) == 0 { + return nil + } + + policyListMap, err := c.batchListLogicalRouterPoliciesByFilter(lrName, logicalRouteRolicies...) + if err != nil { + klog.Error(err) + return err + } + + uuidList := make([]string, 0) + for _, policyList := range policyListMap { + if len(policyList) == 0 { + continue + } + for _, p := range policyList { + uuidList = append(uuidList, p.UUID) + } + } + + // not found,skip + if len(uuidList) == 0 { + return nil + } + + if err := c.BatchDeleteLogicalRouterPolicyByUUID(lrName, uuidList...); err != nil { + klog.Error(err) + return err + } + + return nil +} + // DeleteLogicalRouterPolicy delete some policies from logical router once func (c *OVNNbClient) DeleteLogicalRouterPolicies(lrName string, priority int, externalIDs map[string]string) error { // remove policies from logical router @@ -178,6 +266,28 @@ func (c *OVNNbClient) DeleteLogicalRouterPolicyByUUID(lrName, uuid string) error return nil } +// BatchDeleteLogicalRouterPolicyByUUID batch remove policy from logical router +func (c *OVNNbClient) BatchDeleteLogicalRouterPolicyByUUID(lrName string, uuidList ...string) error { + if len(uuidList) == 0 { + return nil + } + start := time.Now() + ops, err := c.LogicalRouterUpdatePolicyOp(lrName, uuidList, ovsdb.MutateOperationDelete) + if err != nil { + err := fmt.Errorf("generate operations for removing policies '%v' from logical router %s: %w", uuidList, lrName, err) + klog.Error(err) + return err + } + + if err = c.Transact("lr-policy-del", ops); err != nil { + err := fmt.Errorf("delete logical router policies '%v' from logical router %s: %w", uuidList, lrName, err) + klog.Error(err) + return err + } + klog.V(3).Infof("take to %vms batch delete logical router policies %s uuid %v", time.Since(start).Milliseconds(), lrName, uuidList) + return nil +} + func (c *OVNNbClient) DeleteLogicalRouterPolicyByNexthop(lrName string, priority int, nexthop string) error { policyList, err := c.listLogicalRouterPoliciesByFilter(lrName, func(route *ovnnb.LogicalRouterPolicy) bool { if route.Priority != priority { @@ -379,3 +489,123 @@ func (c *OVNNbClient) listLogicalRouterPoliciesByFilter(lrName string, filter fu return policyList, nil } + +func (c *OVNNbClient) batchListLogicalRouterPoliciesByFilter(lrName string, policies ...*ovnnb.LogicalRouterPolicy) (map[*ovnnb.LogicalRouterPolicy][]*ovnnb.LogicalRouterPolicy, error) { + start := time.Now() + lr, err := c.GetLogicalRouter(lrName, false) + if err != nil { + klog.Error(err) + return nil, err + } + lrPolicySet := set.New(lr.Policies...) + + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) + defer cancel() + policyIndex := make([]model.Model, 0) + for _, p := range policies { + policyIndex = append(policyIndex, buildLogicalRouterPolicyIndex(p.Priority, p.Match)) + } + + var policyList []*ovnnb.LogicalRouterPolicy + indexStart := time.Now() + if err := c.ovsDbClient.Where(policyIndex...).List(ctx, &policyList); err != nil { + klog.Error(err) + return nil, err + } + klog.Infof("take to %v batch list logical router policy %s incoming policies len %v query policies len %v by client index", time.Since(indexStart), lrName, len(policies), len(policyList)) + + policySet := make(map[string]*ovnnb.LogicalRouterPolicy) + for _, policy := range policies { + key := createPolicyKey(policy.Priority, policy.Match) + policySet[key] = policy + } + + policyMapByUUID := make(map[string][]*ovnnb.LogicalRouterPolicy) + for _, policy := range policyList { + if lrPolicySet.Has(policy.UUID) { + key := createPolicyKey(policy.Priority, policy.Match) + policyMapByUUID[key] = append(policyMapByUUID[key], policy) + } + } + + policyListMap := make(map[*ovnnb.LogicalRouterPolicy][]*ovnnb.LogicalRouterPolicy) + for policyKey, policy := range policySet { + if matchingPolicies, found := policyMapByUUID[policyKey]; found { + policyListMap[policy] = append(policyListMap[policy], matchingPolicies...) + } else { + policyListMap[policy] = []*ovnnb.LogicalRouterPolicy{} + } + } + + elapsed := float64((time.Since(start)) / time.Millisecond) + if elapsed > 500 { + klog.Infof("take to %vms batch list logical router policy %s policies %d query result policies %d nb policies len %d", elapsed, lrName, len(policies), len(policyList), len(policyListMap)) + } + return policyListMap, nil +} + +func (c *OVNNbClient) matchLogicalRouterPolicies(policy *ovnnb.LogicalRouterPolicy, policyList []*ovnnb.LogicalRouterPolicy) ([]string, *ovnnb.LogicalRouterPolicy) { + var ( + duplicate []string + policyFound *ovnnb.LogicalRouterPolicy + ) + + for _, policyOld := range policyList { + if policyOld.Action != policy.Action || (policyOld.Action == ovnnb.LogicalRouterPolicyActionReroute && !strset.New(policy.Nexthops...).IsEqual(strset.New(policyOld.Nexthops...))) { + duplicate = append(duplicate, policyOld.UUID) + continue + } + if policyFound != nil { + duplicate = append(duplicate, policyFound.UUID) + } else { + policyFound = policyOld + } + } + + return duplicate, policyFound +} + +func (c *OVNNbClient) batchCreateLogicalRouterPolicies(lrName string, policies []*ovnnb.LogicalRouterPolicy) error { + routerPolicies := make([]*ovnnb.LogicalRouterPolicy, 0, len(policies)) + for _, policy := range policies { + routerPolicies = append(routerPolicies, c.newLogicalRouterPolicy(policy.Priority, policy.Match, policy.Action, policy.Nexthops, policy.BFDSessions, policy.ExternalIDs)) + } + if err := c.CreateLogicalRouterPolicies(lrName, routerPolicies...); err != nil { + return fmt.Errorf("failed to batch create policies for router %s: %w", lrName, err) + } + return nil +} + +func (c *OVNNbClient) batchUpdatetLogicalRouterPolicies(updateMap map[*ovnnb.LogicalRouterPolicy]*ovnnb.LogicalRouterPolicy) error { + updateOps := make([]ovsdb.Operation, 0, len(updateMap)) + for policyNew, policyFound := range updateMap { + policy := ptr.To(*policyFound) + policy.ExternalIDs = policyNew.ExternalIDs + ops, err := c.Where(policy).Update(policy, &policy.ExternalIDs) + if err != nil { + return fmt.Errorf("failed to generate operations for updating logical router policy: %w", err) + } + updateOps = append(updateOps, ops...) + } + if err := c.Transact("lr-policy-update", updateOps); err != nil { + err := fmt.Errorf("failed to batch update logical router policy: %w", err) + klog.Error(err) + return err + } + return nil +} + +func createPolicyKey(priority int, match string) string { + return fmt.Sprintf("%s-%d", match, priority) +} + +func buildLogicalRouterPolicyIndex(priority int, match string) *ovnnb.LogicalRouterPolicy { + policy := &ovnnb.LogicalRouterPolicy{} + if match != "" { + policy.Match = match + } + if priority >= 0 { + policy.Priority = priority + } + return policy +} diff --git a/pkg/ovs/ovn-nb-logical_router_policy_test.go b/pkg/ovs/ovn-nb-logical_router_policy_test.go index 8ec011e0e3b..50a051bee08 100644 --- a/pkg/ovs/ovn-nb-logical_router_policy_test.go +++ b/pkg/ovs/ovn-nb-logical_router_policy_test.go @@ -708,3 +708,229 @@ func (suite *OvnClientTestSuite) testDeleteLogicalRouterPolicyByNexthop() { require.Error(t, err) }) } + +func (suite *OvnClientTestSuite) testBatchAddLogicalRouterPolicy() { + t := suite.T() + t.Parallel() + + nbClient := suite.ovnNBClient + lrName := "test-batch-add-policy-lr" + priority := 11011 + match := "ip4.src == $ovn.default.lm2_ip4" + action := ovnnb.LogicalRouterPolicyActionAllow + nextHops := []string{"100.64.0.2"} + lrp := &ovnnb.LogicalRouterPolicy{ + Priority: priority, + Match: match, + Action: action, + Nexthops: nextHops, + } + + err := nbClient.CreateLogicalRouter(lrName) + require.NoError(t, err) + + err = nbClient.BatchAddLogicalRouterPolicy(lrName, lrp) + require.NoError(t, err) + + lr, err := nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + + policyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, policyList, 1) + require.Contains(t, lr.Policies, policyList[0].UUID) + + t.Run("normal add policy", func(t *testing.T) { + err = nbClient.BatchAddLogicalRouterPolicy(lrName, lrp) + require.NoError(t, err) + + lrp.Action = ovnnb.LogicalRouterPolicyActionDrop + err = nbClient.BatchAddLogicalRouterPolicy(lrName, lrp) + require.NoError(t, err) + + finalPolicyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, finalPolicyList, 1) + }) + + t.Run("should log err when logical router does not exist", func(t *testing.T) { + err = nbClient.BatchAddLogicalRouterPolicy("test-nonexist-lr", lrp) + require.Error(t, err) + }) + + t.Run("handle duplicate policies with matching action and nextHops", func(t *testing.T) { + lrp.Action = ovnnb.LogicalRouterPolicyActionAllow + err = nbClient.BatchAddLogicalRouterPolicy(lrName, lrp) + require.NoError(t, err) + + duplicatePolicy := nbClient.newLogicalRouterPolicy(priority, match, action, nextHops, nil, nil) + err = nbClient.CreateLogicalRouterPolicies(lrName, duplicatePolicy) + require.NoError(t, err) + + err = nbClient.BatchAddLogicalRouterPolicy(lrName, lrp) + require.NoError(t, err) + + finalPolicyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, finalPolicyList, 1) + }) + + t.Run("update policy with different externalIDs", func(t *testing.T) { + initialExternalIDs := map[string]string{"key1": "value1"} + lrp.ExternalIDs = initialExternalIDs + + err = nbClient.BatchAddLogicalRouterPolicy(lrName, lrp) + require.NoError(t, err) + + policyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, policyList, 1) + require.Equal(t, initialExternalIDs, policyList[0].ExternalIDs) + + newExternalIDs := map[string]string{"key2": "value2"} + lrp.ExternalIDs = newExternalIDs + + err = nbClient.BatchAddLogicalRouterPolicy(lrName, lrp) + require.NoError(t, err) + + updatedPolicyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, updatedPolicyList, 1) + require.Equal(t, newExternalIDs, updatedPolicyList[0].ExternalIDs) + }) +} + +func (suite *OvnClientTestSuite) testBatchDeleteLogicalRouterPolicy() { + t := suite.T() + t.Parallel() + + nbClient := suite.ovnNBClient + lrName := "test-batch-delete-policy-by-lr" + priority := 11011 + match := "ip4.src == $ovn.default.lm2_ip4" + action := ovnnb.LogicalRouterPolicyActionAllow + nextHops := []string{"100.64.0.2"} + lrp := &ovnnb.LogicalRouterPolicy{ + Priority: priority, + Match: match, + Action: action, + Nexthops: nextHops, + } + + err := nbClient.CreateLogicalRouter(lrName) + require.NoError(t, err) + + err = nbClient.BatchAddLogicalRouterPolicy(lrName, lrp) + require.NoError(t, err) + + lr, err := nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + + policyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, policyList, 1) + require.Contains(t, lr.Policies, policyList[0].UUID) + + t.Run("no err when delete nil router policy", func(t *testing.T) { + lr, err := nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + + policyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, policyList, 1) + require.Contains(t, lr.Policies, policyList[0].UUID) + + err = nbClient.BatchDeleteLogicalRouterPolicy(lrName, []*ovnnb.LogicalRouterPolicy{}) + require.NoError(t, err) + + policyList, err = nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, policyList, 1) + require.Contains(t, lr.Policies, policyList[0].UUID) + }) + + t.Run("no err when delete existent logical router policy", func(t *testing.T) { + lr, err := nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + + policyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, policyList, 1) + require.Contains(t, lr.Policies, policyList[0].UUID) + + err = nbClient.BatchDeleteLogicalRouterPolicy(lrName, []*ovnnb.LogicalRouterPolicy{lrp}) + require.NoError(t, err) + + _, err = nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.ErrorContains(t, err, "not found policy") + + lr, err = nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + require.NotContains(t, lr.Policies, policyList[0].UUID) + }) + + t.Run("no err when delete nonexistent logical router policy", func(t *testing.T) { + lrp.Priority = priority + 1 + _, err = nbClient.GetLogicalRouterPolicy(lrName, lrp.Priority, match, false) + require.ErrorContains(t, err, "not found policy") + + lr, err = nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + require.NotContains(t, lr.Policies, policyList[0].UUID) + + err = nbClient.BatchDeleteLogicalRouterPolicy(lrName, []*ovnnb.LogicalRouterPolicy{lrp}) + require.NoError(t, err) + }) + + t.Run("should log err when logical router does not exist", func(t *testing.T) { + err = nbClient.BatchDeleteLogicalRouterPolicy("test-nonexist-lr", []*ovnnb.LogicalRouterPolicy{lrp}) + require.Error(t, err) + }) +} + +func (suite *OvnClientTestSuite) testBatchDeleteLogicalRouterPolicyByUUID() { + t := suite.T() + t.Parallel() + + nbClient := suite.ovnNBClient + lrName := "test-batch-delete-policy-uuid-by-lr" + priority := 11011 + match := "ip4.src == $ovn.default.lm2_ip4" + action := ovnnb.LogicalRouterPolicyActionAllow + nextHops := []string{"100.64.0.2"} + lrp := &ovnnb.LogicalRouterPolicy{ + Priority: priority, + Match: match, + Action: action, + Nexthops: nextHops, + } + + err := nbClient.CreateLogicalRouter(lrName) + require.NoError(t, err) + + err = nbClient.BatchAddLogicalRouterPolicy(lrName, lrp) + require.NoError(t, err) + + lr, err := nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + + policyList, err := nbClient.GetLogicalRouterPolicy(lrName, priority, match, false) + require.NoError(t, err) + require.Len(t, policyList, 1) + require.Contains(t, lr.Policies, policyList[0].UUID) + uuidList := []string{policyList[0].UUID} + + err = nbClient.BatchDeleteLogicalRouterPolicyByUUID(lrName, uuidList...) + require.NoError(t, err) + + t.Run("should log err when logical router does not exist", func(t *testing.T) { + err = nbClient.BatchDeleteLogicalRouterPolicyByUUID("test-nonexist-lr", uuidList...) + require.Error(t, err) + }) + + t.Run("should no log err when no logical router policy uuid", func(t *testing.T) { + uuidList = []string{} + err = nbClient.BatchDeleteLogicalRouterPolicyByUUID(lrName, uuidList...) + require.NoError(t, err) + }) +} diff --git a/pkg/ovs/ovn-nb-logical_router_route.go b/pkg/ovs/ovn-nb-logical_router_route.go index 0e9d1bd0721..9c4359220b7 100644 --- a/pkg/ovs/ovn-nb-logical_router_route.go +++ b/pkg/ovs/ovn-nb-logical_router_route.go @@ -12,6 +12,7 @@ import ( "github.com/scylladb/go-set/strset" "k8s.io/klog/v2" "k8s.io/utils/ptr" + "k8s.io/utils/set" ovsclient "github.com/kubeovn/kube-ovn/pkg/ovsdb/client" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" @@ -247,6 +248,58 @@ func (c *OVNNbClient) DeleteLogicalRouterStaticRouteByExternalIDs(lrName string, return nil } +// BatchDeleteLogicalRouterStaticRoute batch delete a logical router static route +func (c *OVNNbClient) BatchDeleteLogicalRouterStaticRoute(lrName string, staticRoutes []*ovnnb.LogicalRouterStaticRoute) error { + lr, err := c.GetLogicalRouter(lrName, true) + if lr == nil && err == nil { + return nil + } + + staticRoutesMap := make(map[string]string, len(staticRoutes)) + for _, route := range staticRoutes { + if route == nil { + continue + } + if route.Policy == nil { + route.Policy = &ovnnb.LogicalRouterStaticRoutePolicyDstIP + } + + staticRoutesMap[createStaticRouteKey(route.RouteTable, *route.Policy, route.IPPrefix)] = route.Nexthop + } + routes, err := c.batchListLogicalRouterStaticRoutesForDelete(staticRoutesMap, lr.StaticRoutes) + if err != nil { + klog.Error(err) + return err + } + + // not found, skip + if len(routes) == 0 { + return nil + } + + uuids := make([]string, 0, len(routes)) + for _, route := range routes { + key := createStaticRouteKey(route.RouteTable, *route.Policy, route.IPPrefix) + nexthop, exits := staticRoutesMap[key] + if exits && (nexthop == "" || route.Nexthop == nexthop) { + uuids = append(uuids, route.UUID) + } + } + + // remove static route from logical router + ops, err := c.LogicalRouterUpdateStaticRouteOp(lrName, uuids, ovsdb.MutateOperationDelete) + if err != nil { + klog.Error(err) + return fmt.Errorf("generate operations for removing static routes %v from logical router %s: %w", uuids, lrName, err) + } + if err = c.Transact("lr-route-del", ops); err != nil { + klog.Error(err) + return fmt.Errorf("delete static routes %v from logical router %s: %w", uuids, lrName, err) + } + + return nil +} + // ClearLogicalRouterStaticRoute clear static route from logical router once func (c *OVNNbClient) ClearLogicalRouterStaticRoute(lrName string) error { lr, err := c.GetLogicalRouter(lrName, false) @@ -434,3 +487,31 @@ func (c *OVNNbClient) listLogicalRouterStaticRoutesByFilter(lrName string, filte return routeList, nil } + +// batchListLogicalRouterStaticRoutesForDelete batch list route which match the given condition when need delete static route +func (c *OVNNbClient) batchListLogicalRouterStaticRoutesForDelete(staticRoutes map[string]string, lrStaticRoute []string) ([]*ovnnb.LogicalRouterStaticRoute, error) { + lrStaticRouteSet := set.New(lrStaticRoute...) + fnFilter := func(route *ovnnb.LogicalRouterStaticRoute) bool { + if !lrStaticRouteSet.Has(route.UUID) { + return false + } + key := createStaticRouteKey(route.RouteTable, *route.Policy, route.IPPrefix) + _, exists := staticRoutes[key] + return exists + } + + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) + defer cancel() + + routeList := make([]*ovnnb.LogicalRouterStaticRoute, 0) + if err := c.ovsDbClient.WhereCache(fnFilter).List(ctx, &routeList); err != nil { + klog.Error(err) + return nil, fmt.Errorf("batch list logical staric router %v lr staric route %v route: %w", staticRoutes, lrStaticRoute, err) + } + + return routeList, nil +} + +func createStaticRouteKey(routeTable, policy, ipPrefix string) string { + return fmt.Sprintf("%s-%s-%s", routeTable, policy, ipPrefix) +} diff --git a/pkg/ovs/ovn-nb-logical_router_route_test.go b/pkg/ovs/ovn-nb-logical_router_route_test.go index cb78fb4f187..e48a39be82d 100644 --- a/pkg/ovs/ovn-nb-logical_router_route_test.go +++ b/pkg/ovs/ovn-nb-logical_router_route_test.go @@ -802,3 +802,103 @@ func (suite *OvnClientTestSuite) testGetLogicalRouterStaticRouteEdgeCases() { require.Equal(t, "", route.RouteTable) }) } + +func (suite *OvnClientTestSuite) testBatchDeleteLogicalRouterStaticRoute() { + t := suite.T() + t.Parallel() + + nbClient := suite.ovnNBClient + lrName := "test-batch-del-route-lr" + routeTable := util.MainRouteTable + policy := ovnnb.LogicalRouterStaticRoutePolicyDstIP + ipPrefix := "192.168.30.0/24" + nexthop := "192.168.30.1" + staticRouter := &ovnnb.LogicalRouterStaticRoute{ + Policy: &policy, + IPPrefix: ipPrefix, + Nexthop: nexthop, + RouteTable: routeTable, + } + + err := nbClient.CreateLogicalRouter(lrName) + require.NoError(t, err) + + t.Run("normal static route", func(t *testing.T) { + err = nbClient.AddLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix, nil, nil, nexthop) + require.NoError(t, err) + + lr, err := nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + + route, err := nbClient.GetLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix, nexthop, false) + require.NoError(t, err) + require.Contains(t, lr.StaticRoutes, route.UUID) + + err = nbClient.BatchDeleteLogicalRouterStaticRoute(lrName, []*ovnnb.LogicalRouterStaticRoute{staticRouter}) + require.NoError(t, err) + + lr, err = nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + require.Empty(t, lr.StaticRoutes) + + _, err = nbClient.GetLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix, nexthop, false) + require.ErrorContains(t, err, "not found") + }) + + t.Run("delete non-exist static route", func(t *testing.T) { + staticRouter.IPPrefix = "192.168.40.0/24" + staticRouter.Nexthop = "192.168.40.1" + err = nbClient.BatchDeleteLogicalRouterStaticRoute(lrName, []*ovnnb.LogicalRouterStaticRoute{staticRouter}) + require.NoError(t, err) + }) + + t.Run("delete ecmp policy route", func(t *testing.T) { + ipPrefix := "192.168.40.0/24" + nexthops := []string{"192.168.50.1", "192.168.60.1"} + staticRouter.IPPrefix = ipPrefix + + err = nbClient.AddLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix, nil, nil, nexthops...) + require.NoError(t, err) + + lr, err := nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + + for _, nexthop := range nexthops { + route, err := nbClient.GetLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix, nexthop, false) + require.NoError(t, err) + require.Contains(t, lr.StaticRoutes, route.UUID) + } + + /* delete first route */ + staticRouter.Nexthop = nexthops[0] + err = nbClient.BatchDeleteLogicalRouterStaticRoute(lrName, []*ovnnb.LogicalRouterStaticRoute{staticRouter}) + require.NoError(t, err) + + lr, err = nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + + _, err = nbClient.GetLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix, nexthops[0], false) + require.ErrorContains(t, err, `not found logical router test-batch-del-route-lr static route 'policy dst-ip ip_prefix 192.168.40.0/24 nexthop 192.168.50.1'`) + + route, err := nbClient.GetLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix, nexthops[1], false) + require.NoError(t, err) + require.ElementsMatch(t, []string{route.UUID}, lr.StaticRoutes) + + /* delete second route */ + staticRouter.Nexthop = nexthops[1] + err = nbClient.DeleteLogicalRouterStaticRoute(lrName, &routeTable, &policy, ipPrefix, nexthops[1]) + require.NoError(t, err) + + lr, err = nbClient.GetLogicalRouter(lrName, false) + require.NoError(t, err) + require.Empty(t, lr.StaticRoutes) + + _, err = nbClient.GetLogicalRouterStaticRoute(lrName, routeTable, policy, ipPrefix, nexthops[1], false) + require.ErrorContains(t, err, `not found logical router test-batch-del-route-lr static route 'policy dst-ip ip_prefix 192.168.40.0/24 nexthop 192.168.60.1'`) + }) + + t.Run("delete static route for non-exist logical router", func(t *testing.T) { + err := nbClient.BatchDeleteLogicalRouterStaticRoute("non-exist-lrName", []*ovnnb.LogicalRouterStaticRoute{staticRouter}) + require.NoError(t, err) + }) +} diff --git a/pkg/ovs/ovn-nb-suite_test.go b/pkg/ovs/ovn-nb-suite_test.go index f1f57e98020..be588988899 100644 --- a/pkg/ovs/ovn-nb-suite_test.go +++ b/pkg/ovs/ovn-nb-suite_test.go @@ -694,6 +694,10 @@ func (suite *OvnClientTestSuite) Test_UpdateAddressSet() { suite.testUpdateAddressSet() } +func (suite *OvnClientTestSuite) Test_BatchDeleteAddressSetByNames() { + suite.testBatchDeleteAddressSetByNames() +} + /* acl unit test */ func (suite *OvnClientTestSuite) Test_testUpdateIngressAclOps() { suite.testUpdateIngressACLOps() @@ -856,6 +860,18 @@ func (suite *OvnClientTestSuite) Test_PolicyFilter() { suite.testPolicyFilter() } +func (suite *OvnClientTestSuite) Test_BatchAddLogicalRouterPolicy() { + suite.testBatchAddLogicalRouterPolicy() +} + +func (suite *OvnClientTestSuite) Test_BatchDeleteLogicalRouterPolicyByUUID() { + suite.testBatchDeleteLogicalRouterPolicyByUUID() +} + +func (suite *OvnClientTestSuite) Test_BatchDeleteLogicalRouterPolicy() { + suite.testBatchDeleteLogicalRouterPolicy() +} + /* nat unit test */ func (suite *OvnClientTestSuite) Test_CreateNats() { suite.testCreateNats() @@ -954,6 +970,10 @@ func (suite *OvnClientTestSuite) Test_GetLogicalRouterStaticRouteEdgeCases() { suite.testGetLogicalRouterStaticRouteEdgeCases() } +func (suite *OvnClientTestSuite) Test_tBatchDeleteLogicalRouterStaticRoute() { + suite.testBatchDeleteLogicalRouterStaticRoute() +} + /* dhcp options unit test */ func (suite *OvnClientTestSuite) Test_UpdateDHCPOptions() { suite.testUpdateDHCPOptions() @@ -1331,6 +1351,13 @@ func newNbClient(addr string, timeout int) (client.Client, error) { return nil, err } + dbModel.SetIndexes(map[string][]model.ClientIndex{ + "Logical_Router_Policy": {{Columns: []model.ColumnKey{ + {Column: "match"}, + {Column: "priority"}, + }}, {Columns: []model.ColumnKey{{Column: "priority"}}}, {Columns: []model.ColumnKey{{Column: "match"}}}}, + }) + logger := stdr.New(log.New(os.Stderr, "", log.LstdFlags)). WithName("libovsdb"). WithValues("database", dbModel.Name()) diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index 2a0f6f22e8c..f110df52220 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -7,6 +7,7 @@ import ( "time" "github.com/ovn-org/libovsdb/client" + "github.com/ovn-org/libovsdb/model" "github.com/ovn-org/libovsdb/ovsdb" "k8s.io/klog/v2" @@ -63,6 +64,14 @@ func NewOvnNbClient(ovnNbAddr string, ovnNbTimeout, ovsDbConTimeout, ovsDbInacti return nil, err } + dbModel.SetIndexes(map[string][]model.ClientIndex{ + "Logical_Router_Policy": {{Columns: []model.ColumnKey{ + {Column: "match"}, + {Column: "priority"}, + }}, {Columns: []model.ColumnKey{{Column: "priority"}}}, {Columns: []model.ColumnKey{{Column: "match"}}}}, + }) + klog.Infof("ovn nb table %s client index %v", "Logical_Router_Policy", dbModel.Indexes("Logical_Router_Policy")) + monitors := []client.MonitorOption{ client.WithTable(&ovnnb.ACL{}), client.WithTable(&ovnnb.AddressSet{}),