diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 1ed0cc916fb..bcce3216fd7 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -432,7 +432,15 @@ jobs: max_attempts: 3 shell: bash command: | + for node in $(sudo kubectl get no -o jsonpath='{.items[*].metadata.name}'); do + taint=$(sudo kubectl get no $node -o jsonpath='{.spec.taints[?(@.key=="node-role.kubernetes.io/master")]}') + if [ -n "$taint" ]; then + sudo kubectl taint node $node node-role.kubernetes.io/master:NoSchedule- + fi + done sudo make kind-install + sudo kubectl patch subnet ovn-default --type merge \ + -p '{"spec":{"gatewayType": "centralized", "gatewayNode": "kube-ovn-control-plane"}}' - name: Set up Go 1.x uses: actions/setup-go@v3 diff --git a/dist/images/uninstall.sh b/dist/images/uninstall.sh index 3a7c5f938aa..bd150b5f4cd 100644 --- a/dist/images/uninstall.sh +++ b/dist/images/uninstall.sh @@ -3,7 +3,8 @@ ovs-dpctl del-dp ovs-system iptables -t nat -D POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE -iptables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j RETURN +iptables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN +iptables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE iptables -t nat -D POSTROUTING -p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN iptables -t nat -D POSTROUTING -m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN iptables -t nat -D POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE @@ -24,12 +25,14 @@ sleep 1 ipset destroy ovn40subnets-nat ipset destroy ovn40subnets +ipset destroy ovn40subnets-distributed-gw ipset destroy ovn40local-pod-ip-nat ipset destroy ovn40other-node ipset destroy ovn40services ip6tables -t nat -D POSTROUTING -m mark --mark 0x4000/0x4000 -j MASQUERADE -ip6tables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j RETURN +ip6tables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -m set --match-set ovn60subnets-distributed-gw dst -j RETURN +ip6tables -t nat -D POSTROUTING -m mark --mark 0x80000/0x80000 -j MASQUERADE ip6tables -t nat -D POSTROUTING -p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN ip6tables -t nat -D POSTROUTING -m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN ip6tables -t nat -D POSTROUTING -m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE @@ -50,6 +53,7 @@ sleep 1 ipset destroy ovn6subnets-nat ipset destroy ovn60subnets +ipset destroy ovn60subnets-distributed-gw ipset destroy ovn60local-pod-ip-nat ipset destroy ovn60other-node ipset destroy ovn60services diff --git a/pkg/daemon/gateway.go b/pkg/daemon/gateway.go index cf3fadc1806..f8a1c3fedd8 100644 --- a/pkg/daemon/gateway.go +++ b/pkg/daemon/gateway.go @@ -105,6 +105,27 @@ func (c *Controller) getSubnetsNeedNAT(protocol string) ([]string, error) { return subnetsNeedNat, nil } +func (c *Controller) getSubnetsDistributedGateway(protocol string) ([]string, error) { + subnets, err := c.subnetsLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list subnets: %v", err) + return nil, err + } + + var result []string + for _, subnet := range subnets { + if subnet.DeletionTimestamp == nil && + subnet.Spec.Vlan == "" && + subnet.Spec.Vpc == util.DefaultVpc && + subnet.Spec.GatewayType == kubeovnv1.GWDistributedType && + (subnet.Spec.Protocol == kubeovnv1.ProtocolDual || subnet.Spec.Protocol == protocol) { + cidrBlock := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol) + result = append(result, cidrBlock) + } + } + return result, nil +} + func (c *Controller) getServicesCIDR(protocol string) []string { ret := make([]string, 0) for _, cidr := range strings.Split(c.config.ServiceClusterIPRange, ",") { diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go index 6523a173b83..8d1347c3399 100644 --- a/pkg/daemon/gateway_linux.go +++ b/pkg/daemon/gateway_linux.go @@ -23,12 +23,13 @@ import ( ) const ( - ServiceSet = "services" - SubnetSet = "subnets" - SubnetNatSet = "subnets-nat" - LocalPodSet = "local-pod-ip-nat" - OtherNodeSet = "other-node" - IPSetPrefix = "ovn" + ServiceSet = "services" + SubnetSet = "subnets" + SubnetNatSet = "subnets-nat" + SubnetDistributedGwSet = "subnets-distributed-gw" + LocalPodSet = "local-pod-ip-nat" + OtherNodeSet = "other-node" + IPSetPrefix = "ovn" ) type policyRouteMeta struct { @@ -63,6 +64,11 @@ func (c *Controller) setIPSet() error { klog.Errorf("get need nat subnets failed, %+v", err) return err } + subnetsDistributedGateway, err := c.getSubnetsDistributedGateway(protocol) + if err != nil { + klog.Errorf("failed to get subnets with centralized gateway: %v", err) + return err + } otherNode, err := c.getOtherNodes(protocol) if err != nil { klog.Errorf("failed to get node, %+v", err) @@ -88,6 +94,11 @@ func (c *Controller) setIPSet() error { SetID: SubnetNatSet, Type: ipsets.IPSetTypeHashNet, }, subnetsNeedNat) + c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{ + MaxSize: 1048576, + SetID: SubnetDistributedGwSet, + Type: ipsets.IPSetTypeHashNet, + }, subnetsDistributedGateway) c.ipsets[protocol].AddOrReplaceIPSet(ipsets.IPSetMetadata{ MaxSize: 1048576, SetID: OtherNodeSet, @@ -313,7 +324,9 @@ func (c *Controller) setIptables() error { // nat packets marked by kube-proxy or kube-ovn {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)}, // do not nat node port service traffic with external traffic policy set to local - {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j RETURN`)}, + {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn40subnets-distributed-gw dst -j RETURN`)}, + // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway + {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)}, // do not nat reply packets in direct routing {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)}, // do not nat route traffic @@ -339,7 +352,9 @@ func (c *Controller) setIptables() error { // nat packets marked by kube-proxy or kube-ovn {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x4000/0x4000 -j MASQUERADE`)}, // do not nat node port service traffic with external traffic policy set to local - {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j RETURN`)}, + {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -m set --match-set ovn60subnets-distributed-gw dst -j RETURN`)}, + // nat node port service traffic with external traffic policy set to local for subnets with centralized gateway + {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m mark --mark 0x80000/0x80000 -j MASQUERADE`)}, // do not nat reply packets in direct routing {Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-p tcp --tcp-flags SYN NONE -m conntrack --ctstate NEW -j RETURN`)}, // do not nat route traffic