Skip to content

Commit

Permalink
Change L2 forwarding flows for traffic to gateway and tunnel
Browse files Browse the repository at this point in the history 
Let the packets to the tunnel port go through L2ForwardingCalcTable
as well, to be consistent with other types of traffic.
Redirect packets to gateway and tunnel, as well as BUM packets to
conntrackCommitTable and bypass ingress NetworkPolicy enforcement.
Let l3DecTTLTable decrement TTL for packets from tunnel to local Pods.
  • Loading branch information
@jianjuns
jianjuns committed Nov 24, 2020
1 parent 06053a1 commit d74d739
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 56 deletions.
13 changes: 8 additions & 5 deletions pkg/agent/openflow/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ func (c *client) InstallPodFlows(interfaceName string, podInterfaceIPs []net.IP,
localGatewayMAC := c.nodeConfig.GatewayConfig.MAC
flows := []binding.Flow{
c.podClassifierFlow(ofPort, cookie.Pod),
c.l2ForwardCalcFlow(podInterfaceMAC, ofPort, cookie.Pod),
c.l2ForwardCalcFlow(podInterfaceMAC, ofPort, false, cookie.Pod),
}

// Add support for IPv4 ARP responder.
Expand Down Expand Up @@ -516,7 +516,7 @@ func (c *client) InstallGatewayFlows() error {

flows := []binding.Flow{
c.gatewayClassifierFlow(cookie.Default),
c.l2ForwardCalcFlow(gatewayConfig.MAC, config.HostGatewayOFPort, cookie.Default),
c.l2ForwardCalcFlow(gatewayConfig.MAC, config.HostGatewayOFPort, true, cookie.Default),
}
flows = append(flows, c.gatewayIPSpoofGuardFlows(cookie.Default)...)

Expand Down Expand Up @@ -545,11 +545,14 @@ func (c *client) InstallGatewayFlows() error {
}

func (c *client) InstallDefaultTunnelFlows() error {
flow := c.tunnelClassifierFlow(config.DefaultTunOFPort, cookie.Default)
if err := c.ofEntryOperations.Add(flow); err != nil {
flows := []binding.Flow{
c.tunnelClassifierFlow(config.DefaultTunOFPort, cookie.Default),
c.l2ForwardCalcFlow(globalVirtualMAC, config.DefaultTunOFPort, true, cookie.Default),
}
if err := c.ofEntryOperations.AddAll(flows); err != nil {
return err
}
c.defaultTunnelFlows = []binding.Flow{flow}
c.defaultTunnelFlows = flows
return nil
}

Expand Down
77 changes: 40 additions & 37 deletions pkg/agent/openflow/pipeline.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ const (
)

var (
ingressEntryTable, egressEntryTable binding.TableIDType

// egressTables map records all IDs of tables related to
// egress rules.
egressTables = map[binding.TableIDType]struct{}{
Expand Down Expand Up @@ -687,15 +689,22 @@ func (c *client) serviceLBBypassFlow(ipProtocol binding.Protocol) binding.Flow {
}

// l2ForwardCalcFlow generates the flow that matches dst MAC and loads ofPort to reg.
func (c *client) l2ForwardCalcFlow(dstMAC net.HardwareAddr, ofPort uint32, category cookie.Category) binding.Flow {
func (c *client) l2ForwardCalcFlow(dstMAC net.HardwareAddr, ofPort uint32, skipIngressRules bool, category cookie.Category) binding.Flow {
l2FwdCalcTable := c.pipeline[l2ForwardingCalcTable]
nextTable := ingressEntryTable
if skipIngressRules {
// Skip ingress NetworkPolicy enforcement for traffic to the gateway interface or tunnel.
nextTable = conntrackCommitTable
}
return l2FwdCalcTable.BuildFlow(priorityNormal).
MatchDstMAC(dstMAC).
Action().LoadRegRange(int(PortCacheReg), ofPort, ofPortRegRange).
Action().LoadRegRange(int(marksReg), portFoundMark, ofPortMarkRange).
Action().GotoTable(l2FwdCalcTable.GetNext()).
Action().GotoTable(nextTable).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done()
// Broadcast, multicast, and unknown unicast packets will be dropped by
// the default flow of L2ForwardingOutTable.
}

// traceflowL2ForwardOutputFlows generates Traceflow specific flows that outputs traceflow packets to OVS port and Antrea
Expand Down Expand Up @@ -785,8 +794,7 @@ func (c *client) l3FwdFlowToPod(localGatewayMAC net.HardwareAddr, podInterfaceIP
flows = append(flows, flowBuilder.MatchDstIP(ip).
Action().SetSrcMAC(localGatewayMAC).
Action().SetDstMAC(podInterfaceMAC).
Action().DecTTL().
Action().GotoTable(l3FwdTable.GetNext()).
Action().GotoTable(l3DecTTLTable).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done())
}
Expand All @@ -805,8 +813,7 @@ func (c *client) l3FwdFlowRouteToPod(podInterfaceIPs []net.IP, podInterfaceMAC n
flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol).
MatchDstIP(ip).
Action().SetDstMAC(podInterfaceMAC).
Action().DecTTL().
Action().GotoTable(l3FwdTable.GetNext()).
Action().GotoTable(l3DecTTLTable).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done())
}
Expand All @@ -815,16 +822,14 @@ func (c *client) l3FwdFlowRouteToPod(podInterfaceIPs []net.IP, podInterfaceMAC n

// l3FwdFlowRouteToGW generates the flows to route the traffic to the gateway
// interface. It rewrites the destination MAC of the packets to the gateway
// interface MAC and decreases the TTL. The flow is used in the
// networkPolicyOnly mode for the traffic from a local Pod to remote Pods,
// Nodes, or external network.
// interface MAC. The flow is used in the networkPolicyOnly mode for the traffic
// from a local Pod to remote Pods, Nodes, or external network.
func (c *client) l3FwdFlowRouteToGW(gwMAC net.HardwareAddr, category cookie.Category) []binding.Flow {
l3FwdTable := c.pipeline[l3ForwardingTable]
var flows []binding.Flow
for _, ipProto := range c.ipProtocols {
flows = append(flows, l3FwdTable.BuildFlow(priorityLow).MatchProtocol(ipProto).
Action().SetDstMAC(gwMAC).
Action().DecTTL().
Action().GotoTable(l3FwdTable.GetNext()).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done(),
Expand Down Expand Up @@ -865,10 +870,6 @@ func (c *client) l3FwdFlowToRemote(
// Rewrite src MAC to local gateway MAC and rewrite dst MAC to virtual MAC.
Action().SetSrcMAC(localGatewayMAC).
Action().SetDstMAC(globalVirtualMAC).
// Load ofport of the tunnel interface.
Action().LoadRegRange(int(PortCacheReg), config.DefaultTunOFPort, ofPortRegRange).
// Set MAC-known.
Action().LoadRegRange(int(marksReg), portFoundMark, ofPortMarkRange).
// Flow based tunnel. Set tunnel destination.
Action().SetTunnelDst(tunnelPeer).
Action().GotoTable(l3DecTTLTable).
Expand All @@ -888,10 +889,7 @@ func (c *client) l3FwdFlowToRemoteViaGW(
return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProto).
MatchDstIPNet(peerSubnet).
Action().SetDstMAC(localGatewayMAC).
Action().LoadRegRange(int(PortCacheReg), config.HostGatewayOFPort, ofPortRegRange).
// Set MAC-known.
Action().LoadRegRange(int(marksReg), portFoundMark, ofPortMarkRange).
Action().GotoTable(conntrackCommitTable).
Action().GotoTable(l3FwdTable.GetNext()).
Cookie(c.cookieAllocator.Request(category).Raw()).
Done()
}
Expand Down Expand Up @@ -1777,21 +1775,27 @@ func (c *client) serviceEndpointGroup(groupID binding.GroupIDType, withSessionAf
return group
}

// decTTLFlows decrements TTL by one for the packets from a local Pod to a remote Pod. Antrea doesn't
// decrement TTL for the packets which enter OVS pipeline from antrea-gw0 on the source Node.
// TTL is also decremented on the destination Node in L3Routing Table when the packet is received.
// See the functions of "l3FlowsToPod", "l3ToPodFlow", and "l3ToGWFlow".
// decTTLFlows decrements TTL by one for the packets forwarded across Nodes.
// The TTL decrement should be skipped for the packets which enter OVS pipeline
// from the gateway interface, as the host IP stack should have decremented the
// TTL already for such packets.
func (c *client) decTTLFlows(category cookie.Category) []binding.Flow {
var flows []binding.Flow
decTTLTable := c.pipeline[l3DecTTLTable]
for _, proto := range c.ipProtocols {
flows = append(flows, c.pipeline[l3DecTTLTable].BuildFlow(priorityNormal).MatchPriority(priorityNormal).
MatchProtocol(proto).
MatchRegRange(int(marksReg), markTrafficFromLocal, binding.Range{0, 15}).
Action().DecTTL().
// Bypass l2ForwardingCalcTable and tables for ingress rules (which won't
// apply to packets to remote Nodes).
Action().GotoTable(conntrackCommitTable).
Done())
flows = append(flows,
// Skip packets from the gateway interface.
decTTLTable.BuildFlow(priorityHigh).MatchPriority(priorityNormal).
MatchProtocol(proto).
MatchRegRange(int(marksReg), markTrafficFromGateway, binding.Range{0, 15}).
Action().GotoTable(decTTLTable.GetNext()).
Done(),
decTTLTable.BuildFlow(priorityNormal).MatchPriority(priorityNormal).
MatchProtocol(proto).
Action().DecTTL().
Action().GotoTable(decTTLTable.GetNext()).
Done(),
)
}
return flows
}
Expand All @@ -1810,11 +1814,10 @@ func priorityIndexFunc(obj interface{}) ([]string, error) {
}

func generatePipeline(bridge binding.Bridge, enableProxy, enableAntreaNP bool) map[binding.TableIDType]binding.Table {
var egressEntryTable, IngressEntryTable binding.TableIDType
if enableAntreaNP {
egressEntryTable, IngressEntryTable = AntreaPolicyEgressRuleTable, AntreaPolicyIngressRuleTable
egressEntryTable, ingressEntryTable = AntreaPolicyEgressRuleTable, AntreaPolicyIngressRuleTable
} else {
egressEntryTable, IngressEntryTable = EgressRuleTable, IngressRuleTable
egressEntryTable, ingressEntryTable = EgressRuleTable, IngressRuleTable
}
var pipeline map[binding.TableIDType]binding.Table
if enableProxy {
Expand All @@ -1834,8 +1837,8 @@ func generatePipeline(bridge binding.Bridge, enableProxy, enableAntreaNP bool) m
EgressDefaultTable: bridge.CreateTable(EgressDefaultTable, EgressMetricTable, binding.TableMissActionNext),
EgressMetricTable: bridge.CreateTable(EgressMetricTable, l3ForwardingTable, binding.TableMissActionNext),
l3ForwardingTable: bridge.CreateTable(l3ForwardingTable, l2ForwardingCalcTable, binding.TableMissActionNext),
l3DecTTLTable: bridge.CreateTable(l3DecTTLTable, conntrackCommitTable, binding.TableMissActionNext),
l2ForwardingCalcTable: bridge.CreateTable(l2ForwardingCalcTable, IngressEntryTable, binding.TableMissActionNext),
l3DecTTLTable: bridge.CreateTable(l3DecTTLTable, l2ForwardingCalcTable, binding.TableMissActionNext),
l2ForwardingCalcTable: bridge.CreateTable(l2ForwardingCalcTable, conntrackCommitTable, binding.TableMissActionNext),
IngressRuleTable: bridge.CreateTable(IngressRuleTable, IngressDefaultTable, binding.TableMissActionNext),
IngressDefaultTable: bridge.CreateTable(IngressDefaultTable, IngressMetricTable, binding.TableMissActionNext),
IngressMetricTable: bridge.CreateTable(IngressMetricTable, conntrackCommitTable, binding.TableMissActionNext),
Expand All @@ -1856,8 +1859,8 @@ func generatePipeline(bridge binding.Bridge, enableProxy, enableAntreaNP bool) m
EgressDefaultTable: bridge.CreateTable(EgressDefaultTable, EgressMetricTable, binding.TableMissActionNext),
EgressMetricTable: bridge.CreateTable(EgressMetricTable, l3ForwardingTable, binding.TableMissActionNext),
l3ForwardingTable: bridge.CreateTable(l3ForwardingTable, l2ForwardingCalcTable, binding.TableMissActionNext),
l3DecTTLTable: bridge.CreateTable(l3DecTTLTable, conntrackCommitTable, binding.TableMissActionNext),
l2ForwardingCalcTable: bridge.CreateTable(l2ForwardingCalcTable, IngressEntryTable, binding.TableMissActionNext),
l3DecTTLTable: bridge.CreateTable(l3DecTTLTable, l2ForwardingCalcTable, binding.TableMissActionNext),
l2ForwardingCalcTable: bridge.CreateTable(l2ForwardingCalcTable, conntrackCommitTable, binding.TableMissActionNext),
IngressRuleTable: bridge.CreateTable(IngressRuleTable, IngressDefaultTable, binding.TableMissActionNext),
IngressDefaultTable: bridge.CreateTable(IngressDefaultTable, IngressMetricTable, binding.TableMissActionNext),
IngressMetricTable: bridge.CreateTable(IngressMetricTable, conntrackCommitTable, binding.TableMissActionNext),
Expand Down
34 changes: 20 additions & 14 deletions test/integration/agent/openflow_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ func testInstallNodeFlows(t *testing.T, config *testConfig) {
if err != nil {
t.Fatalf("Failed to install Openflow entries for node connectivity: %v", err)
}
for _, tableFlow := range prepareNodeFlows(config1.DefaultTunOFPort, node.subnet, node.gateway, node.nodeAddress, config.globalMAC, gatewayConfig.MAC) {
for _, tableFlow := range prepareNodeFlows(node.subnet, node.gateway, node.nodeAddress, config.globalMAC, gatewayConfig.MAC) {
ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows)
}
}
Expand All @@ -290,7 +290,7 @@ func testUninstallNodeFlows(t *testing.T, config *testConfig) {
if err != nil {
t.Fatalf("Failed to uninstall Openflow entries for node connectivity: %v", err)
}
for _, tableFlow := range prepareNodeFlows(config1.DefaultTunOFPort, node.subnet, node.gateway, node.nodeAddress, config.globalMAC, gatewayConfig.MAC) {
for _, tableFlow := range prepareNodeFlows(node.subnet, node.gateway, node.nodeAddress, config.globalMAC, gatewayConfig.MAC) {
ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, false, tableFlow.flows)
}
}
Expand Down Expand Up @@ -969,7 +969,7 @@ func preparePodFlows(podIPs []net.IP, podMAC net.HardwareAddr, podOFPort uint32,
[]*ofTestUtils.ExpectFlow{
{
MatchStr: fmt.Sprintf("priority=200,%s,reg0=0x80000/0x80000,%s=%s", ipProto, nwDstField, podIP.String()),
ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,dec_ttl,goto_table:80", gwMAC.String(), podMAC.String()),
ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,goto_table:71", gwMAC.String(), podMAC.String()),
},
},
},
Expand All @@ -995,7 +995,7 @@ func prepareGatewayFlows(gwIPs []net.IP, gwMAC net.HardwareAddr, vMAC net.Hardwa
[]*ofTestUtils.ExpectFlow{
{
MatchStr: fmt.Sprintf("priority=200,dl_dst=%s", gwMAC.String()),
ActStr: fmt.Sprintf("load:0x%x->NXM_NX_REG1[],load:0x1->NXM_NX_REG0[16],goto_table:90", config1.HostGatewayOFPort),
ActStr: fmt.Sprintf("load:0x%x->NXM_NX_REG1[],load:0x1->NXM_NX_REG0[16],goto_table:105", config1.HostGatewayOFPort),
},
},
},
Expand Down Expand Up @@ -1071,10 +1071,19 @@ func prepareTunnelFlows(tunnelPort uint32, vMAC net.HardwareAddr) []expectTableF
},
},
},
{
uint8(80),
[]*ofTestUtils.ExpectFlow{
{
MatchStr: fmt.Sprintf("priority=200,dl_dst=%s", vMAC.String()),
ActStr: fmt.Sprintf("load:0x%x->NXM_NX_REG1[],load:0x1->NXM_NX_REG0[16],goto_table:105", config1.DefaultTunOFPort),
},
},
},
}
}

func prepareNodeFlows(tunnelPort uint32, peerSubnet net.IPNet, peerGwIP, peerNodeIP net.IP, vMAC, localGwMAC net.HardwareAddr) []expectTableFlows {
func prepareNodeFlows(peerSubnet net.IPNet, peerGwIP, peerNodeIP net.IP, vMAC, localGwMAC net.HardwareAddr) []expectTableFlows {
var expFlows []expectTableFlows
var ipProtoStr, nwDstFieldName string
if peerGwIP.To4() != nil {
Expand All @@ -1098,7 +1107,7 @@ func prepareNodeFlows(tunnelPort uint32, peerSubnet net.IPNet, peerGwIP, peerNod
[]*ofTestUtils.ExpectFlow{
{
MatchStr: fmt.Sprintf("priority=200,%s,%s=%s", ipProtoStr, nwDstFieldName, peerSubnet.String()),
ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,load:0x%x->NXM_NX_REG1[],load:0x1->NXM_NX_REG0[16],set_field:%s->tun_dst,goto_table:71", localGwMAC.String(), vMAC.String(), tunnelPort, peerNodeIP.String()),
ActStr: fmt.Sprintf("set_field:%s->eth_src,set_field:%s->eth_dst,set_field:%s->tun_dst,goto_table:71", localGwMAC.String(), vMAC.String(), peerNodeIP.String()),
},
},
})
Expand Down Expand Up @@ -1131,7 +1140,7 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows {
}
table71Flows := expectTableFlows{
tableID: 71,
flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:105"}},
flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:80"}},
}
if config.enableIPv4 {
table31Flows.flows = append(table31Flows.flows,
Expand All @@ -1143,7 +1152,8 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows {
&ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ip", ActStr: "ct(commit,table=106,zone=65520)"},
)
table71Flows.flows = append(table71Flows.flows,
&ofTestUtils.ExpectFlow{MatchStr: "priority=200,ip,reg0=0x2/0xffff", ActStr: "dec_ttl,goto_table:105"},
&ofTestUtils.ExpectFlow{MatchStr: "priority=210,ip,reg0=0x1/0xffff", ActStr: "goto_table:80"},
&ofTestUtils.ExpectFlow{MatchStr: "priority=200,ip", ActStr: "dec_ttl,goto_table:80"},
)
}
if config.enableIPv6 {
Expand All @@ -1156,7 +1166,7 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows {
&ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ipv6", ActStr: "ct(commit,table=106,zone=65510)"},
)
table71Flows.flows = append(table71Flows.flows,
&ofTestUtils.ExpectFlow{MatchStr: "priority=200,ipv6,reg0=0x2/0xffff", ActStr: "dec_ttl,goto_table:105"},
&ofTestUtils.ExpectFlow{MatchStr: "priority=200,ipv6,reg0=0x2/0xffff", ActStr: "dec_ttl,goto_table:80"},
)
}
return []expectTableFlows{
Expand Down Expand Up @@ -1202,13 +1212,9 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows {
uint8(70),
[]*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:80"}},
},
{
uint8(71),
[]*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:105"}},
},
{
uint8(80),
[]*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:90"}},
[]*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:105"}},
},
{
uint8(90),
Expand Down

0 comments on commit d74d739

Please sign in to comment.