Add toggle for Multi-cluster Pod-to-Pod connectivity

antrea-io · Feb 4, 2023 · 259c89b · 259c89b
1 parent bc74667
commit 259c89b
Show file tree

Hide file tree

Showing 12 changed files with 61 additions and 16 deletions.
diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -341,6 +341,9 @@ multicluster:
 # Enable Multi-cluster NetworkPolicy (ingress rules).
 # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
   enableStretchedNetworkPolicy: {{ .enableStretchedNetworkPolicy }}
+# Enable Pod to Pod connectivity.
+# Cluster Pod CIDRs must be provided to enable Pod to Pod connectivity.
+  enablePodToPodConnectivity: {{ .enablePodToPodConnectivity }}
 {{- end }}
 
 {{- if .Values.featureGates.SecondaryNetwork }}

diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml
@@ -333,6 +333,9 @@ multicluster:
   # -- Enable Multi-cluster NetworkPolicy.
   # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
   enableStretchedNetworkPolicy: false
+  # -- Enable Multi-cluster Pod to Pod connectivity.
+  # Cluster Pod CIDRs must be provided to enable Pod to Pod connectivity.
+  enablePodToPodConnectivity: false
 
 testing:
   ## -- enable code coverage measurement (used when testing Antrea only).

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3237,6 +3237,9 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+    # Cluster Pod CIDRs must be provided to enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4294,7 +4297,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a5cdb0e182ac7ccdd59fd7f435fa07bc90d48422fbdc98cdd53359aef80bf59a
+        checksum/config: 5e86a889fca88734845bed60765a31dd090ba17830f29aaecc0b162e83e725ba
       labels:
         app: antrea
         component: antrea-agent
@@ -4535,7 +4538,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a5cdb0e182ac7ccdd59fd7f435fa07bc90d48422fbdc98cdd53359aef80bf59a
+        checksum/config: 5e86a889fca88734845bed60765a31dd090ba17830f29aaecc0b162e83e725ba
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3237,6 +3237,9 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+    # Cluster Pod CIDRs must be provided to enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4294,7 +4297,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a5cdb0e182ac7ccdd59fd7f435fa07bc90d48422fbdc98cdd53359aef80bf59a
+        checksum/config: 5e86a889fca88734845bed60765a31dd090ba17830f29aaecc0b162e83e725ba
       labels:
         app: antrea
         component: antrea-agent
@@ -4536,7 +4539,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a5cdb0e182ac7ccdd59fd7f435fa07bc90d48422fbdc98cdd53359aef80bf59a
+        checksum/config: 5e86a889fca88734845bed60765a31dd090ba17830f29aaecc0b162e83e725ba
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3237,6 +3237,9 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+    # Cluster Pod CIDRs must be provided to enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4294,7 +4297,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 0abb3d19aa5b5e3a83d4f46868d66f1904eab0572aed86ae91eb3c0e4d6cb75a
+        checksum/config: 498f6060a4d4397c8ce36007eebbe29ac4650f30b393a45bdef064db89eff868
       labels:
         app: antrea
         component: antrea-agent
@@ -4533,7 +4536,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 0abb3d19aa5b5e3a83d4f46868d66f1904eab0572aed86ae91eb3c0e4d6cb75a
+        checksum/config: 498f6060a4d4397c8ce36007eebbe29ac4650f30b393a45bdef064db89eff868
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3250,6 +3250,9 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+    # Cluster Pod CIDRs must be provided to enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4307,7 +4310,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 995b258674ed96418bc039673e5d725bf0dac26cabd1447bce5a995175f9a652
+        checksum/config: fb7b9d5088c70ec9e4207281fe8c68fc7b9898fb5952c3e94334e55058c81b63
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -4592,7 +4595,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 995b258674ed96418bc039673e5d725bf0dac26cabd1447bce5a995175f9a652
+        checksum/config: fb7b9d5088c70ec9e4207281fe8c68fc7b9898fb5952c3e94334e55058c81b63
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3237,6 +3237,9 @@ data:
     # Enable Multi-cluster NetworkPolicy (ingress rules).
     # Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
       enableStretchedNetworkPolicy: false
+    # Enable Pod to Pod connectivity.
+    # Cluster Pod CIDRs must be provided to enable Pod to Pod connectivity.
+      enablePodToPodConnectivity: false
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -4294,7 +4297,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 9d9c26964205adb87ebfdd1eac07428ad82014be6014f70d1a46bf6731e8091d
+        checksum/config: de2887f3e07360b2a44c956ada74bf8b1dc9e69fa63c935f0256244bc084ff2a
       labels:
         app: antrea
         component: antrea-agent
@@ -4533,7 +4536,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 9d9c26964205adb87ebfdd1eac07428ad82014be6014f70d1a46bf6731e8091d
+        checksum/config: de2887f3e07360b2a44c956ada74bf8b1dc9e69fa63c935f0256244bc084ff2a
       labels:
         app: antrea
         component: antrea-controller

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -341,6 +341,7 @@ func run(o *Options) error {
 			nodeConfig,
 			mcNamespace,
 			o.config.Multicluster.EnableStretchedNetworkPolicy,
+			o.config.Multicluster.EnablePodToPodConnectivity,
 		)
 	}
 	if enableMulticlusterNP {

diff --git a/docs/multicluster/user-guide.md b/docs/multicluster/user-guide.md
@@ -511,9 +511,10 @@ for more information.
 
 Since Antrea v1.9.0, Multi-cluster supports routing Pod traffic across clusters
 through Multi-cluster Gateways. Pod IPs can be reached in all member clusters
-within a ClusterSet. To enable this feature, the cluster's Pod CIDRs must be set in
-ConfigMap `antrea-mc-controller-config` of each member cluster like the example
-below. Note, **Pod CIDRs must not overlap among clusters to enable cross-cluster
+within a ClusterSet. To enable this feature, the cluster's Pod CIDRs must be set
+in ConfigMap `antrea-mc-controller-config` of each member cluster and make sure
+that `enablePodToPodConnectivity` is set to `true` in antrea-agent ConfigMap.
+Note, **Pod CIDRs must not overlap among clusters to enable cross-cluster
 Pod-to-Pod connectivity**.
 
 ```yaml
@@ -533,6 +534,16 @@ metadata:
   namespace: kube-system
 ```
 
+```yaml
+antrea-controller.conf: |
+  featureGates:
+...
+Multicluster: true
+...
+multicluster:
+  enablePodToPodConnectivity: true # required by both egress and ingres rules
+```
+
 You can edit [antrea-multicluster-member.yml](../../multicluster/build/yamls/antrea-multicluster-member.yml),
 or use `kubectl edit` to change the ConfigMap:
 

diff --git a/pkg/agent/multicluster/mc_route_controller.go b/pkg/agent/multicluster/mc_route_controller.go
@@ -77,6 +77,7 @@ type MCRouteController struct {
 	// The Namespace where Antrea Multi-cluster Controller is running.
 	namespace                    string
 	enableStretchedNetworkPolicy bool
+	enablePodToPodConnectivity   bool
 }
 
 func NewMCRouteController(
@@ -89,6 +90,7 @@ func NewMCRouteController(
 	nodeConfig *config.NodeConfig,
 	namespace string,
 	enableStretchedNetworkPolicy bool,
+	enablePodToPodConnectivity bool,
 ) *MCRouteController {
 	controller := &MCRouteController{
 		mcClient:                     mcClient,
@@ -106,6 +108,7 @@ func NewMCRouteController(
 		installedCIImports:           make(map[string]*mcv1alpha1.ClusterInfoImport),
 		namespace:                    namespace,
 		enableStretchedNetworkPolicy: enableStretchedNetworkPolicy,
+		enablePodToPodConnectivity:   enablePodToPodConnectivity,
 	}
 	controller.gwInformer.Informer().AddEventHandlerWithResyncPeriod(
 		cache.ResourceEventHandlerFuncs{
@@ -351,8 +354,10 @@ func (c *MCRouteController) addMCFlowsForSingleCIImp(activeGW *mcv1alpha1.Gatewa
 	var ciImportNoChange bool
 	if installedCIImp != nil {
 		oldTunnelPeerIPToRemoteGW := getPeerGatewayIP(installedCIImp.Spec)
-		ciImportNoChange = oldTunnelPeerIPToRemoteGW.Equal(tunnelPeerIPToRemoteGW) && installedCIImp.Spec.ServiceCIDR == ciImport.Spec.ServiceCIDR &&
-			sets.NewString(installedCIImp.Spec.PodCIDRs...).Equal(sets.NewString(ciImport.Spec.PodCIDRs...))
+		ciImportNoChange = oldTunnelPeerIPToRemoteGW.Equal(tunnelPeerIPToRemoteGW) && installedCIImp.Spec.ServiceCIDR == ciImport.Spec.ServiceCIDR
+		if c.enablePodToPodConnectivity {
+			ciImportNoChange = ciImportNoChange && sets.NewString(installedCIImp.Spec.PodCIDRs...).Equal(sets.NewString(ciImport.Spec.PodCIDRs...))
+		}
 	}
 
 	if ciImportNoChange && !activeGWChanged {
@@ -362,7 +367,10 @@ func (c *MCRouteController) addMCFlowsForSingleCIImp(activeGW *mcv1alpha1.Gatewa
 
 	klog.InfoS("Adding/updating remote Gateway Node flows for Multi-cluster", "gateway", klog.KObj(activeGW),
 		"node", c.nodeConfig.Name, "peer", tunnelPeerIPToRemoteGW)
-	allCIDRs := append([]string{ciImport.Spec.ServiceCIDR}, ciImport.Spec.PodCIDRs...)
+	allCIDRs := []string{ciImport.Spec.ServiceCIDR}
+	if c.enablePodToPodConnectivity {
+		allCIDRs = append(allCIDRs, ciImport.Spec.PodCIDRs...)
+	}
 	peerConfigs, err := generatePeerConfigs(allCIDRs, tunnelPeerIPToRemoteGW)
 	if err != nil {
 		klog.ErrorS(err, "Parse error for serviceCIDR from remote cluster", "clusterinfoimport", ciImport.Name, "gateway", activeGW.Name)

diff --git a/pkg/agent/multicluster/mc_route_controller_test.go b/pkg/agent/multicluster/mc_route_controller_test.go
@@ -61,6 +61,7 @@ func newMCRouteController(t *testing.T, nodeConfig *config.NodeConfig) (*fakeRou
 		nodeConfig,
 		"default",
 		true,
+		true,
 	)
 	return &fakeRouteController{
 		MCRouteController: c,

diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go
@@ -292,6 +292,9 @@ type MulticlusterConfig struct {
 	// Enable Multi-cluster NetworkPolicy which allows Antrea-native policy ingress rules to select peers
 	// from all clusters in a ClusterSet.
 	EnableStretchedNetworkPolicy bool `yaml:"enableStretchedNetworkPolicy,omitempty"`
+	// Enable Multi-cluster Pod to Pod Connectivity which allows one Pod access another Pod in other member
+	// clusters directly. It also requires the Pod CIDRs in Multi-cluster configuration.
+	EnablePodToPodConnectivity bool `yaml:"enablePodToPodConnectivity,omitempty"`
 }
 
 type ExternalNodeConfig struct {