Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a toggle for Multi-cluster Pod-to-Pod connectivity #4605

Merged
merged 1 commit into from
Feb 16, 2023
Merged

Add a toggle for Multi-cluster Pod-to-Pod connectivity #4605

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions build/charts/antrea/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ Kubernetes: `>= 1.16.0-0`
| multicast.igmpQueryInterval | string | `"125s"` | The interval at which the antrea-agent sends IGMP queries to Pods. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". |
| multicast.multicastInterfaces | list | `[]` | Names of the interfaces on Nodes that are used to forward multicast traffic. |
| multicluster.enableGateway | bool | `false` | Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. This feature is supported only with encap mode. |
| multicluster.enablePodToPodConnectivity | bool | `false` | Enable Multi-cluster Pod to Pod connectivity. |
| multicluster.enableStretchedNetworkPolicy | bool | `false` | Enable Multi-cluster NetworkPolicy. Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy. |
| multicluster.namespace | string | `""` | The Namespace where Antrea Multi-cluster Controller is running. The default is antrea-agent's Namespace. |
| noSNAT | bool | `false` | Whether or not to SNAT (using the Node IP) the egress traffic from a Pod to the external network. |
Expand Down
2 changes: 2 additions & 0 deletions build/charts/antrea/conf/antrea-agent.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,8 @@ multicluster:
# Enable Multi-cluster NetworkPolicy (ingress rules).
# Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
enableStretchedNetworkPolicy: {{ .enableStretchedNetworkPolicy }}
# Enable Pod to Pod connectivity.
enablePodToPodConnectivity: {{ .enablePodToPodConnectivity }}
{{- end }}

{{- if .Values.featureGates.SecondaryNetwork }}
Expand Down
2 changes: 2 additions & 0 deletions build/charts/antrea/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,6 +336,8 @@ multicluster:
# -- Enable Multi-cluster NetworkPolicy.
# Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
enableStretchedNetworkPolicy: false
# -- Enable Multi-cluster Pod to Pod connectivity.
enablePodToPodConnectivity: false

testing:
## -- enable code coverage measurement (used when testing Antrea only).
Expand Down
6 changes: 4 additions & 2 deletions build/yamls/antrea-aks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3240,6 +3240,8 @@ data:
# Enable Multi-cluster NetworkPolicy (ingress rules).
# Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
enableStretchedNetworkPolicy: false
# Enable Pod to Pod connectivity.
enablePodToPodConnectivity: false
antrea-cni.conflist: |
{
"cniVersion":"0.3.0",
Expand Down Expand Up @@ -4297,7 +4299,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 01eb4b4048215050463f3b01c05831615c060b02d6f379d6d27a80346185d544
checksum/config: be4d7318350c398a0362a44ff0d4ff779150a303e577ed1e2265aaa75c00546e
labels:
app: antrea
component: antrea-agent
Expand Down Expand Up @@ -4538,7 +4540,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 01eb4b4048215050463f3b01c05831615c060b02d6f379d6d27a80346185d544
checksum/config: be4d7318350c398a0362a44ff0d4ff779150a303e577ed1e2265aaa75c00546e
labels:
app: antrea
component: antrea-controller
Expand Down
6 changes: 4 additions & 2 deletions build/yamls/antrea-eks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3240,6 +3240,8 @@ data:
# Enable Multi-cluster NetworkPolicy (ingress rules).
# Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
enableStretchedNetworkPolicy: false
# Enable Pod to Pod connectivity.
enablePodToPodConnectivity: false
antrea-cni.conflist: |
{
"cniVersion":"0.3.0",
Expand Down Expand Up @@ -4297,7 +4299,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 01eb4b4048215050463f3b01c05831615c060b02d6f379d6d27a80346185d544
checksum/config: be4d7318350c398a0362a44ff0d4ff779150a303e577ed1e2265aaa75c00546e
labels:
app: antrea
component: antrea-agent
Expand Down Expand Up @@ -4539,7 +4541,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 01eb4b4048215050463f3b01c05831615c060b02d6f379d6d27a80346185d544
checksum/config: be4d7318350c398a0362a44ff0d4ff779150a303e577ed1e2265aaa75c00546e
labels:
app: antrea
component: antrea-controller
Expand Down
6 changes: 4 additions & 2 deletions build/yamls/antrea-gke.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3240,6 +3240,8 @@ data:
# Enable Multi-cluster NetworkPolicy (ingress rules).
# Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
enableStretchedNetworkPolicy: false
# Enable Pod to Pod connectivity.
enablePodToPodConnectivity: false
antrea-cni.conflist: |
{
"cniVersion":"0.3.0",
Expand Down Expand Up @@ -4297,7 +4299,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: aff837005adc6d91f4b5ace3a87c08cfa49e26c60e284ebd234eea34ce5de91f
checksum/config: fca1f2d4967020380202ef0c2394b560055830ee2770e41f791af76b42559659
labels:
app: antrea
component: antrea-agent
Expand Down Expand Up @@ -4536,7 +4538,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: aff837005adc6d91f4b5ace3a87c08cfa49e26c60e284ebd234eea34ce5de91f
checksum/config: fca1f2d4967020380202ef0c2394b560055830ee2770e41f791af76b42559659
labels:
app: antrea
component: antrea-controller
Expand Down
6 changes: 4 additions & 2 deletions build/yamls/antrea-ipsec.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3253,6 +3253,8 @@ data:
# Enable Multi-cluster NetworkPolicy (ingress rules).
# Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
enableStretchedNetworkPolicy: false
# Enable Pod to Pod connectivity.
enablePodToPodConnectivity: false
antrea-cni.conflist: |
{
"cniVersion":"0.3.0",
Expand Down Expand Up @@ -4310,7 +4312,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: e3e3255bb4f4cd13bce262d8c8d5f4aead3e84e52ea1775c34898c69b80fad33
checksum/config: ab53bf1e28a67ba5be2b99989a8d28b31d716d79b207a610cd5258ead514eb6b
checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
labels:
app: antrea
Expand Down Expand Up @@ -4595,7 +4597,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: e3e3255bb4f4cd13bce262d8c8d5f4aead3e84e52ea1775c34898c69b80fad33
checksum/config: ab53bf1e28a67ba5be2b99989a8d28b31d716d79b207a610cd5258ead514eb6b
labels:
app: antrea
component: antrea-controller
Expand Down
6 changes: 4 additions & 2 deletions build/yamls/antrea.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3240,6 +3240,8 @@ data:
# Enable Multi-cluster NetworkPolicy (ingress rules).
# Multi-cluster Gateway must be enabled to enable StretchedNetworkPolicy.
enableStretchedNetworkPolicy: false
# Enable Pod to Pod connectivity.
enablePodToPodConnectivity: false
antrea-cni.conflist: |
{
"cniVersion":"0.3.0",
Expand Down Expand Up @@ -4297,7 +4299,7 @@ spec:
kubectl.kubernetes.io/default-container: antrea-agent
# Automatically restart Pods with a RollingUpdate if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 846220b3b64851cea85fb1e374c3ffdb29376ea729a494dad1cb230b3e5efe8c
checksum/config: 2c1c5158b6a3ea32eff58bc1e498592e80ebecee07f51b10c722b67afce7b964
labels:
app: antrea
component: antrea-agent
Expand Down Expand Up @@ -4536,7 +4538,7 @@ spec:
annotations:
# Automatically restart Pod if the ConfigMap changes
# See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
checksum/config: 846220b3b64851cea85fb1e374c3ffdb29376ea729a494dad1cb230b3e5efe8c
checksum/config: 2c1c5158b6a3ea32eff58bc1e498592e80ebecee07f51b10c722b67afce7b964
labels:
app: antrea
component: antrea-controller
Expand Down
1 change: 1 addition & 0 deletions cmd/antrea-agent/agent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,7 @@ func run(o *Options) error {
nodeConfig,
mcNamespace,
o.config.Multicluster.EnableStretchedNetworkPolicy,
o.config.Multicluster.EnablePodToPodConnectivity,
)
}
if enableMulticlusterNP {
Expand Down
18 changes: 15 additions & 3 deletions docs/multicluster/user-guide.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -511,9 +511,11 @@ for more information.

Since Antrea v1.9.0, Multi-cluster supports routing Pod traffic across clusters
through Multi-cluster Gateways. Pod IPs can be reached in all member clusters
within a ClusterSet. To enable this feature, the cluster's Pod CIDRs must be set in
ConfigMap `antrea-mc-controller-config` of each member cluster like the example
below. Note, **Pod CIDRs must not overlap among clusters to enable cross-cluster
within a ClusterSet. To enable this feature, the cluster's Pod CIDRs must be set
in ConfigMap `antrea-mc-controller-config` of each member cluster and
`multicluster.enablePodToPodConnectivity` must be set to `true` in the `antrea-agent`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suppose we should highlight this toggle is introduced from v1.11? @jianjuns any suggestion?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel it is not important. But if we want, no harm to add the information either.

configuration.
Note, **Pod CIDRs must not overlap among clusters to enable cross-cluster
Pod-to-Pod connectivity**.

```yaml
Expand All @@ -533,6 +535,16 @@ metadata:
namespace: kube-system
```

```yaml
antrea-controller.conf: |
featureGates:
...
Multicluster: true
...
multicluster:
enablePodToPodConnectivity: true
```

You can edit [antrea-multicluster-member.yml](../../multicluster/build/yamls/antrea-multicluster-member.yml),
or use `kubectl edit` to change the ConfigMap:

Expand Down
14 changes: 11 additions & 3 deletions pkg/agent/multicluster/mc_route_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ type MCRouteController struct {
// The Namespace where Antrea Multi-cluster Controller is running.
namespace string
enableStretchedNetworkPolicy bool
enablePodToPodConnectivity bool
}

func NewMCRouteController(
Expand All @@ -89,6 +90,7 @@ func NewMCRouteController(
nodeConfig *config.NodeConfig,
namespace string,
enableStretchedNetworkPolicy bool,
enablePodToPodConnectivity bool,
) *MCRouteController {
controller := &MCRouteController{
mcClient: mcClient,
Expand All @@ -106,6 +108,7 @@ func NewMCRouteController(
installedCIImports: make(map[string]*mcv1alpha1.ClusterInfoImport),
namespace: namespace,
enableStretchedNetworkPolicy: enableStretchedNetworkPolicy,
enablePodToPodConnectivity: enablePodToPodConnectivity,
}
controller.gwInformer.Informer().AddEventHandlerWithResyncPeriod(
cache.ResourceEventHandlerFuncs{
Expand Down Expand Up @@ -351,8 +354,10 @@ func (c *MCRouteController) addMCFlowsForSingleCIImp(activeGW *mcv1alpha1.Gatewa
var ciImportNoChange bool
if installedCIImp != nil {
oldTunnelPeerIPToRemoteGW := getPeerGatewayIP(installedCIImp.Spec)
ciImportNoChange = oldTunnelPeerIPToRemoteGW.Equal(tunnelPeerIPToRemoteGW) && installedCIImp.Spec.ServiceCIDR == ciImport.Spec.ServiceCIDR &&
sets.NewString(installedCIImp.Spec.PodCIDRs...).Equal(sets.NewString(ciImport.Spec.PodCIDRs...))
ciImportNoChange = oldTunnelPeerIPToRemoteGW.Equal(tunnelPeerIPToRemoteGW) && installedCIImp.Spec.ServiceCIDR == ciImport.Spec.ServiceCIDR
if c.enablePodToPodConnectivity {
ciImportNoChange = ciImportNoChange && sets.NewString(installedCIImp.Spec.PodCIDRs...).Equal(sets.NewString(ciImport.Spec.PodCIDRs...))
}
}

if ciImportNoChange && !activeGWChanged {
Expand All @@ -362,7 +367,10 @@ func (c *MCRouteController) addMCFlowsForSingleCIImp(activeGW *mcv1alpha1.Gatewa

klog.InfoS("Adding/updating remote Gateway Node flows for Multi-cluster", "gateway", klog.KObj(activeGW),
"node", c.nodeConfig.Name, "peer", tunnelPeerIPToRemoteGW)
allCIDRs := append([]string{ciImport.Spec.ServiceCIDR}, ciImport.Spec.PodCIDRs...)
allCIDRs := []string{ciImport.Spec.ServiceCIDR}
if c.enablePodToPodConnectivity {
hjiajing marked this conversation as resolved.
Show resolved Hide resolved
allCIDRs = append(allCIDRs, ciImport.Spec.PodCIDRs...)
}
peerConfigs, err := generatePeerConfigs(allCIDRs, tunnelPeerIPToRemoteGW)
if err != nil {
klog.ErrorS(err, "Parse error for serviceCIDR from remote cluster", "clusterinfoimport", ciImport.Name, "gateway", activeGW.Name)
Expand Down
1 change: 1 addition & 0 deletions pkg/agent/multicluster/mc_route_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ func newMCRouteController(t *testing.T, nodeConfig *config.NodeConfig) (*fakeRou
nodeConfig,
"default",
true,
true,
)
return &fakeRouteController{
MCRouteController: c,
Expand Down
4 changes: 4 additions & 0 deletions pkg/config/agent/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,10 @@ type MulticlusterConfig struct {
// Enable Multi-cluster NetworkPolicy which allows Antrea-native policy ingress rules to select peers
// from all clusters in a ClusterSet.
EnableStretchedNetworkPolicy bool `yaml:"enableStretchedNetworkPolicy,omitempty"`
// Enable Multi-cluster Pod to Pod connectivity which allows one Pod access to another Pod in other member
// clusters directly. This feature also requires Pod CIDRs to be provided in the Multi-cluster Controller
// configuration.
EnablePodToPodConnectivity bool `yaml:"enablePodToPodConnectivity,omitempty"`
}

type ExternalNodeConfig struct {
Expand Down