From aa4df1572afbf6018be1af1d808e3f0586ef15ea Mon Sep 17 00:00:00 2001 From: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:16:53 +0200 Subject: [PATCH 1/3] Add mode field to ipsecConfig for better ipsec control API The mode field can have 3 values: - Disabled: Disable the ipsec.service on the node level - External: Enable ipsec.service on the node level, but additional configoration is required (usually by nmstate, usually for external communication) - Full: Enable ipsec.service on the node and configure secure communications between pods in the cluster. If required ipsec can be configured for external secure communications as in the case of mode External Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com> --- operator/v1/types_network.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/operator/v1/types_network.go b/operator/v1/types_network.go index cec3b631810..1bc5e77bd05 100644 --- a/operator/v1/types_network.go +++ b/operator/v1/types_network.go @@ -390,6 +390,8 @@ type OVNKubernetesConfig struct { // ipsecConfig enables and configures IPsec for pods on the pod network within the // cluster. // +optional + // +kubebuilder:default={"mode": "Disabled"} + // +default={"mode": "Disabled"} IPsecConfig *IPsecConfig `json:"ipsecConfig,omitempty"` // policyAuditConfig is the configuration for network policy audit events. If unset, // reported defaults are used. @@ -428,7 +430,19 @@ type HybridOverlayConfig struct { HybridOverlayVXLANPort *uint32 `json:"hybridOverlayVXLANPort,omitempty"` } +// +kubebuilder:validation:XValidation:rule="self == oldSelf || has(self.mode)",message="ipsecConfig.mode is required" type IPsecConfig struct { + // mode defines the behaviour of the ipsec configuration within the platform. + // Valid values are `Disabled`, `External` and `Full`. + // When 'Disabled', ipsec will not be enabled at the node level. + // When 'External', ipsec is enabled on the node level but requires the user to configure the secure communication parameters. + // This mode is for external secure communications and the configuration can be done using the k8s-nmstate operator. + // When 'Full', ipsec is configured on the node level and inter-pod secure communication within the cluster is configured. + // Note with `Full`, if ipsec is desired for communication with external (to the cluster) entities (such as storage arrays), + // this is left to the user to configure. + // +kubebuilder:validation:Enum=Disabled;External;Full + // +optional + Mode IPsecMode `json:"mode,omitempty"` } type IPForwardingMode string @@ -691,3 +705,17 @@ const ( // IPAMTypeStatic uses static IP IPAMTypeStatic IPAMType = "Static" ) + +// IPsecMode enumerates the modes for IPsec configuration +type IPsecMode string + +const ( + // IPsecModeDisabled disables IPsec altogether + IPsecModeDisabled IPsecMode = "Disabled" + // IPsecModeExternal enables IPsec on the node level, but expects the user to configure it using k8s-nmstate or + // other means - it is most useful for secure communication from the cluster to external endpoints + IPsecModeExternal IPsecMode = "External" + // IPsecModeFull enables IPsec on the node level (the same as IPsecModeExternal), and configures it to secure communication + // between pods on the cluster network. + IPsecModeFull IPsecMode = "Full" +) From 48d8d61ea79cf15d7017ee60222f363011fd4c18 Mon Sep 17 00:00:00 2001 From: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:22:45 +0200 Subject: [PATCH 2/3] Generted files for the ipsec API commit (aa4df157) Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com> --- openapi/generated_openapi/zz_generated.openapi.go | 10 ++++++++++ openapi/openapi.json | 11 ++++++++++- ...ter-network-operator_01-CustomNoUpgrade.crd.yaml | 13 +++++++++++++ ..._70_cluster-network-operator_01-Default.crd.yaml | 13 +++++++++++++ ...etwork-operator_01-TechPreviewNoUpgrade.crd.yaml | 13 +++++++++++++ operator/v1/stable.network.testsuite.yaml | 2 +- operator/v1/zz_generated.swagger_doc_generated.go | 8 ++++++++ 7 files changed, 68 insertions(+), 2 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 2f401fe0221..96a11716af3 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -46353,6 +46353,15 @@ func schema_openshift_api_operator_v1_IPsecConfig(ref common.ReferenceCallback) Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "mode": { + SchemaProps: spec.SchemaProps{ + Description: "mode defines the behaviour of the ipsec configuration within the platform. Valid values are `Disabled`, `External` and `Full`. When 'Disabled', ipsec will not be enabled at the node level. When 'External', ipsec is enabled on the node level but requires the user to configure the secure communication parameters. This mode is for external secure communications and the configuration can be done using the k8s-nmstate operator. When 'Full', ipsec is configured on the node level and inter-pod secure communication within the cluster is configured. Note with `Full`, if ipsec is desired for communication with external (to the cluster) entities (such as storage arrays), this is left to the user to configure.", + Type: []string{"string"}, + Format: "", + }, + }, + }, }, }, } @@ -49710,6 +49719,7 @@ func schema_openshift_api_operator_v1_OVNKubernetesConfig(ref common.ReferenceCa "ipsecConfig": { SchemaProps: spec.SchemaProps{ Description: "ipsecConfig enables and configures IPsec for pods on the pod network within the cluster.", + Default: map[string]interface{}{"mode": "Disabled"}, Ref: ref("github.com/openshift/api/operator/v1.IPsecConfig"), }, }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 6c36d56a49a..b503c06c7c7 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -27044,7 +27044,13 @@ } }, "com.github.openshift.api.operator.v1.IPsecConfig": { - "type": "object" + "type": "object", + "properties": { + "mode": { + "description": "mode defines the behaviour of the ipsec configuration within the platform. Valid values are `Disabled`, `External` and `Full`. When 'Disabled', ipsec will not be enabled at the node level. When 'External', ipsec is enabled on the node level but requires the user to configure the secure communication parameters. This mode is for external secure communications and the configuration can be done using the k8s-nmstate operator. When 'Full', ipsec is configured on the node level and inter-pod secure communication within the cluster is configured. Note with `Full`, if ipsec is desired for communication with external (to the cluster) entities (such as storage arrays), this is left to the user to configure.", + "type": "string" + } + } }, "com.github.openshift.api.operator.v1.IPv4GatewayConfig": { "description": "IPV4GatewayConfig holds the configuration paramaters for IPV4 connections in the GatewayConfig for OVN-Kubernetes", @@ -29088,6 +29094,9 @@ }, "ipsecConfig": { "description": "ipsecConfig enables and configures IPsec for pods on the pod network within the cluster.", + "default": { + "mode": "Disabled" + }, "$ref": "#/definitions/com.github.openshift.api.operator.v1.IPsecConfig" }, "mtu": { diff --git a/operator/v1/0000_70_cluster-network-operator_01-CustomNoUpgrade.crd.yaml b/operator/v1/0000_70_cluster-network-operator_01-CustomNoUpgrade.crd.yaml index 4938ac27973..ed528564201 100644 --- a/operator/v1/0000_70_cluster-network-operator_01-CustomNoUpgrade.crd.yaml +++ b/operator/v1/0000_70_cluster-network-operator_01-CustomNoUpgrade.crd.yaml @@ -269,6 +269,19 @@ spec: ipsecConfig: description: ipsecConfig enables and configures IPsec for pods on the pod network within the cluster. type: object + default: + mode: Disabled + properties: + mode: + description: mode defines the behaviour of the ipsec configuration within the platform. Valid values are `Disabled`, `External` and `Full`. When 'Disabled', ipsec will not be enabled at the node level. When 'External', ipsec is enabled on the node level but requires the user to configure the secure communication parameters. This mode is for external secure communications and the configuration can be done using the k8s-nmstate operator. When 'Full', ipsec is configured on the node level and inter-pod secure communication within the cluster is configured. Note with `Full`, if ipsec is desired for communication with external (to the cluster) entities (such as storage arrays), this is left to the user to configure. + type: string + enum: + - Disabled + - External + - Full + x-kubernetes-validations: + - rule: self == oldSelf || has(self.mode) + message: ipsecConfig.mode is required mtu: description: mtu is the MTU to use for the tunnel interface. This must be 100 bytes smaller than the uplink mtu. Default is 1400 type: integer diff --git a/operator/v1/0000_70_cluster-network-operator_01-Default.crd.yaml b/operator/v1/0000_70_cluster-network-operator_01-Default.crd.yaml index 83849f24b7a..2a19187c888 100644 --- a/operator/v1/0000_70_cluster-network-operator_01-Default.crd.yaml +++ b/operator/v1/0000_70_cluster-network-operator_01-Default.crd.yaml @@ -269,6 +269,19 @@ spec: ipsecConfig: description: ipsecConfig enables and configures IPsec for pods on the pod network within the cluster. type: object + default: + mode: Disabled + properties: + mode: + description: mode defines the behaviour of the ipsec configuration within the platform. Valid values are `Disabled`, `External` and `Full`. When 'Disabled', ipsec will not be enabled at the node level. When 'External', ipsec is enabled on the node level but requires the user to configure the secure communication parameters. This mode is for external secure communications and the configuration can be done using the k8s-nmstate operator. When 'Full', ipsec is configured on the node level and inter-pod secure communication within the cluster is configured. Note with `Full`, if ipsec is desired for communication with external (to the cluster) entities (such as storage arrays), this is left to the user to configure. + type: string + enum: + - Disabled + - External + - Full + x-kubernetes-validations: + - rule: self == oldSelf || has(self.mode) + message: ipsecConfig.mode is required mtu: description: mtu is the MTU to use for the tunnel interface. This must be 100 bytes smaller than the uplink mtu. Default is 1400 type: integer diff --git a/operator/v1/0000_70_cluster-network-operator_01-TechPreviewNoUpgrade.crd.yaml b/operator/v1/0000_70_cluster-network-operator_01-TechPreviewNoUpgrade.crd.yaml index 294b5c945f4..43c7758cd4d 100644 --- a/operator/v1/0000_70_cluster-network-operator_01-TechPreviewNoUpgrade.crd.yaml +++ b/operator/v1/0000_70_cluster-network-operator_01-TechPreviewNoUpgrade.crd.yaml @@ -269,6 +269,19 @@ spec: ipsecConfig: description: ipsecConfig enables and configures IPsec for pods on the pod network within the cluster. type: object + default: + mode: Disabled + properties: + mode: + description: mode defines the behaviour of the ipsec configuration within the platform. Valid values are `Disabled`, `External` and `Full`. When 'Disabled', ipsec will not be enabled at the node level. When 'External', ipsec is enabled on the node level but requires the user to configure the secure communication parameters. This mode is for external secure communications and the configuration can be done using the k8s-nmstate operator. When 'Full', ipsec is configured on the node level and inter-pod secure communication within the cluster is configured. Note with `Full`, if ipsec is desired for communication with external (to the cluster) entities (such as storage arrays), this is left to the user to configure. + type: string + enum: + - Disabled + - External + - Full + x-kubernetes-validations: + - rule: self == oldSelf || has(self.mode) + message: ipsecConfig.mode is required mtu: description: mtu is the MTU to use for the tunnel interface. This must be 100 bytes smaller than the uplink mtu. Default is 1400 type: integer diff --git a/operator/v1/stable.network.testsuite.yaml b/operator/v1/stable.network.testsuite.yaml index 3d65b5dcaf7..6c616371b2e 100644 --- a/operator/v1/stable.network.testsuite.yaml +++ b/operator/v1/stable.network.testsuite.yaml @@ -263,4 +263,4 @@ tests: disableNetworkDiagnostics: false logLevel: Normal operatorLogLevel: Normal - migration: {} + migration: {} \ No newline at end of file diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go index d50e77e44d3..48133af8df4 100644 --- a/operator/v1/zz_generated.swagger_doc_generated.go +++ b/operator/v1/zz_generated.swagger_doc_generated.go @@ -1374,6 +1374,14 @@ func (IPFIXConfig) SwaggerDoc() map[string]string { return map_IPFIXConfig } +var map_IPsecConfig = map[string]string{ + "mode": "mode defines the behaviour of the ipsec configuration within the platform. Valid values are `Disabled`, `External` and `Full`. When 'Disabled', ipsec will not be enabled at the node level. When 'External', ipsec is enabled on the node level but requires the user to configure the secure communication parameters. This mode is for external secure communications and the configuration can be done using the k8s-nmstate operator. When 'Full', ipsec is configured on the node level and inter-pod secure communication within the cluster is configured. Note with `Full`, if ipsec is desired for communication with external (to the cluster) entities (such as storage arrays), this is left to the user to configure.", +} + +func (IPsecConfig) SwaggerDoc() map[string]string { + return map_IPsecConfig +} + var map_IPv4GatewayConfig = map[string]string{ "": "IPV4GatewayConfig holds the configuration paramaters for IPV4 connections in the GatewayConfig for OVN-Kubernetes", "internalMasqueradeSubnet": "internalMasqueradeSubnet contains the masquerade addresses in IPV4 CIDR format used internally by ovn-kubernetes to enable host to service traffic. Each host in the cluster is configured with these addresses, as well as the shared gateway bridge interface. The values can be changed after installation. The subnet chosen should not overlap with other networks specified for OVN-Kubernetes as well as other networks used on the host. Additionally the subnet must be large enough to accommodate 6 IPs (maximum prefix length /29). When omitted, this means no opinion and the platform is left to choose a reasonable default which is subject to change over time. The current default subnet is 169.254.169.0/29 The value must be in proper IPV4 CIDR format", From 0e619cd8c633c96c4064c2b126b781ceea7cd57b Mon Sep 17 00:00:00 2001 From: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com> Date: Sun, 17 Dec 2023 18:56:32 +0200 Subject: [PATCH 3/3] Adding test cases for the ipsecConfig API Testing that empty ipsecConfig translates into disabled mode, and that empty strings are not allowed for ipsecConfog.mode Added transition tests (added OnUpdate section to the file) for checking that ipsecConfog.mode can not be removed once set, that ipsec can be safely disabled, and that empty ispecConfig is not changed when changing other ovnKubernetesConfig fields. Additionally updated all successful path tests (without expecterError) since ipsecConfig setting appears now on all the out yaml file as a feature. Signed-off-by: Josh Salomon <41079547+JoshSalomon@users.noreply.github.com> --- operator/v1/stable.network.testsuite.yaml | 144 +++++++++++++++++++++- 1 file changed, 143 insertions(+), 1 deletion(-) diff --git a/operator/v1/stable.network.testsuite.yaml b/operator/v1/stable.network.testsuite.yaml index 6c616371b2e..47d5354280f 100644 --- a/operator/v1/stable.network.testsuite.yaml +++ b/operator/v1/stable.network.testsuite.yaml @@ -35,6 +35,8 @@ tests: routingViaHost: false ipv4: internalMasqueradeSubnet: "169.254.168.0/29" + ipsecConfig: + mode: Disabled disableNetworkDiagnostics: false logLevel: Normal operatorLogLevel: Normal @@ -124,6 +126,8 @@ tests: ipv6: internalMasqueradeSubnet: "abcd:ef01:2345:6789:abcd:ef01:2345:6789/125" routingViaHost: false + ipsecConfig: + mode: Disabled disableNetworkDiagnostics: false logLevel: Normal operatorLogLevel: Normal @@ -147,6 +151,8 @@ tests: routingViaHost: false ipv6: internalMasqueradeSubnet: "abcd:ef01:2345:6789::2345:6789/20" + ipsecConfig: + mode: Disabled disableNetworkDiagnostics: false logLevel: Normal operatorLogLevel: Normal @@ -263,4 +269,140 @@ tests: disableNetworkDiagnostics: false logLevel: Normal operatorLogLevel: Normal - migration: {} \ No newline at end of file + migration: {} + - name: "IPsec - Empty ipsecConfig is allowed in initial state" + initial: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: {} + expected: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: {} + disableNetworkDiagnostics: false + logLevel: Normal + operatorLogLevel: Normal + - name: "IPsec - Populated ipsecConfig is allowed" + initial: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: + mode: Full + expected: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: + mode: Full + disableNetworkDiagnostics: false + logLevel: Normal + operatorLogLevel: Normal + - name: "IPsec - Start without setting ipsecConfig" + initial: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + expected: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: {} + disableNetworkDiagnostics: false + logLevel: Normal + operatorLogLevel: Normal + - name: "IPsec - empty string is not allowed" + initial: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: + mode: "" + expectedError: "Unsupported value: \"\": supported values: \"Disabled\", \"External\", \"Full\"" + onUpdate: + - name: "IPsec - Removing ipsecConfig.mode is not allowed" + initial: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: + mode: Full + updated: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: {} + expectedError: "ipsecConfig.mode is required" + - name: "IPsec - Disabling IPsec" + initial: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: + mode: Full + updated: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: + mode: Disabled + expected: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: + mode: Disabled + disableNetworkDiagnostics: false + logLevel: Normal + operatorLogLevel: Normal + - name: "IPsec - Empty ipsecConfig when changing other parameters" + initial: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: {} + updated: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: {} + mtu: 5888 + expected: | + apiVersion: operator.openshift.io/v1 + kind: Network + spec: + defaultNetwork: + ovnKubernetesConfig: + ipsecConfig: {} + mtu: 5888 + disableNetworkDiagnostics: false + logLevel: Normal + operatorLogLevel: Normal