From 6fee254908b1ca32fe2a0cd38b7e455cc83d1f50 Mon Sep 17 00:00:00 2001 From: oseoin Date: Wed, 13 Dec 2023 11:43:41 +0000 Subject: [PATCH] Lazy loading OSS (#4788) remove plus requirement for lazy loading --- charts/nginx-ingress/README.md | 2 +- cmd/nginx-ingress/flags.go | 7 +----- .../command-line-arguments.md | 4 ++-- .../installing-nic/installation-with-helm.md | 1 + internal/configs/version1/nginx.ingress.tmpl | 16 ++++++------- internal/configs/version1/nginx.tmpl | 19 +++++++++++---- .../version2/nginx.transportserver.tmpl | 4 ++-- .../configs/version2/nginx.virtualserver.tmpl | 24 +++++++++---------- tests/suite/test_tls.py | 3 +-- .../test_transport_server_tcp_load_balance.py | 3 +-- tests/suite/test_virtual_server_tls.py | 3 +-- 11 files changed, 45 insertions(+), 41 deletions(-) diff --git a/charts/nginx-ingress/README.md b/charts/nginx-ingress/README.md index a38ff2fc23..eaee8c1cd4 100644 --- a/charts/nginx-ingress/README.md +++ b/charts/nginx-ingress/README.md @@ -458,7 +458,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont |`controller.defaultHTTPListenerPort` | Sets the port for the HTTP `default_server` listener. | 80 | |`controller.defaultHTTPSListenerPort` | Sets the port for the HTTPS `default_server` listener. | 443 | |`controller.readOnlyRootFilesystem` | Configure root filesystem as read-only and add volumes for temporary data. | false | -|`controller.enableSSLDynamicReload` | Enable lazy loading for SSL Certificates for NGINX Plus. | true | +|`controller.enableSSLDynamicReload` | Enable lazy loading for SSL Certificates. | true | |`rbac.create` | Configures RBAC. | true | |`prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | true | |`prometheus.port` | Configures the port to scrape the metrics. | 9113 | diff --git a/cmd/nginx-ingress/flags.go b/cmd/nginx-ingress/flags.go index c8879b5297..da51ac0ebd 100644 --- a/cmd/nginx-ingress/flags.go +++ b/cmd/nginx-ingress/flags.go @@ -199,7 +199,7 @@ var ( defaultHTTPSListenerPort = flag.Int("default-https-listener-port", 443, "Sets a custom port for the HTTPS `default_server`. [1024 - 65535]") - enableDynamicSSLReload = flag.Bool(dynamicSSLReloadParam, true, "Enable reloading of SSL Certificates without restarting the NGINX process. Requires -nginx-plus") + enableDynamicSSLReload = flag.Bool(dynamicSSLReloadParam, true, "Enable reloading of SSL Certificates without restarting the NGINX process.") startupCheckFn func() error ) @@ -275,11 +275,6 @@ func parseFlags() { if *ingressLink != "" && *externalService != "" { glog.Fatal("ingresslink and external-service cannot both be set") } - - if *enableDynamicSSLReload && !*nginxPlus { - glog.V(3).Infof("%s flag requires -nginx-plus and will not be enabled", dynamicSSLReloadParam) - *enableDynamicSSLReload = false - } } func initialChecks() { diff --git a/docs/content/configuration/global-configuration/command-line-arguments.md b/docs/content/configuration/global-configuration/command-line-arguments.md index ff10f1830d..cb6e815977 100644 --- a/docs/content/configuration/global-configuration/command-line-arguments.md +++ b/docs/content/configuration/global-configuration/command-line-arguments.md @@ -530,8 +530,8 @@ Default `443`. ### -ssl-dynamic-reload -Used to activate or deactivate lazy loading for SSL Certificates for NGINX Plus. +Used to activate or deactivate lazy loading for SSL Certificates. -The default value is `true` when using NGINX Plus. +The default value is `true`. diff --git a/docs/content/installation/installing-nic/installation-with-helm.md b/docs/content/installation/installing-nic/installation-with-helm.md index 1b86e9774e..1f09773e14 100644 --- a/docs/content/installation/installing-nic/installation-with-helm.md +++ b/docs/content/installation/installing-nic/installation-with-helm.md @@ -418,6 +418,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **controller.strategy** | Specifies the strategy used to replace old Pods with new ones. Docs for [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) and [Daemonset update strategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) | {} | | **controller.disableIPV6** | Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. | false | | **controller.readOnlyRootFilesystem** | Configure root filesystem as read-only and add volumes for temporary data. | false | +| **controller.enableSSLDynamicReload** | Enable lazy loading for SSL Certificates. | true | | **rbac.create** | Configures RBAC. | true | | **prometheus.create** | Expose NGINX or NGINX Plus metrics in the Prometheus format. | true | | **prometheus.port** | Configures the port to scrape the metrics. | 9113 | diff --git a/internal/configs/version1/nginx.ingress.tmpl b/internal/configs/version1/nginx.ingress.tmpl index c5d110548c..ce88993c55 100644 --- a/internal/configs/version1/nginx.ingress.tmpl +++ b/internal/configs/version1/nginx.ingress.tmpl @@ -15,8 +15,8 @@ server { {{- if $server.SpiffeCerts}} listen 443 ssl; {{- if not $server.DisableIPV6}}listen [::]:443 ssl;{{end}} - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- else}} {{- if not $server.GRPCOnly}} {{- range $port := $server.Ports}} @@ -42,8 +42,8 @@ server { {{- if $server.SSLRejectHandshake}} ssl_reject_handshake on; {{- else}} - ssl_certificate {{$server.SSLCertificate}}; - ssl_certificate_key {{$server.SSLCertificateKey}}; + ssl_certificate {{ makeSecretPath $server.SSLCertificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $server.SSLCertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end}} {{- end}} {{- end}} @@ -155,8 +155,8 @@ server { grpc_buffer_size {{$location.ProxyBufferSize}}; {{- end}} {{- if $.SpiffeClientCerts}} - grpc_ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - grpc_ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + grpc_ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + grpc_ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; grpc_ssl_trusted_certificate /etc/nginx/secrets/spiffe_rootca.pem; grpc_ssl_server_name on; grpc_ssl_verify on; @@ -209,8 +209,8 @@ server { proxy_max_temp_file_size {{$location.ProxyMaxTempFileSize}}; {{- end}} {{- if $.SpiffeClientCerts}} - proxy_ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - proxy_ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + proxy_ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + proxy_ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; proxy_ssl_trusted_certificate /etc/nginx/secrets/spiffe_rootca.pem; proxy_ssl_server_name on; proxy_ssl_verify on; diff --git a/internal/configs/version1/nginx.tmpl b/internal/configs/version1/nginx.tmpl index 7a57c72803..aa452d9fa4 100644 --- a/internal/configs/version1/nginx.tmpl +++ b/internal/configs/version1/nginx.tmpl @@ -49,6 +49,11 @@ http { '' $sent_http_grpc_status; } + {{- if .DynamicSSLReloadEnabled }} + map $nginx_version $secret_dir_path { + default "{{ .StaticSSLPath }}"; + } + {{- end }} {{- if .AccessLogOff}} access_log off; {{- else}} @@ -125,8 +130,8 @@ http { {{- if .SSLRejectHandshake}} ssl_reject_handshake on; {{- else}} - ssl_certificate /etc/nginx/secrets/default; - ssl_certificate_key /etc/nginx/secrets/default; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/default" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/default" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; {{- end}} {{- range $setRealIPFrom := .SetRealIPFrom}} @@ -219,8 +224,8 @@ http { listen 443 ssl; {{if not .DisableIPV6}}listen [::]:443 ssl;{{end}} server_name {{.InternalRouteServerName}}; - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" .StaticSSLPath "$secret_dir_path" .DynamicSSLReloadEnabled }}; ssl_client_certificate /etc/nginx/secrets/spiffe_rootca.pem; ssl_verify_client on; ssl_verify_depth 25; @@ -248,6 +253,12 @@ stream { map_hash_max_size {{.MapHashMaxSize}}; {{if .MapHashBucketSize}}map_hash_bucket_size {{.MapHashBucketSize}};{{end}} + {{- if .DynamicSSLReloadEnabled }} + map $nginx_version $secret_dir_path { + default "{{ .StaticSSLPath }}"; + } + {{- end }} + {{- if .TLSPassthrough}} map $ssl_preread_server_name $dest_internal_passthrough { default unix:/var/lib/nginx/passthrough-https.sock; diff --git a/internal/configs/version2/nginx.transportserver.tmpl b/internal/configs/version2/nginx.transportserver.tmpl index bd5afde5c5..901def05fa 100644 --- a/internal/configs/version2/nginx.transportserver.tmpl +++ b/internal/configs/version2/nginx.transportserver.tmpl @@ -29,8 +29,8 @@ server { {{- end }} {{- if $ssl.Enabled }} - ssl_certificate {{ $ssl.Certificate }}; - ssl_certificate_key {{ $ssl.CertificateKey }}; + ssl_certificate {{ makeSecretPath $ssl.Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $ssl.CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- end }} diff --git a/internal/configs/version2/nginx.virtualserver.tmpl b/internal/configs/version2/nginx.virtualserver.tmpl index 4d5811e725..6469634d59 100644 --- a/internal/configs/version2/nginx.virtualserver.tmpl +++ b/internal/configs/version2/nginx.virtualserver.tmpl @@ -65,18 +65,18 @@ server { {{- if $ssl.RejectHandshake }} ssl_reject_handshake on; {{- else if $.SpiffeCerts }} - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- else }} - ssl_certificate {{ $ssl.Certificate }}; - ssl_certificate_key {{ $ssl.CertificateKey }}; + ssl_certificate {{ makeSecretPath $ssl.Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath $ssl.CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- else }} {{- if $.SpiffeCerts }} listen 443 ssl; {{if not $s.DisableIPV6}}listen [::]:443 ssl;{{end}} - ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- end }} @@ -149,8 +149,8 @@ server { {{- with $s.EgressMTLS }} {{- if .Certificate }} - proxy_ssl_certificate {{ .Certificate }}; - proxy_ssl_certificate_key {{ .CertificateKey }}; + proxy_ssl_certificate {{ makeSecretPath .Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + proxy_ssl_certificate_key {{ makeSecretPath .CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{- if .TrustedCert }} proxy_ssl_trusted_certificate {{ .TrustedCert }}; @@ -255,8 +255,8 @@ server { {{- with $l.EgressMTLS }} {{- if .Certificate }} - {{ $proxyOrGRPC }}_ssl_certificate {{ .Certificate }}; - {{ $proxyOrGRPC }}_ssl_certificate_key {{ .CertificateKey }}; + {{ $proxyOrGRPC }}_ssl_certificate {{ makeSecretPath .Certificate $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + {{ $proxyOrGRPC }}_ssl_certificate_key {{ makeSecretPath .CertificateKey $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{- end }} {{ if .TrustedCert }} {{ $proxyOrGRPC }}_ssl_trusted_certificate {{ .TrustedCert }}; @@ -371,8 +371,8 @@ server { add_header {{ $h.Name }} "{{ $h.Value }}" {{ if $h.Always }}always{{ end }}; {{- end }} {{- if $.SpiffeClientCerts }} - {{ $proxyOrGRPC }}_ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; - {{ $proxyOrGRPC }}_ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; + {{ $proxyOrGRPC }}_ssl_certificate {{ makeSecretPath "/etc/nginx/secrets/spiffe_cert.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; + {{ $proxyOrGRPC }}_ssl_certificate_key {{ makeSecretPath "/etc/nginx/secrets/spiffe_key.pem" $.StaticSSLPath "$secret_dir_path" $.DynamicSSLReloadEnabled }}; {{ $proxyOrGRPC }}_ssl_trusted_certificate /etc/nginx/secrets/spiffe_rootca.pem; {{ $proxyOrGRPC }}_ssl_server_name on; {{ $proxyOrGRPC }}_ssl_verify on; diff --git a/tests/suite/test_tls.py b/tests/suite/test_tls.py index c4f5787f27..59a2162e4c 100644 --- a/tests/suite/test_tls.py +++ b/tests/suite/test_tls.py @@ -138,7 +138,7 @@ def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_name wait_before_test(1) assert_us_subject(ingress_controller_endpoint, tls_setup.ingress_host) - # for OSS and and Plus with -ssl-dynamic-reload=false, we expect + # with -ssl-dynamic-reload=false, we expect # replacing a secret to trigger a reload count_before_replace = get_reload_count(tls_setup.metrics_url) @@ -153,7 +153,6 @@ def test_tls_termination(self, kube_apis, ingress_controller_endpoint, test_name assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" -@pytest.mark.skip_for_nginx_oss @pytest.mark.ingresses @pytest.mark.parametrize( "ingress_controller, tls_setup", diff --git a/tests/suite/test_transport_server_tcp_load_balance.py b/tests/suite/test_transport_server_tcp_load_balance.py index 12c1942fef..bd812b8902 100644 --- a/tests/suite/test_transport_server_tcp_load_balance.py +++ b/tests/suite/test_transport_server_tcp_load_balance.py @@ -623,7 +623,7 @@ def test_secure_tcp_request_load_balanced( endpoint = response.decode() print(f"Connected securely to: {endpoint}") - # for OSS and and Plus with -ssl-dynamic-reload=false, we expect + # with -ssl-dynamic-reload=false, we expect # replacing a secret to trigger a reload count_before_replace = get_reload_count(transport_server_setup.metrics_url) print(f"replacing: {sec_name} in {transport_server_setup.namespace}") @@ -639,7 +639,6 @@ def test_secure_tcp_request_load_balanced( delete_items_from_yaml(kube_apis, src_sec_yaml, transport_server_setup.namespace) -@pytest.mark.skip_for_nginx_oss @pytest.mark.ts @pytest.mark.skip_for_loadbalancer @pytest.mark.parametrize( diff --git a/tests/suite/test_virtual_server_tls.py b/tests/suite/test_virtual_server_tls.py index 5d052b03a6..28294997bf 100644 --- a/tests/suite/test_virtual_server_tls.py +++ b/tests/suite/test_virtual_server_tls.py @@ -130,7 +130,7 @@ def test_tls_termination(self, kube_apis, crd_ingress_controller, virtual_server wait_before_test(1) assert_us_subject(virtual_server_setup) - # for OSS and and Plus with -ssl-dynamic-reload=false, we expect + # with -ssl-dynamic-reload=false, we expect # replacing a secret to trigger a reload count_before_replace = get_reload_count(virtual_server_setup.metrics_url) @@ -150,7 +150,6 @@ def test_tls_termination(self, kube_apis, crd_ingress_controller, virtual_server assert reloads == expected_reloads, f"expected {expected_reloads} reloads, got {reloads}" -@pytest.mark.skip_for_nginx_oss @pytest.mark.vs @pytest.mark.smoke @pytest.mark.parametrize(