Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

initial workaound for IPv6 workload support #2361

Closed
wants to merge 3 commits into from
Closed

initial workaound for IPv6 workload support #2361

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2ddfa8f
initial workaound for IPv6 workload support
bonnyr-f5 Jan 17, 2022
a0ffc38
adding ipv6 support to templates to enable tests
bonnyr-f5 Feb 1, 2022
a59eeb4
use net package functions instead of string functions
bonnyr-f5 Feb 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions build/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
&& apt-get install -y bsdutils mount util-linux libuuid1 libmount1 libblkid1 libsmartcols1 \
&& curl -fsSL https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /etc/apt/trusted.gpg.d/nginx_signing.gpg \
&& curl -fsSL -o /etc/apt/apt.conf.d/90pkgs-nginx https://cs.nginx.com/static/files/90pkgs-nginx \
&& DEBIAN_VERSION=$(awk -F '=' '/^VERSION_CODENAME=/ {print $2}' /etc/os-release) \
&& DEBIAN_VERSION=bullseye \
bonnyr marked this conversation as resolved.
Show resolved Hide resolved
&& printf "%s\n" "Acquire::https::pkgs.nginx.com::User-Agent \"k8s-ic-$IC_VERSION${BUILD_OS##debian-plus}-apt\";" >> /etc/apt/apt.conf.d/90pkgs-nginx \
&& printf "%s\n" "deb https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION^^}/debian ${DEBIAN_VERSION} nginx-plus" > /etc/apt/sources.list.d/nginx-plus.list \
&& apt-get update \
Expand All @@ -57,7 +57,16 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
&& rm -rf /var/lib/apt/lists/*


############################################# Base image for Debian with NGINX Plus and App Protect WAF/DoS #############################################

#RUN \
# DEBIAN_VERSION=$(awk -F '=' '/^VERSION_CODENAME=/ {print $2}' /etc/os-release) \
# && ls -l /etc/ \
# && cat /etc/os-release \
# && echo "Debug"
#


############################################# Base image for Debian with NGINX Plus and App Protect #############################################
FROM debian-plus as debian-plus-nap
ARG NGINX_PLUS_VERSION
ARG NAP_MODULES
Expand Down
3 changes: 3 additions & 0 deletions internal/configs/version1/nginx-plus.ingress.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@ upstream {{$upstream.Name}} {
server {
{{if $server.SpiffeCerts}}
listen 443 ssl;
listen [::]:443 ssl;
ssl_certificate /etc/nginx/secrets/spiffe_cert.pem;
ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem;
{{else}}
{{if not $server.GRPCOnly}}
{{range $port := $server.Ports}}
listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
listen [::]:{{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
{{- end}}
{{end}}

Expand All @@ -39,6 +41,7 @@ server {
{{else}}
{{- range $port := $server.SSLPorts}}
listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
listen [::]:{{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
{{- end}}
{{end}}
{{if $server.SSLRejectHandshake}}
Expand Down
5 changes: 5 additions & 0 deletions internal/configs/version1/nginx-plus.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,13 +149,15 @@ http {
set $service "";

listen 80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}};
listen [::]:80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}};

{{if .TLSPassthrough}}
listen unix:/var/lib/nginx/passthrough-https.sock ssl default_server{{if .HTTP2}} http2{{end}} proxy_protocol;
set_real_ip_from unix:;
real_ip_header proxy_protocol;
{{else}}
listen 443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}};
listen [::]:443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}};
{{end}}

{{if .SSLRejectHandshake}}
Expand Down Expand Up @@ -196,6 +198,7 @@ http {
# NGINX Plus APIs
server {
listen {{.NginxStatusPort}};
listen [::]:{{.NginxStatusPort}};

root /usr/share/nginx/html;

Expand Down Expand Up @@ -255,6 +258,7 @@ http {
{{if .InternalRouteServer}}
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name {{.InternalRouteServerName}};
ssl_certificate /etc/nginx/secrets/spiffe_cert.pem;
ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem;
Expand Down Expand Up @@ -290,6 +294,7 @@ stream {

server {
listen 443;
listen [::]:443;

ssl_preread on;

Expand Down
2 changes: 2 additions & 0 deletions internal/configs/version1/nginx.ingress.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ server {
{{if not $server.GRPCOnly}}
{{range $port := $server.Ports}}
listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
listen [::]:{{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
{{- end}}
{{end}}

Expand All @@ -24,6 +25,7 @@ server {
{{else}}
{{- range $port := $server.SSLPorts}}
listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
listen [::]:{{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}};
{{- end}}
{{end}}
{{if $server.SSLRejectHandshake}}
Expand Down
2 changes: 2 additions & 0 deletions internal/configs/version1/nginx.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,15 @@ http {
set $service "";

listen 80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}};
listen [::]:80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}};

{{if .TLSPassthrough}}
listen unix:/var/lib/nginx/passthrough-https.sock ssl default_server{{if .HTTP2}} http2{{end}} proxy_protocol;
set_real_ip_from unix:;
real_ip_header proxy_protocol;
{{else}}
listen 443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}};
listen [::]:443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}};
{{end}}

{{if .SSLRejectHandshake}}
Expand Down
1 change: 1 addition & 0 deletions internal/configs/version2/nginx-plus.transportserver.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ server {
set_real_ip_from unix:;
{{ else }}
listen {{ $s.Port }}{{ if $s.UDP }} udp{{ end }};
listen [::]:{{ $s.Port }}{{ if $s.UDP }} udp{{ end }};
{{ end }}

status_zone {{ $s.StatusZone }};
Expand Down
2 changes: 2 additions & 0 deletions internal/configs/version2/nginx-plus.virtualserver.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ match {{ $m.Name }} {
{{ $s := .Server }}
server {
listen 80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};
listen [::]:80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};

server_name {{ $s.ServerName }};
status_zone {{ $s.StatusZone }};
Expand Down Expand Up @@ -89,6 +90,7 @@ server {
real_ip_header proxy_protocol;
{{ else }}
listen 443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};
listen [::]:443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};
{{ end }}

{{ if $ssl.RejectHandshake }}
Expand Down
1 change: 1 addition & 0 deletions internal/configs/version2/nginx.transportserver.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ server {
set_real_ip_from unix:;
{{ else }}
listen {{ $s.Port }}{{ if $s.UDP }} udp{{ end }};
listen [::]:{{ $s.Port }}{{ if $s.UDP }} udp{{ end }};
{{ end }}

{{ if $s.ProxyRequests }}
Expand Down
2 changes: 2 additions & 0 deletions internal/configs/version2/nginx.virtualserver.tmpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ limit_req_zone {{ $z.Key }} zone={{ $z.ZoneName }}:{{ $z.ZoneSize }} rate={{ $z.
{{ $s := .Server }}
server {
listen 80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};
listen [::]:80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};

server_name {{ $s.ServerName }};

Expand All @@ -56,6 +57,7 @@ server {
real_ip_header proxy_protocol;
{{ else }}
listen 443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};
listen [::]:443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};
{{ end }}

{{ if $ssl.RejectHandshake }}
Expand Down
38 changes: 33 additions & 5 deletions tests/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@ KIND_KUBE_CONFIG_FOLDER = $${HOME}/.kube/kind
SHOW_IC_LOGS = no
PYTEST_ARGS =
DOCKERFILEPATH = docker/Dockerfile
KIND_TEMPLATE_FILE_PATH=kind/kind-config-template.yaml
IP_FAMILY ?= ipv4
IP_ADDR_PROPERTY = .IPAddress
IP_ADDR_FMT = "%s"

ifeq (${IP_FAMILY}, ipv6)
IP_ADDR_PROPERTY = .GlobalIPv6Address
IP_ADDR_FMT = "[%s]"
endif

.PHONY: build
build:
Expand All @@ -21,17 +30,36 @@ build:
run-tests:
docker run --rm -v $(KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) --context=$(CONTEXT) --image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) --deployment-type=$(DEPLOYMENT_TYPE) --ic-type=$(IC_TYPE) --service=$(SERVICE) --node-ip=$(NODE_IP) --show-ic-logs=$(SHOW_IC_LOGS) $(PYTEST_ARGS)


.PHONY: get-kind-cluster-ip
get-kind-cluster-ip:
$(eval KIND_CLUSTER_IP=$(shell docker inspect -f '{{range .NetworkSettings.Networks}}{{${IP_ADDR_PROPERTY}}}{{end}}' kind-control-plane ))
$(eval KIND_CLUSTER_IP=$(shell printf ${IP_ADDR_FMT} "${KIND_CLUSTER_IP}"))


.PHONY: update-test-kind-config
update-test-kind-config: get-kind-cluster-ip
sed -ir "s|server:.*|server: https://${KIND_CLUSTER_IP}:6443|" $(KIND_KUBE_CONFIG_FOLDER)/config

.PHONY: run-tests-in-kind
run-tests-in-kind:
$(eval KIND_CLUSTER_IP=$(shell docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kind-control-plane))
sed -i "" "s|server:.*|server: https://$(KIND_CLUSTER_IP):6443|" $(KIND_KUBE_CONFIG_FOLDER)/config
docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) --context=kind-kind --image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) --deployment-type=$(DEPLOYMENT_TYPE) --ic-type=$(IC_TYPE) --service=nodeport --node-ip=$(KIND_CLUSTER_IP) --show-ic-logs=$(SHOW_IC_LOGS) $(PYTEST_ARGS)
run-tests-in-kind: update-test-kind-config
docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) \
--context=kind-kind \
--image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) \
--deployment-type=$(DEPLOYMENT_TYPE) \
--ic-type=$(IC_TYPE) \
--service=nodeport \
--node-ip=$(KIND_CLUSTER_IP) \
--show-ic-logs=$(SHOW_IC_LOGS) \
$(PYTEST_ARGS)

.PHONY: create-kind-cluster
create-kind-cluster:
$(eval K8S_VERSION=$(shell grep "K8S_VERSION:" ../.github/workflows/ci.yml | awk -F" " '{print $$2}'))
kind create cluster --image kindest/node:v$(K8S_VERSION)
cat ${KIND_TEMPLATE_FILE_PATH} | sed "s/{{.IPFamily}}/${IP_FAMILY}/" > kind-config.yaml
kind create cluster --image kindest/node:v$(K8S_VERSION) --config kind-config.yaml
kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG_FOLDER)/config
kind load docker-image ${BUILD_IMAGE}

.PHONY: delete-kind-cluster
delete-kind-cluster:
Expand Down
11 changes: 11 additions & 0 deletions tests/kind/kind-config-template.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
ipFamily: {{.IPFamily}}
#apiServerAddress: "127.0.0.1"
#apiServerPort: 6443
nodes:
- role: control-plane
- role: worker
- role: worker
- role: worker