diff --git a/build/Dockerfile b/build/Dockerfile index cdb8afbd9a..9a67630a95 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -48,7 +48,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && apt-get install -y bsdutils mount util-linux libuuid1 libmount1 libblkid1 libsmartcols1 \ && curl -fsSL https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /etc/apt/trusted.gpg.d/nginx_signing.gpg \ && curl -fsSL -o /etc/apt/apt.conf.d/90pkgs-nginx https://cs.nginx.com/static/files/90pkgs-nginx \ - && DEBIAN_VERSION=$(awk -F '=' '/^VERSION_CODENAME=/ {print $2}' /etc/os-release) \ + && DEBIAN_VERSION=bullseye \ && printf "%s\n" "Acquire::https::pkgs.nginx.com::User-Agent \"k8s-ic-$IC_VERSION${BUILD_OS##debian-plus}-apt\";" >> /etc/apt/apt.conf.d/90pkgs-nginx \ && printf "%s\n" "deb https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION^^}/debian ${DEBIAN_VERSION} nginx-plus" > /etc/apt/sources.list.d/nginx-plus.list \ && apt-get update \ @@ -57,7 +57,16 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && rm -rf /var/lib/apt/lists/* -############################################# Base image for Debian with NGINX Plus and App Protect WAF/DoS ############################################# + +#RUN \ +# DEBIAN_VERSION=$(awk -F '=' '/^VERSION_CODENAME=/ {print $2}' /etc/os-release) \ +# && ls -l /etc/ \ +# && cat /etc/os-release \ +# && echo "Debug" +# + + +############################################# Base image for Debian with NGINX Plus and App Protect ############################################# FROM debian-plus as debian-plus-nap ARG NGINX_PLUS_VERSION ARG NAP_MODULES diff --git a/internal/configs/ingress.go b/internal/configs/ingress.go index 1865b84ccb..d2612037a9 100644 --- a/internal/configs/ingress.go +++ b/internal/configs/ingress.go @@ -2,6 +2,7 @@ package configs import ( "fmt" + "net" "sort" "strconv" "strings" @@ -456,10 +457,16 @@ func createUpstream(ingEx *IngressEx, name string, backend *networking.IngressBa } for _, endp := range endps { - addressport := strings.Split(endp, ":") + addr, port, err := net.SplitHostPort(endp) + + if err != nil { + glog.Warningf("invalid endpoint address: %s", err.Error()) + continue + } + upsServers = append(upsServers, version1.UpstreamServer{ - Address: addressport[0], - Port: addressport[1], + Address: addr, + Port: port, MaxFails: cfg.MaxFails, MaxConns: cfg.MaxConns, FailTimeout: cfg.FailTimeout, diff --git a/internal/configs/version1/nginx-plus.ingress.tmpl b/internal/configs/version1/nginx-plus.ingress.tmpl index 58191c987f..529747051f 100644 --- a/internal/configs/version1/nginx-plus.ingress.tmpl +++ b/internal/configs/version1/nginx-plus.ingress.tmpl @@ -22,12 +22,14 @@ upstream {{$upstream.Name}} { server { {{if $server.SpiffeCerts}} listen 443 ssl; + listen [::]:443 ssl; ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; {{else}} {{if not $server.GRPCOnly}} {{range $port := $server.Ports}} listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}}; + listen [::]:{{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}}; {{- end}} {{end}} @@ -39,6 +41,7 @@ server { {{else}} {{- range $port := $server.SSLPorts}} listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}}; + listen [::]:{{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}}; {{- end}} {{end}} {{if $server.SSLRejectHandshake}} diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 0167a1ac47..5e9d828abb 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -149,6 +149,7 @@ http { set $service ""; listen 80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}}; + listen [::]:80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}}; {{if .TLSPassthrough}} listen unix:/var/lib/nginx/passthrough-https.sock ssl default_server{{if .HTTP2}} http2{{end}} proxy_protocol; @@ -156,6 +157,7 @@ http { real_ip_header proxy_protocol; {{else}} listen 443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}}; + listen [::]:443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}}; {{end}} {{if .SSLRejectHandshake}} @@ -196,6 +198,7 @@ http { # NGINX Plus APIs server { listen {{.NginxStatusPort}}; + listen [::]:{{.NginxStatusPort}}; root /usr/share/nginx/html; @@ -255,6 +258,7 @@ http { {{if .InternalRouteServer}} server { listen 443 ssl; + listen [::]:443 ssl; server_name {{.InternalRouteServerName}}; ssl_certificate /etc/nginx/secrets/spiffe_cert.pem; ssl_certificate_key /etc/nginx/secrets/spiffe_key.pem; @@ -290,6 +294,7 @@ stream { server { listen 443; + listen [::]:443; ssl_preread on; diff --git a/internal/configs/version1/nginx.ingress.tmpl b/internal/configs/version1/nginx.ingress.tmpl index 8ef108f478..02d357d096 100644 --- a/internal/configs/version1/nginx.ingress.tmpl +++ b/internal/configs/version1/nginx.ingress.tmpl @@ -13,6 +13,7 @@ server { {{if not $server.GRPCOnly}} {{range $port := $server.Ports}} listen {{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}}; + listen [::]:{{$port}}{{if $server.ProxyProtocol}} proxy_protocol{{end}}; {{- end}} {{end}} @@ -24,6 +25,7 @@ server { {{else}} {{- range $port := $server.SSLPorts}} listen {{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}}; + listen [::]:{{$port}} ssl{{if $server.HTTP2}} http2{{end}}{{if $server.ProxyProtocol}} proxy_protocol{{end}}; {{- end}} {{end}} {{if $server.SSLRejectHandshake}} diff --git a/internal/configs/version1/nginx.tmpl b/internal/configs/version1/nginx.tmpl index 6360c58de6..553b00623a 100644 --- a/internal/configs/version1/nginx.tmpl +++ b/internal/configs/version1/nginx.tmpl @@ -107,6 +107,7 @@ http { set $service ""; listen 80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}}; + listen [::]:80 default_server{{if .ProxyProtocol}} proxy_protocol{{end}}; {{if .TLSPassthrough}} listen unix:/var/lib/nginx/passthrough-https.sock ssl default_server{{if .HTTP2}} http2{{end}} proxy_protocol; @@ -114,6 +115,7 @@ http { real_ip_header proxy_protocol; {{else}} listen 443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}}; + listen [::]:443 ssl default_server{{if .HTTP2}} http2{{end}}{{if .ProxyProtocol}} proxy_protocol{{end}}; {{end}} {{if .SSLRejectHandshake}} diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index 2252221f42..1817cdf852 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -35,6 +35,7 @@ server { set_real_ip_from unix:; {{ else }} listen {{ $s.Port }}{{ if $s.UDP }} udp{{ end }}; + listen [::]:{{ $s.Port }}{{ if $s.UDP }} udp{{ end }}; {{ end }} status_zone {{ $s.StatusZone }}; diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl index b3929c1397..4b2df3ae97 100644 --- a/internal/configs/version2/nginx-plus.virtualserver.tmpl +++ b/internal/configs/version2/nginx-plus.virtualserver.tmpl @@ -59,6 +59,7 @@ match {{ $m.Name }} { {{ $s := .Server }} server { listen 80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; + listen [::]:80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; server_name {{ $s.ServerName }}; status_zone {{ $s.StatusZone }}; @@ -89,6 +90,7 @@ server { real_ip_header proxy_protocol; {{ else }} listen 443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; + listen [::]:443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; {{ end }} {{ if $ssl.RejectHandshake }} diff --git a/internal/configs/version2/nginx.transportserver.tmpl b/internal/configs/version2/nginx.transportserver.tmpl index 52e18a7e1f..b3d485fbee 100644 --- a/internal/configs/version2/nginx.transportserver.tmpl +++ b/internal/configs/version2/nginx.transportserver.tmpl @@ -23,6 +23,7 @@ server { set_real_ip_from unix:; {{ else }} listen {{ $s.Port }}{{ if $s.UDP }} udp{{ end }}; + listen [::]:{{ $s.Port }}{{ if $s.UDP }} udp{{ end }}; {{ end }} {{ if $s.ProxyRequests }} diff --git a/internal/configs/version2/nginx.virtualserver.tmpl b/internal/configs/version2/nginx.virtualserver.tmpl index 96dd4d3408..dc9c95db59 100644 --- a/internal/configs/version2/nginx.virtualserver.tmpl +++ b/internal/configs/version2/nginx.virtualserver.tmpl @@ -41,6 +41,7 @@ limit_req_zone {{ $z.Key }} zone={{ $z.ZoneName }}:{{ $z.ZoneSize }} rate={{ $z. {{ $s := .Server }} server { listen 80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; + listen [::]:80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; server_name {{ $s.ServerName }}; @@ -56,6 +57,7 @@ server { real_ip_header proxy_protocol; {{ else }} listen 443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; + listen [::]:443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; {{ end }} {{ if $ssl.RejectHandshake }} diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 0eb0170bf0..49c936b02f 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -19,6 +19,8 @@ package k8s import ( "context" "fmt" + "net" + "strconv" "strings" "sync" "time" @@ -2972,7 +2974,7 @@ func getEndpointsBySubselectedPods(targetPort int32, pods []*api_v1.Pod, svcEps } for _, address := range subset.Addresses { if address.IP == pod.Status.PodIP { - addr := fmt.Sprintf("%v:%v", pod.Status.PodIP, targetPort) + addr := ipv6SafeAddrPort(pod.Status.PodIP, targetPort) ownerType, ownerName := getPodOwnerTypeAndName(pod) podEnd := podEndpoint{ Address: addr, @@ -2991,6 +2993,10 @@ func getEndpointsBySubselectedPods(targetPort int32, pods []*api_v1.Pod, svcEps return endps } +func ipv6SafeAddrPort(addr string, port int32) string { + return net.JoinHostPort(addr, strconv.Itoa(int(port))) +} + func getPodName(pod *api_v1.ObjectReference) string { if pod != nil { return pod.Name @@ -3103,7 +3109,7 @@ func (lbc *LoadBalancerController) getEndpointsForPort(endps api_v1.Endpoints, b if port.Port == targetPort { var endpoints []podEndpoint for _, address := range subset.Addresses { - addr := fmt.Sprintf("%v:%v", address.IP, port.Port) + addr := ipv6SafeAddrPort(address.IP, port.Port) podEnd := podEndpoint{ Address: addr, } diff --git a/tests/Makefile b/tests/Makefile index 54ad9c7f27..a0926943c1 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -12,6 +12,15 @@ KIND_KUBE_CONFIG_FOLDER = $${HOME}/.kube/kind SHOW_IC_LOGS = no PYTEST_ARGS = DOCKERFILEPATH = docker/Dockerfile +KIND_TEMPLATE_FILE_PATH=kind/kind-config-template.yaml +IP_FAMILY ?= ipv4 +IP_ADDR_PROPERTY = .IPAddress +IP_ADDR_FMT = "%s" + +ifeq (${IP_FAMILY}, ipv6) +IP_ADDR_PROPERTY = .GlobalIPv6Address +IP_ADDR_FMT = "[%s]" +endif .PHONY: build build: @@ -21,17 +30,36 @@ build: run-tests: docker run --rm -v $(KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) --context=$(CONTEXT) --image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) --deployment-type=$(DEPLOYMENT_TYPE) --ic-type=$(IC_TYPE) --service=$(SERVICE) --node-ip=$(NODE_IP) --show-ic-logs=$(SHOW_IC_LOGS) $(PYTEST_ARGS) + +.PHONY: get-kind-cluster-ip +get-kind-cluster-ip: + $(eval KIND_CLUSTER_IP=$(shell docker inspect -f '{{range .NetworkSettings.Networks}}{{${IP_ADDR_PROPERTY}}}{{end}}' kind-control-plane )) + $(eval KIND_CLUSTER_IP=$(shell printf ${IP_ADDR_FMT} "${KIND_CLUSTER_IP}")) + + +.PHONY: update-test-kind-config +update-test-kind-config: get-kind-cluster-ip + sed -ir "s|server:.*|server: https://${KIND_CLUSTER_IP}:6443|" $(KIND_KUBE_CONFIG_FOLDER)/config + .PHONY: run-tests-in-kind -run-tests-in-kind: - $(eval KIND_CLUSTER_IP=$(shell docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kind-control-plane)) - sed -i "" "s|server:.*|server: https://$(KIND_CLUSTER_IP):6443|" $(KIND_KUBE_CONFIG_FOLDER)/config - docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) --context=kind-kind --image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) --deployment-type=$(DEPLOYMENT_TYPE) --ic-type=$(IC_TYPE) --service=nodeport --node-ip=$(KIND_CLUSTER_IP) --show-ic-logs=$(SHOW_IC_LOGS) $(PYTEST_ARGS) +run-tests-in-kind: update-test-kind-config + docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(PREFIX):$(TAG) \ + --context=kind-kind \ + --image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) \ + --deployment-type=$(DEPLOYMENT_TYPE) \ + --ic-type=$(IC_TYPE) \ + --service=nodeport \ + --node-ip=$(KIND_CLUSTER_IP) \ + --show-ic-logs=$(SHOW_IC_LOGS) \ + $(PYTEST_ARGS) .PHONY: create-kind-cluster create-kind-cluster: $(eval K8S_VERSION=$(shell grep "K8S_VERSION:" ../.github/workflows/ci.yml | awk -F" " '{print $$2}')) - kind create cluster --image kindest/node:v$(K8S_VERSION) + cat ${KIND_TEMPLATE_FILE_PATH} | sed "s/{{.IPFamily}}/${IP_FAMILY}/" > kind-config.yaml + kind create cluster --image kindest/node:v$(K8S_VERSION) --config kind-config.yaml kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG_FOLDER)/config + kind load docker-image ${BUILD_IMAGE} .PHONY: delete-kind-cluster delete-kind-cluster: diff --git a/tests/kind/kind-config-template.yaml b/tests/kind/kind-config-template.yaml new file mode 100644 index 0000000000..59edf8fae3 --- /dev/null +++ b/tests/kind/kind-config-template.yaml @@ -0,0 +1,11 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: {{.IPFamily}} + #apiServerAddress: "127.0.0.1" + #apiServerPort: 6443 +nodes: + - role: control-plane + - role: worker + - role: worker + - role: worker