Solve dupe uid for vault secrets (kubernetes#21)

* deadlock detected

* Add debug and fix duped UID

* clean comments

Makefile

@@ -12,7 +12,13 @@ deploy:
 .PHONY: build
 build:
 	bin/package.sh
-	bin/docker-build.sh $(name)
+	bin/docker-build.sh $(name) $(file)
+	bin/clean.sh
+
+
+build-debug:
+	bin/package.sh
+	bin/docker-build.sh stratio/ingress-nginx-debug Dockerfile.debug
 	bin/clean.sh
 
 chart:

bin/docker-build.sh

@@ -3,7 +3,8 @@
 BASEDIR="$(realpath "$(dirname "$0")/..")"
 
 name="${1:-stratio/ingress-nginx}"
+file="${2:-Dockerfile.stratio}"
 
 cd "$BASEDIR"
 
-docker build --network host -t "${name}:test" -f "rootfs/Dockerfile.stratio" .
+docker build --network host -t "${name}:test" -f "rootfs/${file}" .

bin/package-debug.sh

@@ -0,0 +1,38 @@
+#!/bin/bash -e
+
+BASEDIR="$(realpath "$(dirname "$0")/..")"
+GODIR_ROOT="$BASEDIR/cmd"
+
+#### PREPARE
+rm -rf "${BASEDIR}/dist"
+mkdir "${BASEDIR}/dist"
+
+if [[ -z "$1" ]]; then
+	VERSION=$(cat $BASEDIR/VERSION)
+else
+	VERSION=$1
+fi
+
+export GOPATH="${BASEDIR}/dist"
+export HOME="${BASEDIR}/go"
+
+echo "Seting the dlv"
+#CGO_ENABLED=0 go get -ldflags "-s -w -extldflags  '-static'" github.com/go-delve/delve/cmd/dlv 
+#CGO_ENABLED=0 go build -buildvcs=false -gcflags "all=-N -l" -o  "${BASEDIR}/dist/dlv"
+cd "$GODIR_ROOT/nginx"
+echo "Building ingress-nginx nginx-ingress-controller..."
+CGO_ENABLED=0 GOOS=linux /usr/local/go/bin/go build -buildvcs=false -gcflags "-N -l" -o "${BASEDIR}/dist/nginx-ingress-controller" ${GODIR_ROOT}/nginx
+echo "Cleaning ingress-nginx nginx dependencies..."
+#/usr/local/go/bin/go clean -modcache
+
+cd "$GODIR_ROOT/dbg"
+echo "Building ingress-nginx dbg..."
+CGO_ENABLED=0 GOOS=linux /usr/local/go/bin/go build -buildvcs=false -gcflags "-N -l" -o "${BASEDIR}/dist/dbg" ${GODIR_ROOT}/dbg
+echo "Cleaning ingress-nginx dbg dependencies..."
+/usr/local/go/bin/go clean -modcache
+
+cd "$GODIR_ROOT/waitshutdown"
+echo "Building ingress-nginx wait-shutdown..."
+CGO_ENABLED=0 GOOS=linux /usr/local/go/bin/go build -buildvcs=false -gcflags "-N -l" -o "${BASEDIR}/dist/wait-shutdown" ${GODIR_ROOT}/waitshutdown
+echo "Cleaning ingress-nginx waitshutdown dependencies..."
+/usr/local/go/bin/go clean -modcache

docs/user-guide/nginx-configuration/annotations.md

@@ -106,7 +106,7 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
 |[nginx.ingress.kubernetes.io/session-cookie-conditional-samesite-none](#cookie-affinity)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/ssl-redirect](#server-side-https-enforcement-through-redirect)|"true" or "false"|
 |[nginx.ingress.kubernetes.io/ssl-passthrough](#ssl-passthrough)|"true" or "false"|
-|[nginx.ingress.kubernetes.io/default-ssl-certificate-vault](#default-ssl-certificate-vault)|string|
+|[nginx.ingress.kubernetes.io/tls-cert-vault](#tls-cert-vault)|string|
 |[nginx.ingress.kubernetes.io/stream-snippet](#stream-snippet)|string|
 |[nginx.ingress.kubernetes.io/upstream-hash-by](#custom-nginx-upstream-hashing)|string|
 |[nginx.ingress.kubernetes.io/x-forwarded-prefix](#x-forwarded-prefix-header)|string|
@@ -976,7 +976,7 @@ For more information on the mirror module see [ngx_http_mirror_module](https://n
 
 ### TLS Certificate stored in vault
 
- Stratio custom nginx-ingress-controller supports fetching certificates to be used in TLS communications with the annotation: `nginx.ingress.kubernetes.io/default-ssl-certificate-vault`
+ Stratio custom nginx-ingress-controller supports fetching certificates to be used in TLS communications with the annotation: `nginx.ingress.kubernetes.io/tls-cert-vault`
 SecretName field from the TLS area will be ignored if this annotation is provided, instead the certificate stored in the vault path will be used.
 The path must be in the form `/<path>/<to>/<secret>/<CN>`
 

internal/ingress/annotations/vaultcertificate/main.go

@@ -42,15 +42,15 @@ func (a backendCertVaultPath) Parse(ing *networking.Ingress) (interface{}, error
 		return EmptyVaultPath, nil
 	}
 
-	VaultCertificate, err := parser.GetStringAnnotation("default-ssl-certificate-vault", ing)
+	VaultCertificate, err := parser.GetStringAnnotation("tls-cert-vault", ing)
 	if err != nil {
 		return EmptyVaultPath, nil
 	}
 
 	VaultCertificate = strings.TrimSpace(VaultCertificate)
 	if !validVaultUrl.MatchString(VaultCertificate) {
-		klog.Errorf("URL %v is not a valid value for the default-ssl-certificate-vault annotation. Regex rule is: %v", VaultCertificate, validVaultUrl)
-		err := errors.New("not a valid value for the default-ssl-certificate-vault annotation")
+		klog.Errorf("URL %v is not a valid value for the tls-cert-vault annotation. Regex rule is: %v", VaultCertificate, validVaultUrl)
+		err := errors.New("not a valid value for the tls-cert-vault annotation")
 		return EmptyVaultPath, err
 	}
 

internal/ingress/annotations/vaultcertificate/main_test.go

@@ -48,7 +48,7 @@ func TestParseInvalidAnnotations(t *testing.T) {
 	// Test no annotations set
 	i, err := NewParser(&resolver.Mock{}).Parse(ing)
 	if err != nil {
-		t.Errorf("unexpected error parsing ingress with default-ssl-certificate-vault")
+		t.Errorf("unexpected error parsing ingress with tls-cert-vaultt")
 	}
 	_, ok := i.(string)
 	if !ok {
@@ -61,15 +61,15 @@ func TestParseInvalidAnnotations(t *testing.T) {
 	// Test with empty annotations
 	i, err = NewParser(&resolver.Mock{}).Parse(ing)
 	if err != nil {
-		t.Errorf("unexpected error parsing ingress with default-ssl-certificate-vault")
+		t.Errorf("unexpected error parsing ingress with tls-cert-vault")
 	}
 	_, ok = i.(string)
 	if !ok {
 		t.Errorf("expected a string type")
 	}
 
 	// Test invalid annotation set
-	data[parser.GetAnnotationWithPrefix("default-ssl-certificate-vault")] = "INVALID&data"
+	data[parser.GetAnnotationWithPrefix("tls-cert-vault")] = "INVALID&data"
 	ing.SetAnnotations(data)
 
 	_, err = NewParser(&resolver.Mock{}).Parse(ing)
@@ -83,12 +83,12 @@ func TestParseAnnotations(t *testing.T) {
 	ing := buildIngress()
 
 	data := map[string]string{}
-	data[parser.GetAnnotationWithPrefix("default-ssl-certificate-vault")] = "userland/certificates/mycertificado"
+	data[parser.GetAnnotationWithPrefix("tls-cert-vault")] = "userland/certificates/mycertificado"
 	ing.SetAnnotations(data)
 
 	i, err := NewParser(&resolver.Mock{}).Parse(ing)
 	if err != nil {
-		t.Errorf("unexpected error parsing ingress with default-ssl-certificate-vault")
+		t.Errorf("unexpected error parsing ingress with tls-cert-vault")
 	}
 	_, ok := i.(string)
 	if !ok {

internal/ingress/controller/controller.go

@@ -1334,8 +1334,9 @@ func (n *NGINXController) createServers(data []*ingress.Ingress,
 			//The certificate is retrieved for storing it in the ingress storage and being able to use it later
 			tlsSecretName := extractTLSSecretName(host, ing, n.store.GetLocalSSLCert)
 
-			klog.V(3).Info("Reading TLS certificates in secretName or in default-ssl-certificate-vault annotation")
+			klog.V(3).Info("Reading TLS certificates in secretName or in tls-cert-vault annotation")
 			// If no certificate stored in Vault is defined in annotations and no secretname stored in k8s, we use default
+			klog.V(3).Info("Printing value of anotation vaultpath: %q", anns.VaultPathTLS)
 			if (anns.VaultPathTLS == "") && (tlsSecretName == "") {
 				klog.V(3).Infof("Host %q is listed in the TLS section but secretName is empty. Using default certificate", host)
 				servers[host].SSLCert = n.getDefaultSSLCertificate()

internal/ingress/controller/nginx.go

@@ -486,7 +486,6 @@ func (n NGINXController) generateTemplate(cfg ngx_config.Configuration, ingressC
 	// https://trac.nginx.org/nginx/ticket/631
 	var longestName int
 	var serverNameBytes int
-
 	for _, srv := range ingressCfg.Servers {
 		hostnameLength := len(srv.Hostname)
 		if srv.RedirectFromToWWW {
@@ -976,6 +975,7 @@ type sslConfiguration struct {
 // configureCertificates JSON encodes certificates and POSTs it to an internal HTTP endpoint
 // that is handled by Lua
 func configureCertificates(rawServers []*ingress.Server) error {
+	// TODO la madre del cordero
 	configuration := &sslConfiguration{
 		Certificates: map[string]string{},
 		Servers:      map[string]string{},

internal/ingress/controller/store/backend_ssl.go

@@ -185,13 +185,20 @@ func (s *k8sStore) getPemCertificate(secretName string, usingVault bool) (*ingre
 	// We  check if we have got a vault secret annotation and proceed accordingly
 	if usingVault {
 		// UID is needed for checking certificates in a later step, but it has to be consistent and not random
-		secretNameHex := hex.EncodeToString([]byte(secretName))
-
+		// In addition, two certificates cannot have the same UID so we are reversing the vault path.
+		vaultSplit := strings.Split(secretName, "/")
+		var secretNameHex string
+		var reverseSecretName string
+		for i := len(vaultSplit) - 1; i > 0; i-- {
+			fmt.Println(vaultSplit[i])
+			klog.V(3).InfoS("Value of reverse chain", "split", (vaultSplit[i]), "chain", (vaultSplit[i]), "i", i)
+			reverseSecretName = reverseSecretName + vaultSplit[i]
+		}
+		secretNameHex = hex.EncodeToString([]byte(reverseSecretName))
 		uid = fmt.Sprintf("%s-%s-%s-%s-%s", secretNameHex[:8], secretNameHex[9:13], secretNameHex[14:18], secretNameHex[19:23], secretNameHex[24:36])
-		klog.InfoS("Trying secret as a Vault Path", "secret", secretName)
+		klog.InfoS("Trying secret as a Vault Path", "secret", secretName, "uid", uid)
 
 		sslCertName = secretName
-		// Check if secretName has a namespace defined
 		sslCertNamespace = "vault"
 
 		if !strings.HasPrefix(sslCertName, "/") {
@@ -270,7 +277,6 @@ func (s *k8sStore) getPemCertificate(secretName string, usingVault bool) (*ingre
 				}
 			}
 		}
-
 		msg := fmt.Sprintf("Configuring Secret %q for TLS encryption (CN: %v)", secretName, sslCert.CN)
 		if ca != nil {
 			msg += " and authentication"
@@ -315,12 +321,22 @@ func (s *k8sStore) getPemCertificate(secretName string, usingVault bool) (*ingre
 	sslCert.Namespace = sslCertNamespace
 
 	// the default SSL certificate needs to be present on disk
-	if secretName == s.defaultSSLCertificate {
+	if s.defaultVaultSSLCertificate != "" {
+		if secretName == s.defaultVaultSSLCertificate {
+			klog.V(3).InfoS("secretName and defaultVaultSSLCertificate are the same, storing it in disk")
+			path, err := ssl.StoreSSLCertOnDisk(nsSecName, sslCert)
+			if err != nil {
+				return nil, fmt.Errorf("storing Vault default SSL Certificate: %w", err)
+			}
+			klog.V(3).InfoS("Doing the sslCert.PemFileName in vault", "path", path)
+			sslCert.PemFileName = path
+		}
+	} else if secretName == s.defaultSSLCertificate {
+		klog.V(3).InfoS("secretName and defaultSSLCertificate are the same, storing it in disk")
 		path, err := ssl.StoreSSLCertOnDisk(nsSecName, sslCert)
 		if err != nil {
 			return nil, fmt.Errorf("storing default SSL Certificate: %w", err)
 		}
-
 		sslCert.PemFileName = path
 	}
 

internal/ingress/controller/store/store.go

@@ -925,7 +925,7 @@ func (s *k8sStore) updateSecretIngressMap(ing *networkingv1.Ingress) {
 		"proxy-ssl-secret",
 		"proxy-ssl-vault",
 		"secure-verify-ca-secret",
-		"default-ssl-certificate-vault",
+		"tls-cert-vault",
 	}
 	for _, ann := range secretAnnotations {
 		klog.V(3).InfoS("Checking annotation for updating Secrets Ingress Map", "annotation", ann)
@@ -950,7 +950,7 @@ func objectRefAnnotationNsKey(ann string, ing *networkingv1.Ingress) (string, er
 	vaultAnnotations := []string{
 		"auth-tls-vault",
 		"proxy-ssl-vault",
-		"default-ssl-certificate-vault",
+		"tls-cert-vault",
 	}
 
 	klog.V(3).InfoS("Getting the annotation", "annotation", ann)
@@ -983,12 +983,17 @@ func (s *k8sStore) syncSecrets(ing *networkingv1.Ingress) {
 	var usingVault bool
 	validVaultUrl := regexp.MustCompile(`^/(?:[\w-]+)[\w\*\.\/\-\_]+$`)
 	key = k8s.MetaNamespaceKey(ing)
+	klog.V(3).InfoS("The key is", "key", key)
+	klog.V(3).Infof("fThe key is", "key", key)
 	for _, secrKey := range s.secretIngressMap.ReferencedBy(key) {
+		klog.V(3).InfoS("The secretKey in the loop", "secrKey", secrKey)
+
 		if validVaultUrl.MatchString(secrKey) {
-			klog.V(3).Infof("Secret key seems a vault url, We are using vault secret stored in  vault to sync the secrets in the ingressmap storage", "key", secrKey)
+			klog.V(3).Infof("Secret defined as %v seems a vault url, We are using vault secret stored in  vault to sync the secrets in the ingressmap storage, for %v", key, secrKey)
 			usingVault = true
 		} else {
-			klog.V(3).Infof("Using k8s secret defined as %v, syncing the secret stored in the ingressmap", "key", secrKey)
+			usingVault = false
+			klog.V(3).Infof("Using k8s secret defined as %v, syncing the secret stored in the ingressmap", key, secrKey)
 		}
 
 		s.syncSecret(secrKey, usingVault)
@@ -1152,7 +1157,7 @@ func (s *k8sStore) GetDefaultBackend() defaults.Backend {
 }
 
 func (s *k8sStore) GetVaultAnnotation(ing *networkingv1.Ingress) (bool, string) {
-	klog.Info("Getting annotation default-ssl-certificate-vault status by checking the annotation field")
+	klog.Info("Getting annotation tls-cert-vault status by checking the annotation field")
 	vaultCertificatePath := annotations.NewAnnotationExtractor(s).Extract(ing).VaultPathTLS
 	if vaultCertificatePath != "" {
 		return true, vaultCertificatePath

rootfs/Dockerfile.debug

@@ -0,0 +1,101 @@
+# Copyright 2015 The Kubernetes Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+FROM qa.int.stratio.com/ingress-nginx/nginx:81c2afd975a6f9a9847184472286044d7d5296f6@sha256:a71ac64dd8cfd68341ba47dbdc4d8c2cb91325fce669875193ea0319118201b5
+
+
+WORKDIR  /etc/nginx
+
+RUN apk update \
+  && apk upgrade \
+  && apk add --no-cache \
+  && apk upgrade openssl --repository=https://dl-cdn.alpinelinux.org/alpine/v3.14/main \  
+  && apk add make --repository=https://dl-cdn.alpinelinux.org/alpine/v3.14/main \
+    diffutils make unzip \
+  && apk upgrade libxml2 \
+  && apk add --no-cache libc6-compat \
+  && apk add gdb \
+  && rm -rf /var/cache/apk/*
+
+# JWT manipulation dependencies
+ENV LUAROCKS_VERSION 3.8.0
+ENV LUAROCKS_SHA ab6612ca9ab87c6984871d2712d05525775e8b50172701a0a1cabddf76de2be7
+
+
+RUN wget -O /tmp/luarocks.tgz \
+  https://github.com/luarocks/luarocks/archive/v${LUAROCKS_VERSION}.tar.gz \
+  && echo "${LUAROCKS_SHA} */tmp/luarocks.tgz" | sha256sum -c - \
+  && tar -C /tmp -xzf /tmp/luarocks.tgz \
+  && cd /tmp/luarocks* \
+  && ./configure \
+  && make install
+
+RUN  luarocks install lua-resty-jwt
+# End manipulation dependencies
+
+
+
+COPY rootfs/etc /etc
+RUN chown -R root:root /etc/nginx
+
+
+COPY rootfs/etc /etc
+RUN chown -R www-data:www-data /etc/nginx
+
+COPY dist/dbg /
+RUN chown -R www-data:www-data /dbg
+COPY dist/nginx-ingress-controller /
+RUN chown -R www-data:www-data /nginx-ingress-controller
+COPY dist/wait-shutdown /
+RUN chown -R www-data:www-data /wait-shutdown
+
+# Fix permission during the build to avoid issues at runtime
+# with volumes (custom templates)
+RUN bash -xeu -c ' \
+  writeDirs=( \
+    /etc/ingress-controller \
+    /etc/ingress-controller/ssl \
+    /etc/ingress-controller/auth \
+    /var/log \
+    /var/log/nginx \
+    /tmp/nginx \
+  ); \
+  for dir in "${writeDirs[@]}"; do \
+    mkdir -p ${dir}; \
+    chown -R www-data.www-data ${dir}; \
+  done'
+
+RUN apk add --no-cache libcap \
+  && setcap    cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap    cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
+  && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
+  && setcap    cap_net_bind_service=+ep /usr/bin/dumb-init \
+  && setcap -v cap_net_bind_service=+ep /usr/bin/dumb-init \
+  && apk del libcap \
+  && ln -sf /usr/local/nginx/sbin/nginx /usr/bin/nginx
+
+
+USER www-data
+
+# Create symlinks to redirect nginx logs to stdout and stderr docker log collector
+RUN  ln -sf /dev/stdout /var/log/nginx/access.log \
+  && ln -sf /dev/stderr /var/log/nginx/error.log
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+
+CMD ["/nginx-ingress-controller"]