From 7c246ec3130862b40bc3b6f993a67ec1be396804 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 27 Apr 2018 14:25:58 +0200 Subject: [PATCH 1/4] Use local image name for e2e tests --- Makefile | 4 ++-- test/manifests/ingress-controller/with-rbac.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9f54a9afe6..910a717d79 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,7 @@ endif $(DOCKER) build -t $(MULTI_ARCH_IMG):$(TAG) $(TEMP_DIR)/rootfs ifeq ($(ARCH), amd64) - # This is for to maintain the backward compatibility + # This is for maintaining backward compatibility $(DOCKER) tag $(MULTI_ARCH_IMG):$(TAG) $(IMAGE):$(TAG) endif @@ -159,7 +159,7 @@ lua-test: .PHONY: e2e-image e2e-image: sub-container-amd64 - TAG=$(TAG) IMAGE=$(MULTI_ARCH_IMG) docker tag $(IMAGE):$(TAG) $(IMAGE):test + $(DOCKER) tag $(MULTI_ARCH_IMG):$(TAG) $(IMGNAME):e2e docker images .PHONY: e2e-test diff --git a/test/manifests/ingress-controller/with-rbac.yaml b/test/manifests/ingress-controller/with-rbac.yaml index be2dd4b54e..9dc1656acb 100644 --- a/test/manifests/ingress-controller/with-rbac.yaml +++ b/test/manifests/ingress-controller/with-rbac.yaml @@ -16,7 +16,7 @@ spec: #serviceAccountName: nginx-ingress-serviceaccount containers: - name: nginx-ingress-controller - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.14.0 + image: nginx-ingress-controller:e2e args: - /nginx-ingress-controller - --default-backend-service=$(POD_NAMESPACE)/default-http-backend From c93f39f01910c89ceb1c558eb0f15cbcd74397f0 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 27 Apr 2018 14:27:20 +0200 Subject: [PATCH 2/4] Bump echoserver version used in e2e test (1.10) --- test/e2e/framework/echo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/framework/echo.go b/test/e2e/framework/echo.go index 9e07277136..bda12f7e5a 100644 --- a/test/e2e/framework/echo.go +++ b/test/e2e/framework/echo.go @@ -60,7 +60,7 @@ func (f *Framework) NewEchoDeploymentWithReplicas(replicas int32) error { Containers: []corev1.Container{ { Name: "http-svc", - Image: "gcr.io/google_containers/echoserver:1.8", + Image: "gcr.io/google_containers/echoserver:1.10", Env: []corev1.EnvVar{}, Ports: []corev1.ContainerPort{ { From 553df8a0cc0f37aa956348f0732c748692b8da58 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 27 Apr 2018 14:29:08 +0200 Subject: [PATCH 3/4] Refactor e2e framework for TLS tests --- .../ingress/controller/store/store_test.go | 6 +- test/e2e/framework/ssl.go | 92 ++++++++++++++----- test/e2e/lua/dynamic_configuration.go | 2 +- test/e2e/settings/no_auth_locations.go | 2 +- test/e2e/settings/proxy_protocol.go | 2 +- test/e2e/settings/server_tokens.go | 2 +- test/e2e/ssl/secret_update.go | 2 +- 7 files changed, 79 insertions(+), 29 deletions(-) diff --git a/internal/ingress/controller/store/store_test.go b/internal/ingress/controller/store/store_test.go index 3376dfa2cc..c329e03b8e 100644 --- a/internal/ingress/controller/store/store_test.go +++ b/internal/ingress/controller/store/store_test.go @@ -304,7 +304,7 @@ func TestStore(t *testing.T) { storer.Run(stopCh) secretName := "not-referenced" - _, _, _, err = framework.CreateIngressTLSSecret(clientSet, []string{"foo"}, secretName, ns) + _, err = framework.CreateIngressTLSSecret(clientSet, []string{"foo"}, secretName, ns) if err != nil { t.Errorf("unexpected error creating secret: %v", err) } @@ -418,7 +418,7 @@ func TestStore(t *testing.T) { t.Errorf("unexpected error waiting for secret: %v", err) } - _, _, _, err = framework.CreateIngressTLSSecret(clientSet, []string{"foo"}, secretName, ns) + _, err = framework.CreateIngressTLSSecret(clientSet, []string{"foo"}, secretName, ns) if err != nil { t.Errorf("unexpected error creating secret: %v", err) } @@ -558,7 +558,7 @@ func TestStore(t *testing.T) { t.Errorf("expected 0 events of type Delete but %v occurred", del) } - _, _, _, err = framework.CreateIngressTLSSecret(clientSet, secretHosts, name, ns) + _, err = framework.CreateIngressTLSSecret(clientSet, secretHosts, name, ns) if err != nil { t.Errorf("unexpected error creating secret: %v", err) } diff --git a/test/e2e/framework/ssl.go b/test/e2e/framework/ssl.go index 5b9fdcf6af..f8f1a700fb 100644 --- a/test/e2e/framework/ssl.go +++ b/test/e2e/framework/ssl.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/rand" "crypto/rsa" + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" @@ -27,11 +28,13 @@ import ( "io" "math/big" "net" + net_url "net/url" "strings" "time" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ) @@ -40,18 +43,22 @@ const ( validFor = 365 * 24 * time.Hour ) -// CreateIngressTLSSecret creates a secret containing TLS certificates for the given Ingress. -// If a secret with the same name already pathExists in the namespace of the -// Ingress, it's updated. -func CreateIngressTLSSecret(client kubernetes.Interface, hosts []string, secretName, namespace string) (host string, rootCA, privKey []byte, err error) { +// CreateIngressTLSSecret creates or updates a Secret containing a TLS +// certificate for the given Ingress and returns a TLS configuration suitable +// for HTTP clients to use against that particular Ingress. +func CreateIngressTLSSecret(client kubernetes.Interface, hosts []string, secretName, namespace string) (*tls.Config, error) { + if len(hosts) == 0 { + return nil, fmt.Errorf("require a non-empty host for client hello") + } + var k, c bytes.Buffer - host = strings.Join(hosts, ",") - if err = generateRSACerts(host, true, &k, &c); err != nil { - return + host := strings.Join(hosts, ",") + if err := generateRSACert(host, true, &k, &c); err != nil { + return nil, err } - cert := c.Bytes() - key := k.Bytes() - secret := &v1.Secret{ + + cert, key := c.Bytes(), k.Bytes() + newSecret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, }, @@ -60,22 +67,31 @@ func CreateIngressTLSSecret(client kubernetes.Interface, hosts []string, secretN v1.TLSPrivateKeyKey: key, }, } - var s *v1.Secret - if s, err = client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{}); err == nil { - s.Data = secret.Data - _, err = client.CoreV1().Secrets(namespace).Update(s) + + var apierr error + curSecret, err := client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err == nil && curSecret != nil { + curSecret.Data = newSecret.Data + _, apierr = client.CoreV1().Secrets(namespace).Update(curSecret) } else { - _, err = client.CoreV1().Secrets(namespace).Create(secret) + _, apierr = client.CoreV1().Secrets(namespace).Create(newSecret) + } + if apierr != nil { + return nil, apierr } - return host, cert, key, err + + serverName := hosts[0] + return tlsConfig(serverName, cert) +} + +// WaitForTLS waits until the TLS handshake with a given server completes successfully. +func WaitForTLS(url string, tlsConfig *tls.Config) error { + return wait.Poll(Poll, 30*time.Second, matchTLSServerName(url, tlsConfig)) } -// generateRSACerts generates a basic self signed certificate using a key length +// generateRSACert generates a basic self signed certificate using a key length // of rsaBits, valid for validFor time. -func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error { - if len(host) == 0 { - return fmt.Errorf("require a non-empty host for client hello") - } +func generateRSACert(host string, isCA bool, keyOut, certOut io.Writer) error { priv, err := rsa.GenerateKey(rand.Reader, rsaBits) if err != nil { return fmt.Errorf("failed to generate key: %v", err) @@ -129,3 +145,37 @@ func generateRSACerts(host string, isCA bool, keyOut, certOut io.Writer) error { } return nil } + +// tlsConfig returns a client TLS configuration for the given server name and +// CA certificate (PEM). +func tlsConfig(serverName string, pemCA []byte) (*tls.Config, error) { + rootCAPool := x509.NewCertPool() + if !rootCAPool.AppendCertsFromPEM(pemCA) { + return nil, fmt.Errorf("error creating CA certificate pool (%s)", serverName) + } + return &tls.Config{ + ServerName: serverName, + RootCAs: rootCAPool, + }, nil +} + +// matchTLSServerName connects to the network address corresponding to the +// given URL using the given TLS configuration and returns whether the TLS +// handshake completed successfully. +func matchTLSServerName(url string, tlsConfig *tls.Config) wait.ConditionFunc { + return func() (ready bool, err error) { + u, err := net_url.Parse(url) + if err != nil { + return + } + + conn, err := tls.Dial("tcp", u.Host, tlsConfig) + if err != nil { + return false, nil + } + conn.Close() + + ready = true + return + } +} diff --git a/test/e2e/lua/dynamic_configuration.go b/test/e2e/lua/dynamic_configuration.go index 14c4a1acd7..c44f560437 100644 --- a/test/e2e/lua/dynamic_configuration.go +++ b/test/e2e/lua/dynamic_configuration.go @@ -184,7 +184,7 @@ var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() { }, } - _, _, _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, + _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, ingress.Spec.TLS[0].Hosts, ingress.Spec.TLS[0].SecretName, ingress.Namespace) diff --git a/test/e2e/settings/no_auth_locations.go b/test/e2e/settings/no_auth_locations.go index 61779db341..a91c473f21 100644 --- a/test/e2e/settings/no_auth_locations.go +++ b/test/e2e/settings/no_auth_locations.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package setting +package settings import ( "fmt" diff --git a/test/e2e/settings/proxy_protocol.go b/test/e2e/settings/proxy_protocol.go index 645ee1b530..0201a1dbdb 100644 --- a/test/e2e/settings/proxy_protocol.go +++ b/test/e2e/settings/proxy_protocol.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package setting +package settings import ( "fmt" diff --git a/test/e2e/settings/server_tokens.go b/test/e2e/settings/server_tokens.go index 3ff80196f6..cbbef9f168 100644 --- a/test/e2e/settings/server_tokens.go +++ b/test/e2e/settings/server_tokens.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package setting +package settings import ( "strings" diff --git a/test/e2e/ssl/secret_update.go b/test/e2e/ssl/secret_update.go index 1f744a6454..e4269730bc 100644 --- a/test/e2e/ssl/secret_update.go +++ b/test/e2e/ssl/secret_update.go @@ -58,7 +58,7 @@ var _ = framework.IngressNginxDescribe("SSL", func() { Expect(err).ToNot(HaveOccurred()) Expect(ing).ToNot(BeNil()) - _, _, _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, + _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, ing.Spec.TLS[0].Hosts, ing.Spec.TLS[0].SecretName, ing.Namespace) From ca423e15679e5e1010b97665aaa02e4b1e44dfd1 Mon Sep 17 00:00:00 2001 From: Antoine Cotten Date: Fri, 27 Apr 2018 20:36:22 +0200 Subject: [PATCH 4/4] Add tests for global TLS settings --- test/e2e/settings/server_tokens.go | 10 +- test/e2e/settings/tls.go | 189 +++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 test/e2e/settings/tls.go diff --git a/test/e2e/settings/server_tokens.go b/test/e2e/settings/server_tokens.go index cbbef9f168..48f35c63cc 100644 --- a/test/e2e/settings/server_tokens.go +++ b/test/e2e/settings/server_tokens.go @@ -49,9 +49,9 @@ var _ = framework.IngressNginxDescribe("Server Tokens", func() { Expect(ing).NotTo(BeNil()) err = f.WaitForNginxConfiguration( - func(server string) bool { - return strings.Contains(server, "server_tokens off") && - strings.Contains(server, "more_set_headers \"Server: \"") + func(cfg string) bool { + return strings.Contains(cfg, "server_tokens off") && + strings.Contains(cfg, "more_set_headers \"Server: \"") }) Expect(err).NotTo(HaveOccurred()) }) @@ -92,8 +92,8 @@ var _ = framework.IngressNginxDescribe("Server Tokens", func() { Expect(ing).NotTo(BeNil()) err = f.WaitForNginxConfiguration( - func(server string) bool { - return strings.Contains(server, "server_tokens on") + func(cfg string) bool { + return strings.Contains(cfg, "server_tokens on") }) Expect(err).NotTo(HaveOccurred()) }) diff --git a/test/e2e/settings/tls.go b/test/e2e/settings/tls.go new file mode 100644 index 0000000000..80fb97aa32 --- /dev/null +++ b/test/e2e/settings/tls.go @@ -0,0 +1,189 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package settings + +import ( + "crypto/tls" + "fmt" + "net/http" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Settings - TLS)", func() { + f := framework.NewDefaultFramework("settings-tls") + host := "settings-tls" + + BeforeEach(func() { + err := f.NewEchoDeployment() + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + }) + + It("should configure TLS protocol", func() { + sslCiphers := "ssl-ciphers" + sslProtocols := "ssl-protocols" + + // Two ciphers supported by each of TLSv1.2 and TLSv1. + // https://www.openssl.org/docs/man1.1.0/apps/ciphers.html - "CIPHER SUITE NAMES" + testCiphers := "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA" + + tlsConfig, err := tlsEndpoint(f, host) + Expect(err).NotTo(HaveOccurred()) + + err = framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig) + Expect(err).NotTo(HaveOccurred()) + + By("setting cipher suite") + + err = f.UpdateNginxConfigMapData(sslCiphers, testCiphers) + Expect(err).NotTo(HaveOccurred()) + + err = f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, fmt.Sprintf("ssl_ciphers '%s';", testCiphers)) + }) + Expect(err).NotTo(HaveOccurred()) + + resp, _, errs := gorequest.New(). + Get(f.IngressController.HTTPSURL). + TLSClientConfig(tlsConfig). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(resp.TLS.Version).Should(BeNumerically("==", tls.VersionTLS12)) + Expect(resp.TLS.CipherSuite).Should(BeNumerically("==", tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384)) + + By("enforcing TLS v1.0") + + err = f.UpdateNginxConfigMapData(sslProtocols, "TLSv1") + Expect(err).NotTo(HaveOccurred()) + + err = f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "ssl_protocols TLSv1;") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPSURL). + TLSClientConfig(tlsConfig). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(resp.TLS.Version).Should(BeNumerically("==", tls.VersionTLS10)) + Expect(resp.TLS.CipherSuite).Should(BeNumerically("==", tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA)) + }) + + It("should configure HSTS policy header", func() { + hstsMaxAge := "hsts-max-age" + hstsIncludeSubdomains := "hsts-include-subdomains" + hstsPreload := "hsts-preload" + + tlsConfig, err := tlsEndpoint(f, host) + Expect(err).NotTo(HaveOccurred()) + + err = framework.WaitForTLS(f.IngressController.HTTPSURL, tlsConfig) + Expect(err).NotTo(HaveOccurred()) + + By("setting max-age parameter") + + err = f.UpdateNginxConfigMapData(hstsMaxAge, "86400") + Expect(err).NotTo(HaveOccurred()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "Strict-Transport-Security: max-age=86400; includeSubDomains\"") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, _, errs := gorequest.New(). + Get(f.IngressController.HTTPSURL). + TLSClientConfig(tlsConfig). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(resp.Header.Get("Strict-Transport-Security")).Should(ContainSubstring("max-age=86400")) + + By("setting includeSubDomains parameter") + + err = f.UpdateNginxConfigMapData(hstsIncludeSubdomains, "false") + Expect(err).NotTo(HaveOccurred()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "Strict-Transport-Security: max-age=86400\"") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPSURL). + TLSClientConfig(tlsConfig). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(resp.Header.Get("Strict-Transport-Security")).ShouldNot(ContainSubstring("includeSubDomains")) + + By("setting preload parameter") + + err = f.UpdateNginxConfigMapData(hstsPreload, "true") + Expect(err).NotTo(HaveOccurred()) + + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "Strict-Transport-Security: max-age=86400; preload\"") + }) + Expect(err).NotTo(HaveOccurred()) + + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPSURL). + TLSClientConfig(tlsConfig). + Set("Host", host). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(resp.Header.Get("Strict-Transport-Security")).Should(ContainSubstring("preload")) + }) +}) + +func tlsEndpoint(f *framework.Framework, host string) (*tls.Config, error) { + ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, nil)) + if err != nil { + return nil, err + } + + return framework.CreateIngressTLSSecret(f.KubeClientSet, + ing.Spec.TLS[0].Hosts, + ing.Spec.TLS[0].SecretName, + ing.Namespace) +}