Skip to content

Commit

Permalink
test: Add e2e tests for serve and push with TLS
Browse files Browse the repository at this point in the history
Unfortunately helm client does not support custom CA certificates or
self-signed certs so the validation has been disabled, but the e2e tests
still add some value.
  • Loading branch information
jimmidyson committed Sep 13, 2022
1 parent 1b0a3d2 commit 2d452b4
Show file tree
Hide file tree
Showing 5 changed files with 337 additions and 104 deletions.
4 changes: 2 additions & 2 deletions test/e2e/helmbundle/create_bundle_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"github.com/spf13/cobra"

createhelmbundle "github.com/mesosphere/mindthegap/cmd/mindthegap/create/helmbundle"
"github.com/mesosphere/mindthegap/test/e2e/helpers"
"github.com/mesosphere/mindthegap/test/e2e/helmbundle/helpers"
)

var _ = Describe("Create Bundle", func() {
Expand All @@ -28,7 +28,7 @@ var _ = Describe("Create Bundle", func() {

bundleFile = filepath.Join(tmpDir, "helm-bundle.tar")

cmd = helpers.NewCommand(createhelmbundle.NewCommand)
cmd = helpers.NewCommand(GinkgoT(), createhelmbundle.NewCommand)
cmd.SilenceUsage = true
})

Expand Down
206 changes: 206 additions & 0 deletions test/e2e/helmbundle/helpers/helpers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
// Copyright 2021 D2iQ, Inc. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//go:build e2e
// +build e2e

package helpers

import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"math/big"
"net"
"os"
"path/filepath"
"strconv"
"time"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"

"github.com/mesosphere/dkp-cli-runtime/core/output"

createhelmbundle "github.com/mesosphere/mindthegap/cmd/mindthegap/create/helmbundle"
"github.com/mesosphere/mindthegap/helm"
)

func CreateBundle(t ginkgo.GinkgoTInterface, bundleFile, cfgFile string) {
createBundleCmd := NewCommand(t, createhelmbundle.NewCommand)
createBundleCmd.SetArgs([]string{
"--output-file", bundleFile,
"--helm-charts-file", cfgFile,
})
gomega.ExpectWithOffset(1, createBundleCmd.Execute()).To(gomega.Succeed())
}

func NewCommand(
t ginkgo.GinkgoTInterface,
newFn func(out output.Output) *cobra.Command,
) *cobra.Command {
t.Helper()
cmd := newFn(output.NewNonInteractiveShell(ginkgo.GinkgoWriter, ginkgo.GinkgoWriter, 10))
cmd.SilenceUsage = true
return cmd
}

// GetFirstNonLoopbackIP returns the first non-loopback IP of the current host.
func GetFirstNonLoopbackIP(t ginkgo.GinkgoTInterface) net.IP {
t.Helper()
addrs, err := net.InterfaceAddrs()
gomega.ExpectWithOffset(1, err).ToNot(gomega.HaveOccurred())
for _, address := range addrs {
// check the address type and if it is not a loopback the display it
if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil {
return ipnet.IP
}
}
}
ginkgo.Fail("no available non-loopback IP address")
return net.IP{}
}

func WaitForTCPPort(t ginkgo.GinkgoTInterface, addr string, port int) {
t.Helper()
gomega.EventuallyWithOffset(1, func() error {
conn, err := net.DialTimeout(
"tcp",
net.JoinHostPort(addr, strconv.Itoa(port)),
1*time.Second,
)
if err != nil {
return err
}
defer conn.Close()
return nil
}, 5*time.Second).Should(gomega.Succeed())
}

func GenerateCertificateAndKeyWithIPSAN(
t ginkgo.GinkgoTInterface, destDir string, ipAddr net.IP,
) (caCertFile, caKeyFile, certFile, keyFile string) {
t.Helper()

caPriv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
caTemplate := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
Organization: []string{"d2iq", "mindthegap", "e2e-ca"},
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(1 * time.Hour),

KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
IsCA: true,
}

caDerBytes, err := x509.CreateCertificate(
rand.Reader,
&caTemplate,
&caTemplate,
&caPriv.PublicKey,
caPriv,
)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())

caCertFile = filepath.Join(destDir, "ca.crt")
caCertF, err := os.Create(caCertFile)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
defer caCertF.Close()
gomega.ExpectWithOffset(1, pem.Encode(caCertF, &pem.Block{Type: "CERTIFICATE", Bytes: caDerBytes})).
To(gomega.Succeed())

b, err := x509.MarshalECPrivateKey(caPriv)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
pemBlock := pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
caKeyFile = filepath.Join(destDir, "ca.key")
caKeyF, err := os.Create(caKeyFile)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
defer caKeyF.Close()
gomega.ExpectWithOffset(1, pem.Encode(caKeyF, &pemBlock)).To(gomega.Succeed())

priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
template := x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{
Organization: []string{"d2iq", "mindthegap", "e2e"},
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(1 * time.Hour),

KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}

template.IPAddresses = append(template.IPAddresses, ipAddr)

derBytes, err := x509.CreateCertificate(
rand.Reader,
&template,
&caTemplate,
&priv.PublicKey,
caPriv,
)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())

certFile = filepath.Join(destDir, "tls.crt")
certF, err := os.Create(certFile)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
defer certF.Close()
gomega.ExpectWithOffset(1, pem.Encode(certF, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})).
To(gomega.Succeed())

b, err = x509.MarshalECPrivateKey(priv)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
pemBlock = pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
keyFile = filepath.Join(destDir, "tls.key")
keyF, err := os.Create(keyFile)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
defer keyF.Close()
gomega.ExpectWithOffset(1, pem.Encode(keyF, &pemBlock)).To(gomega.Succeed())

return caCertFile, caKeyFile, certFile, keyFile
}

func ValidateChartIsAvailable(
t ginkgo.GinkgoTInterface,
addr string,
port int,
chartName, chartVersion string,
pullOpts ...action.PullOpt,
) {
t.Helper()
h, cleanup := helm.NewClient(
output.NewNonInteractiveShell(ginkgo.GinkgoWriter, ginkgo.GinkgoWriter, 10),
)
ginkgo.DeferCleanup(cleanup)

helmTmpDir := t.TempDir()

d, err := h.GetChartFromRepo(
helmTmpDir,
"",
fmt.Sprintf("%s://%s:%v/charts/%s", helm.OCIScheme, addr, port, chartName),
chartVersion,
[]helm.ConfigOpt{helm.RegistryClientConfigOpt()},
pullOpts...,
)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
chrt, err := helm.LoadChart(d)
gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
gomega.ExpectWithOffset(1, chrt.Metadata.Name).To(gomega.Equal(chartName))
gomega.ExpectWithOffset(1, chrt.Metadata.Version).To(gomega.Equal(chartVersion))
}
112 changes: 69 additions & 43 deletions test/e2e/helmbundle/push_bundle_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,17 @@ package helmbundle_test
import (
"context"
"fmt"
"net"
"path/filepath"
"strconv"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/phayes/freeport"
"github.com/spf13/cobra"
"helm.sh/helm/v3/pkg/action"

"github.com/mesosphere/dkp-cli-runtime/core/output"

createhelmbundle "github.com/mesosphere/mindthegap/cmd/mindthegap/create/helmbundle"
pushhelmbundle "github.com/mesosphere/mindthegap/cmd/mindthegap/push/helmbundle"
"github.com/mesosphere/mindthegap/docker/registry"
"github.com/mesosphere/mindthegap/helm"
"github.com/mesosphere/mindthegap/test/e2e/helpers"
"github.com/mesosphere/mindthegap/test/e2e/helmbundle/helpers"
)

var _ = Describe("Push Bundle", func() {
Expand All @@ -41,16 +34,15 @@ var _ = Describe("Push Bundle", func() {

bundleFile = filepath.Join(tmpDir, "helm-bundle.tar")

cmd = helpers.NewCommand(pushhelmbundle.NewCommand)
cmd = helpers.NewCommand(GinkgoT(), pushhelmbundle.NewCommand)
})

It("Success", func() {
createBundleCmd := helpers.NewCommand(createhelmbundle.NewCommand)
createBundleCmd.SetArgs([]string{
"--output-file", bundleFile,
"--helm-charts-file", filepath.Join("testdata", "create-success.yaml"),
})
Expect(createBundleCmd.Execute()).To(Succeed())
It("Without TLS", func() {
helpers.CreateBundle(
GinkgoT(),
bundleFile,
filepath.Join("testdata", "create-success.yaml"),
)

port, err := freeport.GetFreePort()
Expect(err).NotTo(HaveOccurred())
Expand All @@ -69,24 +61,7 @@ var _ = Describe("Push Bundle", func() {
close(done)
}()

Eventually(func() error {
conn, err := net.DialTimeout(
"tcp",
net.JoinHostPort("localhost", strconv.Itoa(port)),
1*time.Second,
)
DeferCleanup(func() {
if conn != nil {
conn.Close()
}
})
return err
}).ShouldNot(HaveOccurred())

h, cleanup := helm.NewClient(output.NewNonInteractiveShell(GinkgoWriter, GinkgoWriter, 10))
DeferCleanup(cleanup)

helmTmpDir := GinkgoT().TempDir()
helpers.WaitForTCPPort(GinkgoT(), "localhost", port)

cmd.SetArgs([]string{
"--helm-bundle", bundleFile,
Expand All @@ -96,19 +71,70 @@ var _ = Describe("Push Bundle", func() {

Expect(cmd.Execute()).To(Succeed())

d, err := h.GetChartFromRepo(
helmTmpDir,
"",
fmt.Sprintf("%s://localhost:%v/charts/podinfo", helm.OCIScheme, port),
helpers.ValidateChartIsAvailable(
GinkgoT(),
"localhost",
port,
"podinfo",
"6.2.0",
[]helm.ConfigOpt{helm.RegistryClientConfigOpt()},
func(p *action.Pull) { p.InsecureSkipTLSverify = true },
helm.InsecureSkipTLSverifyOpt(),
)

Expect(reg.Shutdown(context.Background())).To((Succeed()))

Eventually(done).Should(BeClosed())
})

It("With TLS", func() {
helpers.CreateBundle(
GinkgoT(),
bundleFile,
filepath.Join("testdata", "create-success.yaml"),
)

ipAddr := helpers.GetFirstNonLoopbackIP(GinkgoT())

tempCertDir := GinkgoT().TempDir()
_, _, certFile, keyFile := helpers.GenerateCertificateAndKeyWithIPSAN(
GinkgoT(),
tempCertDir,
ipAddr,
)

port, err := freeport.GetFreePort()
Expect(err).NotTo(HaveOccurred())
chrt, err := helm.LoadChart(d)
reg, err := registry.NewRegistry(registry.Config{
StorageDirectory: filepath.Join(tmpDir, "registry"),
Host: ipAddr.String(),
Port: uint16(port),
TLS: registry.TLS{
Certificate: certFile,
Key: keyFile,
},
})
Expect(err).NotTo(HaveOccurred())
Expect(chrt.Metadata.Name).To(Equal("podinfo"))
Expect(chrt.Metadata.Version).To(Equal("6.2.0"))

done := make(chan struct{})
go func() {
defer GinkgoRecover()

Expect(reg.ListenAndServe()).To(Succeed())

close(done)
}()

helpers.WaitForTCPPort(GinkgoT(), ipAddr.String(), port)

cmd.SetArgs([]string{
"--helm-bundle", bundleFile,
"--to-registry", fmt.Sprintf("%s:%v/charts", ipAddr, port),
"--to-registry-insecure-skip-tls-verify",
})

Expect(cmd.Execute()).To(Succeed())

// TODO Reenable once Helm supports custom CA certs and self-signed certs.
// helpers.ValidateChartIsAvailable(GinkgoT(), ipAddr.String(), port, "podinfo", "6.2.0", helm.CAFileOpt(caCertFile))

Expect(reg.Shutdown(context.Background())).To((Succeed()))

Expand Down
Loading

0 comments on commit 2d452b4

Please sign in to comment.