test(e2e): Add tests for pulling from mirror

test/e2e/cluster/kind.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -21,8 +22,6 @@ import (
 	"github.com/mesosphere/kubelet-image-credential-provider-shim/test/e2e/seedrng"
 )
 
-const E2EKubeconfig = "e2e-kubeconfig"
-
 // kindCluster represents a KinD cluster.
 type kindCluster struct {
 	name     string
@@ -36,16 +35,22 @@ func NewKinDCluster(
 	ctx context.Context,
 	providerOpts []cluster.ProviderOption,
 	createOpts []cluster.CreateOption,
-) (Cluster, error) {
+) (Cluster, string, error) {
 	seedrng.EnsureSeeded()
 
 	name := strings.ReplaceAll(namesgenerator.GetRandomName(0), "_", "-")
 
 	provider := cluster.NewProvider(providerOpts...)
 
-	// Export kubeconfig to file by default. This allows creating clients from other e2e tests. Can be overridden by
-	// using create option to configure this via `cluster.CreateWithKubeconfigPath` when calling `NewKindCluster`.
-	mergedCreateOpts := []cluster.CreateOption{cluster.CreateWithKubeconfigPath(E2EKubeconfig)}
+	// Do not export kubeconfig to file by default, makes cleanup easier. This can be overridden by using create option
+	// to configure this via `cluster.CreateWithKubeconfigPath` when calling `NewKindCluster`.
+	tempDir, err := os.MkdirTemp("", fmt.Sprintf("%s-kubeconfig-*", name))
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to create temp dir: %w", err)
+	}
+	defer os.RemoveAll(tempDir)
+	tempKubeconfig := filepath.Join(tempDir, "kubeconfig")
+	mergedCreateOpts := []cluster.CreateOption{cluster.CreateWithKubeconfigPath(tempKubeconfig)}
 	mergedCreateOpts = append(mergedCreateOpts, createOpts...)
 
 	kindClusterErr := make(chan error, 1)
@@ -55,39 +60,39 @@ func NewKinDCluster(
 
 	select {
 	case <-ctx.Done():
-		if err := provider.Delete(name, E2EKubeconfig); err != nil {
-			return nil, fmt.Errorf("failed to delete KinD cluster after spec timeout: %w", err)
+		if err := provider.Delete(name, ""); err != nil {
+			return nil, "", fmt.Errorf("failed to delete KinD cluster after spec timeout: %w", err)
 		}
-		return nil, nil
+		return nil, "", nil
 	case err := <-kindClusterErr:
 		if err != nil {
-			return nil, fmt.Errorf("failed to create KinD cluster: %w", err)
+			return nil, "", fmt.Errorf("failed to create KinD cluster: %w", err)
 		}
 	}
 
 	const warningDeleteKinD = "WARNING: failed to delete KinD cluster: %v"
 	kubeconfig, err := provider.KubeConfig(name, false)
 	if err != nil {
-		if deleteErr := provider.Delete(name, E2EKubeconfig); deleteErr != nil {
+		if deleteErr := provider.Delete(name, ""); deleteErr != nil {
 			_, _ = fmt.Fprintf(os.Stderr, warningDeleteKinD, deleteErr)
 		}
-		return nil, fmt.Errorf("failed to retrieve kubeconfig: %w", err)
+		return nil, "", fmt.Errorf("failed to retrieve kubeconfig: %w", err)
 	}
 
 	restConfig, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeconfig))
 	if err != nil {
 		if deleteErr := provider.Delete(name, ""); deleteErr != nil {
 			_, _ = fmt.Fprintf(os.Stderr, warningDeleteKinD, deleteErr)
 		}
-		return nil, fmt.Errorf("failed to build REST config from kubeconfig: %w", err)
+		return nil, "", fmt.Errorf("failed to build REST config from kubeconfig: %w", err)
 	}
 
 	kubeClient, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
 		if deleteErr := provider.Delete(name, ""); deleteErr != nil {
 			_, _ = fmt.Fprintf(os.Stderr, warningDeleteKinD, deleteErr)
 		}
-		return nil, fmt.Errorf("failed to create Kubernetes client: %w", err)
+		return nil, "", fmt.Errorf("failed to create Kubernetes client: %w", err)
 	}
 
 	err = wait.PollInfiniteWithContext(ctx, time.Second*1, func(ctx context.Context) (bool, error) {
@@ -105,16 +110,15 @@ func NewKinDCluster(
 		if err := provider.Delete(name, ""); err != nil {
 			_, _ = fmt.Fprintf(os.Stderr, warningDeleteKinD, err)
 		}
-		return nil, fmt.Errorf("failed to wait for default service account to exist: %w", err)
+		return nil, "", fmt.Errorf("failed to wait for default service account to exist: %w", err)
 	}
 
 	return &kindCluster{
 		name:     name,
 		provider: provider,
-	}, nil
+	}, kubeconfig, nil
 }
 
 func (c *kindCluster) Delete(context.Context) error {
-	_ = os.Remove(E2EKubeconfig)
 	return c.provider.Delete(c.name, "")
 }

test/e2e/docker/client.go

@@ -5,6 +5,8 @@ package docker
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -45,6 +47,7 @@ func RunContainerInBackground(
 	containerName string,
 	containerCfg *container.Config,
 	hostCfg *container.HostConfig,
+	pullUsername, pullPassword string,
 ) (containerInspect types.ContainerJSON, cleanup func(context.Context) error, err error) {
 	dClient, err := ClientFromEnv()
 	if err != nil {
@@ -88,7 +91,7 @@ func RunContainerInBackground(
 		)
 	}
 	defer out.Close()
-	_, _ = io.Copy(io.Discard, out)
+	_, _ = io.Copy(os.Stderr, out)
 
 	created, err := dClient.ContainerCreate(ctx, containerCfg, hostCfg, nil, nil, containerName)
 	if err != nil {
@@ -142,3 +145,58 @@ func ForceDeleteContainer(ctx context.Context, containerID string) error {
 	}
 	return nil
 }
+
+func RetagAndPushImage( //nolint:revive // Lots of args is fine in these tests.
+	ctx context.Context,
+	srcImage, destImage string,
+	pullUsername, pullPassword string,
+	pushUsername, pushPassword string,
+) error {
+	dClient, err := ClientFromEnv()
+	if err != nil {
+		return err
+	}
+
+	out, err := dClient.ImagePull(
+		ctx,
+		srcImage,
+		types.ImagePullOptions{RegistryAuth: authString(pullUsername, pullPassword)},
+	)
+	if err != nil {
+		return fmt.Errorf(
+			"failed to pull image %q: %w",
+			srcImage,
+			err,
+		)
+	}
+	defer out.Close()
+	_, _ = io.Copy(os.Stderr, out)
+
+	if err := dClient.ImageTag(ctx, srcImage, destImage); err != nil {
+		return fmt.Errorf("failed to retag image: %w", err)
+	}
+	defer func() { _, _ = dClient.ImageRemove(ctx, destImage, types.ImageRemoveOptions{}) }()
+
+	out, err = dClient.ImagePush(
+		ctx,
+		destImage,
+		types.ImagePushOptions{RegistryAuth: authString(pushUsername, pushPassword)},
+	)
+	if err != nil {
+		return fmt.Errorf("failed to push retagged image: %w", err)
+	}
+	defer out.Close()
+	_, _ = io.Copy(os.Stderr, out)
+
+	return nil
+}
+
+func authString(username, password string) string {
+	authConfig := types.AuthConfig{
+		Username: username,
+		Password: password,
+	}
+	encodedJSON, _ := json.Marshal(authConfig)
+
+	return base64.URLEncoding.EncodeToString(encodedJSON)
+}

test/e2e/registry/registry.go

@@ -20,12 +20,11 @@ import (
 	"github.com/sethvargo/go-password/password"
 
 	"github.com/mesosphere/kubelet-image-credential-provider-shim/test/e2e/docker"
+	"github.com/mesosphere/kubelet-image-credential-provider-shim/test/e2e/env"
 	"github.com/mesosphere/kubelet-image-credential-provider-shim/test/e2e/seedrng"
 	"github.com/mesosphere/kubelet-image-credential-provider-shim/test/e2e/tls"
 )
 
-const E2ERegistryAddressFile = "e2e-registry-address"
-
 type registryOptions struct {
 	image         string
 	dockerNetwork string
@@ -151,6 +150,8 @@ func NewRegistry(ctx context.Context, opts ...Opt) (*Registry, error) {
 		containerName,
 		&containerCfg,
 		&hostCfg,
+		env.DockerHubUsername(),
+		env.DockerHubPassword(),
 	)
 	if err != nil {
 		if cleanupErr := cleanupTLSCerts(); cleanupErr != nil {
@@ -187,8 +188,6 @@ func NewRegistry(ctx context.Context, opts ...Opt) (*Registry, error) {
 		hostPortAddress: net.JoinHostPort(publishedPort[0].HostIP, publishedPort[0].HostPort),
 		caCertFile:      filepath.Join(tlsCertsDir, "ca.crt"),
 		cleanup: func(ctx context.Context) error {
-			_ = os.Remove(E2ERegistryAddressFile)
-
 			if cleanupErr := cleanupTLSCerts(); cleanupErr != nil {
 				_, _ = fmt.Fprintf(os.Stderr, warningTLSDir, cleanupErr)
 			}

test/e2e/suites/mirror/mirror_suite_test.go

@@ -6,6 +6,7 @@
 package mirror_test
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -32,10 +33,23 @@ func TestE2E(t *testing.T) {
 	RunSpecs(t, "Mirror Suite")
 }
 
+type e2eSetupConfig struct {
+	Registry   e2eRegistryConfig `json:"registry"`
+	Kubeconfig string            `json:"kubeconfig"`
+}
+
+type e2eRegistryConfig struct {
+	Username        string `json:"username"`
+	Password        string `json:"password"`
+	Address         string `json:"address"`
+	HostPortAddress string `json:"hostPortAddress"`
+	CACertFile      string `json:"caCertFile"`
+}
+
 var (
 	kindClusterRESTConfig *rest.Config
 	kindClusterClient     kubernetes.Interface
-	mirrorRegistryAddress string
+	e2eConfig             e2eSetupConfig
 )
 
 //nolint:gosec // No credentials here.
@@ -58,16 +72,13 @@ func testdataPath(f string) string {
 }
 
 var _ = SynchronizedBeforeSuite(
-	func(ctx SpecContext) {
+	func(ctx SpecContext) []byte {
 		By("Starting Docker registry")
 		mirrorRegistry, err := registry.NewRegistry(ctx)
 		Expect(err).ShouldNot(HaveOccurred())
 		DeferCleanup(func(ctx SpecContext) error {
 			return mirrorRegistry.Delete(ctx)
 		}, NodeTimeout(time.Second))
-		Expect(
-			os.WriteFile(registry.E2ERegistryAddressFile, []byte(mirrorRegistry.Address()), 0o600),
-		).To(Succeed())
 
 		By("Setting up kubelet credential providers")
 		providerBinDir := GinkgoT().TempDir()
@@ -134,7 +145,7 @@ var _ = SynchronizedBeforeSuite(
 		)).To(Succeed())
 
 		By("Starting KinD cluster")
-		kindCluster, err := cluster.NewKinDCluster(
+		kindCluster, kubeconfig, err := cluster.NewKinDCluster(
 			ctx,
 			[]kindcluster.ProviderOption{kindcluster.ProviderWithDocker()},
 			[]kindcluster.CreateOption{
@@ -176,18 +187,30 @@ var _ = SynchronizedBeforeSuite(
 		DeferCleanup(func(ctx SpecContext) error {
 			return kindCluster.Delete(ctx)
 		}, NodeTimeout(time.Minute))
+
+		configBytes, _ := json.Marshal(e2eSetupConfig{
+			Registry: e2eRegistryConfig{
+				Username:        mirrorRegistry.Username(),
+				Password:        mirrorRegistry.Password(),
+				Address:         mirrorRegistry.Address(),
+				HostPortAddress: mirrorRegistry.HostPortAddress(),
+				CACertFile:      mirrorRegistry.CACertFile(),
+			},
+			Kubeconfig: kubeconfig,
+		})
+
+		return configBytes
 	},
-	func() {
-		kubeconfigBytes, err := os.ReadFile("e2e-kubeconfig")
-		Expect(err).NotTo(HaveOccurred())
-		kindClusterRESTConfig, err = clientcmd.RESTConfigFromKubeConfig(kubeconfigBytes)
+	func(configBytes []byte) {
+		Expect(json.Unmarshal(configBytes, &e2eConfig)).To(Succeed())
+
+		var err error
+		kindClusterRESTConfig, err = clientcmd.RESTConfigFromKubeConfig(
+			[]byte(e2eConfig.Kubeconfig),
+		)
 		Expect(err).NotTo(HaveOccurred())
 		kindClusterClient, err = kubernetes.NewForConfig(kindClusterRESTConfig)
 		Expect(err).NotTo(HaveOccurred())
-
-		mirrorRegistryAddressBytes, err := os.ReadFile(registry.E2ERegistryAddressFile)
-		Expect(err).NotTo(HaveOccurred())
-		mirrorRegistryAddress = string(mirrorRegistryAddressBytes)
 	},
 	NodeTimeout(time.Minute*2), GracePeriod(time.Minute*2),
 )