test: bind mount gomodcache to cache go dependencies (#41846)

* test: bind mount gomodcache to cache go dependencies

try to speed up ci jobs integration tests

* test(metricbeat): do not make gomodcache readonly

* lint: replace ioutil usage

* test: fail on empty gomodcache

* test: rework cache bind approach

instead of binding the volume on each dockerfile, run the container
with the proper volume arg

* test: bind gomodcache in kubernetes runner

* test: sync the gomodcache with rsync and fix linter issues

can't really bind using hostpath in k8s

* test: rework bind approach with k8s runner

since it's a local cluster, use kind to bind the hostpath to the node
then bind the node path to the container path

* lint: remove unused imports

* lint: fix linter issues

dev-tools/mage/integtest_docker.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"go/build"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -99,11 +98,9 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
 	if err != nil {
 		return err
 	}
-	dockerRepoRoot := filepath.Join("/go/src", repo.CanonicalRootImportPath)
-	dockerGoCache := filepath.Join(dockerRepoRoot, "build/docker-gocache")
 	magePath := filepath.Join("/go/src", repo.CanonicalRootImportPath, repo.SubDir, "build/mage-linux-"+GOARCH)
-	goPkgCache := filepath.Join(filepath.SplitList(build.Default.GOPATH)[0], "pkg/mod/cache/download")
-	dockerGoPkgCache := "/gocache"
+	goPkgCache := filepath.Join(filepath.SplitList(build.Default.GOPATH)[0], "pkg/mod")
+	dockerGoPkgCache := "/go/pkg/mod"
 
 	// Execute the inside of docker-compose.
 	args := []string{"-p", DockerComposeProjectName(), "run",
@@ -114,10 +111,8 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
 		// compose.EnsureUp needs to know the environment type.
 		"-e", "STACK_ENVIRONMENT=" + StackEnvironment,
 		"-e", "TESTING_ENVIRONMENT=" + StackEnvironment,
-		"-e", "GOCACHE=" + dockerGoCache,
 		// Use the host machine's pkg cache to minimize external downloads.
-		"-v", goPkgCache + ":" + dockerGoPkgCache + ":ro",
-		"-e", "GOPROXY=file://" + dockerGoPkgCache + ",direct",
+		"-v", goPkgCache + ":" + dockerGoPkgCache,
 	}
 	args, err = addUidGidEnvArgs(args)
 	if err != nil {
@@ -356,7 +351,7 @@ func StartIntegTestContainers() error {
 
 func StopIntegTestContainers() error {
 	// Docker-compose rm is noisy. So only pass through stderr when in verbose.
-	out := ioutil.Discard
+	out := io.Discard
 	if mg.Verbose() {
 		out = os.Stderr
 	}
@@ -368,7 +363,7 @@ func StopIntegTestContainers() error {
 
 	_, err = sh.Exec(
 		composeEnv,
-		ioutil.Discard,
+		io.Discard,
 		out,
 		"docker-compose",
 		"-p", DockerComposeProjectName(),

dev-tools/mage/kubernetes/kind.go

@@ -19,7 +19,8 @@ package kubernetes
 
 import (
 	"fmt"
-	"io/ioutil"
+	"go/build"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -66,8 +67,8 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 	}
 
 	clusterName := kubernetesClusterName()
-	stdOut := ioutil.Discard
-	stdErr := ioutil.Discard
+	stdOut := io.Discard
+	stdErr := io.Discard
 	if mg.Verbose() {
 		stdOut = os.Stdout
 		stdErr = os.Stderr
@@ -86,9 +87,29 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 		return err
 	}
 
+	cfg, err := os.CreateTemp("", "kind-")
+	if err != nil {
+		return err
+	}
+	if _, err := cfg.WriteString(fmt.Sprintf(`
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+nodes:
+  - role: control-plane
+    extraMounts:
+      - hostPath: %s
+        containerPath: /go/pkg/mod
+`, filepath.Join(build.Default.GOPATH, "pkg", "mod"))); err != nil {
+		return err
+	}
+	if err := cfg.Close(); err != nil {
+		return err
+	}
+
 	args := []string{
 		"create",
 		"cluster",
+		"--config", cfg.Name(),
 		"--name", clusterName,
 		"--kubeconfig", kubeConfig,
 		"--wait",
@@ -116,8 +137,8 @@ func (m *KindIntegrationTestStep) Setup(env map[string]string) error {
 
 // Teardown destroys the kubernetes cluster.
 func (m *KindIntegrationTestStep) Teardown(env map[string]string) error {
-	stdOut := ioutil.Discard
-	stdErr := ioutil.Discard
+	stdOut := io.Discard
+	stdErr := io.Discard
 	if mg.Verbose() {
 		stdOut = os.Stdout
 		stdErr = os.Stderr

dev-tools/mage/kubernetes/kuberemote.go

@@ -23,14 +23,14 @@ import (
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/x509"
-	"encoding/hex"
 	"encoding/pem"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"log"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"os/exec"
 	"strings"
 	"time"
@@ -127,11 +127,17 @@ func (r *KubeRemote) Run(env map[string]string, stdout io.Writer, stderr io.Writ
 	if err != nil {
 		return err
 	}
-	go f.ForwardPorts()
+	go func() {
+		if err := f.ForwardPorts(); err != nil {
+			log.Printf("forward port error: %v\n", err)
+		}
+	}()
 	<-readyChannel
 
 	// perform the rsync
-	r.rsync(randomPort, stderr, stderr)
+	if err := r.rsync(randomPort, stderr, stderr, r.syncDir, r.destDir); err != nil {
+		return fmt.Errorf("rsync failed: %w", err)
+	}
 
 	// stop port forwarding
 	close(stopChannel)
@@ -258,14 +264,14 @@ func (r *KubeRemote) portForward(ports []string, stopChannel, readyChannel chan
 	}
 
 	path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", r.namespace, r.name)
-	hostIP := strings.TrimLeft(r.cfg.Host, "https://")
+	hostIP := strings.TrimPrefix(r.cfg.Host, "https://")
 	serverURL := url.URL{Scheme: "https", Path: path, Host: hostIP}
 	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, &serverURL)
 	return portforward.New(dialer, ports, stopChannel, readyChannel, stdout, stderr)
 }
 
 // rsync performs the rsync of sync directory to destination directory inside of the pod.
-func (r *KubeRemote) rsync(port uint16, stdout, stderr io.Writer) error {
+func (r *KubeRemote) rsync(port uint16, stdout, stderr io.Writer, src string, dst string) error {
 	privateKeyFile, err := createTempFile(r.privateKey)
 	if err != nil {
 		return err
@@ -274,8 +280,8 @@ func (r *KubeRemote) rsync(port uint16, stdout, stderr io.Writer) error {
 	rsh := fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -p %d -i %s", port, privateKeyFile)
 	args := []string{
 		"--rsh", rsh,
-		"-a", fmt.Sprintf("%s/", r.syncDir),
-		fmt.Sprintf("root@localhost:%s", r.destDir),
+		"-a", fmt.Sprintf("%s/", src),
+		fmt.Sprintf("root@localhost:%s", dst),
 	}
 	cmd := exec.Command("rsync", args...)
 	cmd.Stdout = stdout
@@ -491,6 +497,10 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 							Name:      "destdir",
 							MountPath: destDir,
 						},
+						{
+							Name:      "gomodcache",
+							MountPath: "/go/pkg/mod",
+						},
 					},
 				},
 			},
@@ -510,6 +520,14 @@ func createPodManifest(name string, image string, env map[string]string, cmd []s
 						EmptyDir: &apiv1.EmptyDirVolumeSource{},
 					},
 				},
+				{
+					Name: "gomodcache",
+					VolumeSource: apiv1.VolumeSource{
+						HostPath: &apiv1.HostPathVolumeSource{
+							Path: "/go/pkg/mod",
+						},
+					},
+				},
 			},
 		},
 	}
@@ -542,7 +560,7 @@ func isInitContainersReady(pod *apiv1.Pod) bool {
 }
 
 func isScheduled(pod *apiv1.Pod) bool {
-	if &pod.Status != nil && len(pod.Status.Conditions) > 0 {
+	if len(pod.Status.Conditions) > 0 {
 		for _, condition := range pod.Status.Conditions {
 			if condition.Type == apiv1.PodScheduled &&
 				condition.Status == apiv1.ConditionTrue {
@@ -554,18 +572,15 @@ func isScheduled(pod *apiv1.Pod) bool {
 }
 
 func isInitContainersRunning(pod *apiv1.Pod) bool {
-	if &pod.Status != nil {
-		if len(pod.Spec.InitContainers) != len(pod.Status.InitContainerStatuses) {
+	if len(pod.Spec.InitContainers) != len(pod.Status.InitContainerStatuses) {
+		return false
+	}
+	for _, status := range pod.Status.InitContainerStatuses {
+		if status.State.Running == nil {
 			return false
 		}
-		for _, status := range pod.Status.InitContainerStatuses {
-			if status.State.Running == nil {
-				return false
-			}
-		}
-		return true
 	}
-	return false
+	return true
 }
 
 func containerRunning(containerName string) func(watch.Event) (bool, error) {
@@ -620,9 +635,7 @@ func podDone(event watch.Event) (bool, error) {
 }
 
 func createTempFile(content []byte) (string, error) {
-	randBytes := make([]byte, 16)
-	rand.Read(randBytes)
-	tmpfile, err := ioutil.TempFile("", hex.EncodeToString(randBytes))
+	tmpfile, err := os.CreateTemp("", "kuberemote-")
 	if err != nil {
 		return "", err
 	}