Optimize Dockerized integTest with GOCACHE (#17957)

Create a reusable GOCACHE located at $repoRoot/docker-gocache to speed up building the Linux
system test binaries and running tests.

This can yield proportionally large speed-up when doing selective testing as you often do in the development cycle.
For example when testing a single module in x-pack/filebeat:

```
NOSE_TESTMATCH=misp time mage -v pythonIntegTest
      199.62 real        20.86 user        35.55 sys

NOSE_TESTMATCH=misp time mage -v pythonIntegTest
      104.97 real        21.44 user        35.55 sys
```

I also compared running mage integTest for x-pack/filebeat without a cache then with a cache:

```
Go Unit Test: 2m35s
Go Test Compile: 30.2s
real	12m2.626s
user	0m21.141s
sys	0m35.212s
```

```
Go Unit Test: 47s
Go Test Compile: 36s
real	9m16.326s
user	0m21.697s
sys	0m38.908s
```

dev-tools/mage/crossbuild.go

@@ -297,7 +297,9 @@ func DockerChown(path string) {
 func chownPaths(uid, gid int, path string) error {
 	start := time.Now()
 	numFixed := 0
-	defer log.Printf("chown took: %v, changed %d files", time.Now().Sub(start), numFixed)
+	defer func() {
+		log.Printf("chown took: %v, changed %d files", time.Now().Sub(start), numFixed)
+	}()
 
 	return filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
 		if err != nil {

dev-tools/mage/gotest.go

@@ -456,5 +456,10 @@ func BuildSystemTestGoBinary(binArgs TestBinaryArgs) error {
 	if len(binArgs.InputFiles) > 0 {
 		args = append(args, binArgs.InputFiles...)
 	}
+
+	start := time.Now()
+	defer func() {
+		log.Printf("BuildSystemTestGoBinary (go %v) took %v.", strings.Join(args, " "), time.Since(start))
+	}()
 	return sh.RunV("go", args...)
 }

dev-tools/mage/integtest_docker.go

@@ -90,17 +90,20 @@ func (d *DockerIntegrationTester) Test(_ string, mageTarget string, env map[stri
 	if err != nil {
 		return err
 	}
+	dockerRepoRoot := filepath.Join("/go/src", repo.CanonicalRootImportPath)
+	dockerGoCache := filepath.Join(dockerRepoRoot, "build/docker-gocache")
 	magePath := filepath.Join("/go/src", repo.CanonicalRootImportPath, repo.SubDir, "build/mage-linux-amd64")
 
 	// Execute the inside of docker-compose.
 	args := []string{"-p", dockerComposeProjectName(), "run",
 		"-e", "DOCKER_COMPOSE_PROJECT_NAME=" + dockerComposeProjectName(),
-		// Disable strict.perms because we moust host dirs inside containers
+		// Disable strict.perms because we mount host dirs inside containers
 		// and the UID/GID won't meet the strict requirements.
 		"-e", "BEAT_STRICT_PERMS=false",
 		// compose.EnsureUp needs to know the environment type.
 		"-e", "STACK_ENVIRONMENT=" + StackEnvironment,
 		"-e", "TESTING_ENVIRONMENT=" + StackEnvironment,
+		"-e", "GOCACHE=" + dockerGoCache,
 	}
 	args, err = addUidGidEnvArgs(args)
 	if err != nil {

libbeat/docker-compose.yml

@@ -5,7 +5,6 @@ services:
     depends_on:
       - proxy_dep
     environment:
-      - LIBBEAT_PATH=/go/src/github.com/elastic/beats/libbeat
       - BEAT_STRICT_PERMS=false
       - REDIS_HOST=redis
       - REDIS_PORT=6379

packetbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.7
+FROM golang:1.13.10
 
 RUN \
     apt-get update \