all: update linux-ppc64le builders

Fixes golang/go#21189
Updates golang/go#21260
Updates golang/go#34830 (ppc64 big endian remains)

Change-Id: I7ff38edb519bf4f34cd05bb795b6a41f72b10640
Reviewed-on: https://go-review.googlesource.com/c/build/+/201637
Reviewed-by: Carlos Eduardo Seo <[email protected]>
Reviewed-by: Bryan C. Mills <[email protected]>

cmd/buildlet/reverse.go

@@ -71,7 +71,10 @@ func dialCoordinator() error {
 	devMode := isDevReverseMode()
 
 	if *hostname == "" {
-		*hostname, _ = os.Hostname()
+		*hostname = os.Getenv("HOSTNAME")
+		if *hostname == "" {
+			*hostname, _ = os.Hostname()
+		}
 	}
 
 	var modes, keys []string

cmd/buildlet/stage0/stage0.go

@@ -92,8 +92,6 @@ func main() {
 		}
 	case "linux/ppc64":
 		initOregonStatePPC64()
-	case "linux/ppc64le":
-		initOregonStatePPC64le()
 	case "darwin/amd64":
 		// The MacStadium builders' baked-in stage0.sh
 		// bootstrap file doesn't set GO_BUILDER_ENV
@@ -167,6 +165,10 @@ Download:
 	case "host-linux-mipsle-mengzhuo":
 		cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...)
 		cmd.Args = append(cmd.Args, os.ExpandEnv("--workdir=${WORKDIR}"))
+	case "host-linux-ppc64le-power9-osu":
+		cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...)
+	case "host-linux-ppc64le-osu": // power8
+		cmd.Args = append(cmd.Args, reverseHostTypeArgs(buildEnv)...)
 	}
 	switch osArch {
 	case "linux/s390x":
@@ -191,10 +193,6 @@ Download:
 		// Assume OSU (osuosl.org) host type for now. If we get more, use
 		// GO_BUILD_HOST_TYPE (see above) and check that.
 		cmd.Args = append(cmd.Args, reverseHostTypeArgs("host-linux-ppc64-osu")...)
-	case "linux/ppc64le":
-		// Assume OSU (osuosl.org) host type for now. If we get more, use
-		// GO_BUILD_HOST_TYPE (see above) and check that.
-		cmd.Args = append(cmd.Args, reverseHostTypeArgs("host-linux-ppc64le-osu")...)
 	case "solaris/amd64", "illumos/amd64":
 		hostType := buildEnv
 		cmd.Args = append(cmd.Args, reverseHostTypeArgs(hostType)...)
@@ -416,11 +414,6 @@ func initOregonStatePPC64() {
 	initBootstrapDir("/usr/local/go-bootstrap", "/usr/local/go-bootstrap.tar.gz")
 }
 
-func initOregonStatePPC64le() {
-	aptGetInstall("gcc", "strace", "libc6-dev", "gdb")
-	initBootstrapDir("/usr/local/go-bootstrap", "/usr/local/go-bootstrap.tar.gz")
-}
-
 func isUnix() bool {
 	switch runtime.GOOS {
 	case "plan9", "windows":

cmd/coordinator/status.go

@@ -141,6 +141,7 @@ func addHealthCheckers(ctx context.Context) {
 	addHealthChecker(newPacketHealthChecker())
 	addHealthChecker(newOSUPPC64Checker())
 	addHealthChecker(newOSUPPC64leChecker())
+	addHealthChecker(newOSUPPC64lePower9Checker())
 	addHealthChecker(newBasepinChecker())
 	addHealthChecker(newGitMirrorChecker())
 	addHealthChecker(newTipGolangOrgChecker(ctx))
@@ -440,12 +441,26 @@ func newOSUPPC64Checker() *healthChecker {
 func newOSUPPC64leChecker() *healthChecker {
 	var hosts []string
 	for i := 1; i <= expectedHosts("host-linux-ppc64le-osu"); i++ {
-		name := fmt.Sprintf("go-le-%v", i)
+		name := fmt.Sprintf("power_%02d", i)
 		hosts = append(hosts, name)
 	}
 	return &healthChecker{
 		ID:     "osuppc64le",
-		Title:  "OSU linux/ppc64le machines",
+		Title:  "OSU linux/ppc64le POWER8 machines",
+		DocURL: "https://github.com/golang/build/tree/master/env/linux-ppc64le/osuosl",
+		Check:  reverseHostChecker(hosts),
+	}
+}
+
+func newOSUPPC64lePower9Checker() *healthChecker {
+	var hosts []string
+	for i := 1; i <= expectedHosts("host-linux-ppc64le-power9-osu"); i++ {
+		name := fmt.Sprintf("power_%02d", i)
+		hosts = append(hosts, name)
+	}
+	return &healthChecker{
+		ID:     "osuppc64lepower9",
+		Title:  "OSU linux/ppc64le POWER9 machines",
 		DocURL: "https://github.com/golang/build/tree/master/env/linux-ppc64le/osuosl",
 		Check:  reverseHostChecker(hosts),
 	}

cmd/retrybuilds/retrybuilds.go

@@ -134,6 +134,13 @@ var flakePhrases = []string{
 	"lookup www.mit.edu on ",
 	"undefined: runtime.SetMutexProfileFraction", // ppc64 builders had not-quite-go1.8 bootstrap
 	"make.bat: The parameter is incorrect",
+	"killed",
+	"memory",
+	"allocate",
+	"Killed",
+	"Error running API checker: exit status 1",
+	"/compile: exit status 1",
+	"cmd/link: exit status 1",
 }
 
 func isFlaky(failLog string) bool {

cmd/rundockerbuildlet/rundockerbuildlet.go

@@ -19,16 +19,19 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 )
 
 var (
-	image    = flag.String("image", "", "docker image to run; required.")
-	numInst  = flag.Int("n", 1, "number of containers to keep running at once")
-	basename = flag.String("basename", "builder", "prefix before the builder number to use for the container names and host names")
-	memory   = flag.String("memory", "3g", "memory limit flag for docker run")
-	keyFile  = flag.String("key", "/etc/gobuild.key", "go build key file")
+	image      = flag.String("image", "golang/builder", "docker image to run; required.")
+	numInst    = flag.Int("n", 1, "number of containers to keep running at once")
+	basename   = flag.String("basename", "builder", "prefix before the builder number to use for the container names and host names")
+	memory     = flag.String("memory", "3g", "memory limit flag for docker run")
+	keyFile    = flag.String("key", "/etc/gobuild.key", "go build key file")
+	builderEnv = flag.String("env", "", "optional GO_BUILDER_ENV environment variable value to set in the guests")
+	cpu        = flag.Int("cpu", 0, "if non-zero, how many CPUs to assign from the host and pass to docker run --cpuset-cpus")
 )
 
 var (
@@ -49,8 +52,14 @@ func main() {
 		log.Fatalf("docker --image is required")
 	}
 
-	if _, err := os.Stat("/usr/local/bin/oc-metadata"); err == nil {
-		initScalewayMeta()
+	// If we're on linux/arm, see if we're running in a scaleway
+	// image (which contains the oc-metadata command) and, if so,
+	// fetch our instance metadata to make the reported hostname match
+	// the instance hostname.
+	if runtime.GOOS == "linux" && runtime.GOARCH == "arm" {
+		if _, err := os.Stat("/usr/local/bin/oc-metadata"); err == nil {
+			initScalewayMeta()
+		}
 	}
 
 	log.Printf("Started. Will keep %d copies of %s running.", *numInst, *image)
@@ -139,14 +148,21 @@ func checkFix() error {
 		if err := ioutil.WriteFile(keyFile, buildKey, 0600); err != nil {
 			return err
 		}
-		out, err := exec.Command("docker", "run",
+		cmd := exec.Command("docker", "run",
 			"-d",
 			"--memory="+*memory,
 			"--name="+name,
 			"-v", filepath.Dir(keyFile)+":/buildkey/",
 			"-e", "HOSTNAME="+name,
-			"--tmpfs=/workdir:rw,exec",
-			*image).CombinedOutput()
+			"--tmpfs=/workdir:rw,exec")
+		if *builderEnv != "" {
+			cmd.Args = append(cmd.Args, "-e", "GO_BUILDER_ENV="+*builderEnv)
+		}
+		if *cpu > 0 {
+			cmd.Args = append(cmd.Args, fmt.Sprintf("--cpuset-cpus=%d-%d", *cpu*(num-1), *cpu*num-1))
+		}
+		cmd.Args = append(cmd.Args, *image)
+		out, err := cmd.CombinedOutput()
 		if err != nil {
 			log.Printf("Error creating %s: %v, %s", name, err, out)
 			continue

dashboard/builders.go

@@ -413,25 +413,22 @@ var Hosts = map[string]*HostConfig{
 		env:             []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
 		ReverseAliases:  []string{"linux-ppc64-buildlet"},
 		SSHUsername:     "root",
-		HermeticReverse: false, // TODO: use rundockerbuildlet like arm64
+		HermeticReverse: false, // TODO: run in chroots with overlayfs? https://github.com/golang/go/issues/34830#issuecomment-543386764
 	},
 	"host-linux-ppc64le-osu": &HostConfig{
-		Notes:           "Debian jessie; run by Go team on osuosl.org",
+		Notes:           "Debian Buster; run by Go team on osuosl.org; see x/build/env/linux-ppc64le/osuosl",
 		IsReverse:       true,
 		ExpectNum:       5,
 		env:             []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
-		ReverseAliases:  []string{"linux-ppc64le-buildlet"},
 		SSHUsername:     "root",
-		HermeticReverse: false, // TODO: use rundockerbuildlet like arm64
+		HermeticReverse: true,
 	},
 	"host-linux-ppc64le-power9-osu": &HostConfig{
-		Notes:           "Debian jessie; run by IBM",
-		OwnerGithub:     "ceseo",
+		Notes:           "Debian Buster; run by Go team on osuosl.org; see x/build/env/linux-ppc64le/osuosl",
 		IsReverse:       true,
 		env:             []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"},
-		ReverseAliases:  []string{"linux-ppc64le-power9osu"},
 		SSHUsername:     "root",
-		HermeticReverse: false, // TODO: use rundockerbuildlet like arm64
+		HermeticReverse: true,
 	},
 	"host-linux-arm64-packet": &HostConfig{
 		Notes:           "On 96 core packet.net host (Xenial) in Docker containers (Debian Buster); run by Go team. See x/build/env/linux-arm64/packet",

env/linux-ppc64/osuosl/NOTES

@@ -8,7 +8,7 @@ Filing tickets: https://support.osuosl.org/
 
 Machines:
 
-* go-be-xenial-3: (10 GB RAM, 50 GB disk, 10 cores, POWER8)
+* go-be-xenial-3: (20 GB RAM, 50 GB disk, 10 cores, POWER8)
 
   ssh -i ~/.ssh/id_ed25519_golang1 [email protected]
   (key from http://go/go-builders-ssh)

env/linux-ppc64le/osuosl/.gitignore

@@ -0,0 +1 @@
+rundockerbuildlet.ppc64le

env/linux-ppc64le/osuosl/Dockerfile

@@ -0,0 +1,26 @@
+# Copyright 2019 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM debian:buster
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && \
+    apt-get install --yes \
+          gcc curl strace \
+          ca-certificates netbase \
+          procps lsof psmisc \
+          openssh-server
+
+RUN mkdir /usr/local/go-bootstrap && \
+    curl --silent https://storage.googleapis.com/go-builder-data/gobootstrap-linux-ppc64le.tar.gz | \
+    tar -C /usr/local/go-bootstrap -zxv
+
+ENV GOROOT_BOOTSTRAP /usr/local/go-bootstrap
+ADD stage0 /usr/local/bin/stage0
+
+ENV GO_BUILD_KEY_DELETE_AFTER_READ true
+ENV GO_BUILD_KEY_PATH /buildkey/gobuildkey
+
+CMD ["/usr/local/bin/stage0"]

env/linux-ppc64le/osuosl/Makefile

@@ -0,0 +1,5 @@
+power8:
+	./setup-remote.sh [email protected] host-linux-ppc64le-osu
+
+power9:
+	./setup-remote.sh [email protected] host-linux-ppc64le-power9-osu

env/linux-ppc64le/osuosl/NOTES

@@ -1,38 +1,24 @@
 The linux-ppc64le buildlets run on PPC64 VMs at osuosl.org (OSU Open Source Lab).
 
-They run Debian jessie.
+They run Ubuntu bionic on the host, with Docker images running
+debian:buster running the buildlets.
 
 Filing tickets: https://support.osuosl.org/
     Ticket username: [email protected]
     Ticket password: http://go/pw-osuosl-ppc64
 
 Machines:
 
-* go-le-bionic-1: (10 GB RAM, 50 GB disk, 10 cores, POWER9)
+* go-le-bionic-1: (20 GB RAM, 50 GB disk, 10 cores, POWER9)
+  Linux go-le-bionic-1 4.15.0-65-generic #74-Ubuntu SMP Tue Sep 17 17:08:54 UTC 2019 ppc64le ppc64le ppc64le GNU/Linux
 
   ssh -i ~/.ssh/id_ed25519_golang1 [email protected]
   (key from http://go/go-builders-ssh)
 
-  Linux go-le-bionic-1 4.15.0-65-generic #74-Ubuntu SMP Tue Sep 17 17:08:54 UTC 2019 ppc64le ppc64le ppc64le GNU/Linux
-
-* go-le-bionic-2: (10 GB RAM, 50 GB disk, 10 cores, POWER8)
+* go-le-bionic-2: (20 GB RAM, 50 GB disk, 10 cores, POWER8)
+  Linux go-le-bionic-2 4.15.0-65-generic #74-Ubuntu SMP Tue Sep 17 17:08:54 UTC 2019 ppc64le ppc64le ppc64le GNU/Linux
 
   ssh -i ~/.ssh/id_ed25519_golang1 [email protected]
   (key from http://go/go-builders-ssh)
 
-  Linux go-le-bionic-2 4.15.0-65-generic #74-Ubuntu SMP Tue Sep 17 17:08:54 UTC 2019 ppc64le ppc64le ppc64le GNU/Linux
-
-* Old machines, to be returned: (currently Debian buster):
-
-  go-le-2: [email protected]
-  go-le-3: [email protected]
-  go-le-4: [email protected]
-  go-le-5: [email protected]
-
-Each was once configured by scping setup.bash to them, logging in to
-each, and running:
-
-$ sudo ./setup.bash <BUILDKEY_HERE>
-
-TODO: these should be updated like linux-arm and linux-arm64 to use Docker
-      per build. That is https://golang.org/issue/21189
+To configure the hosts, see the Makefile and trace what that does.

env/linux-ppc64le/osuosl/install-buildlet.bash

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+set -x
+
+HOST_TYPE=$1
+if [ "$HOST_TYPE" = "" ]; then
+        echo "Missing host type arg; this file is not supposed to be run directly; see Makefile for usage" >&2
+        exit 2
+fi
+
+sudo mv .gobuildkey /etc/gobuild.key
+sudo install rundockerbuildlet.ppc64le /usr/local/bin/rundockerbuildlet
+
+curl -o stage0 https://storage.googleapis.com/go-builder-data/buildlet-stage0.linux-ppc64le
+chmod +x stage0
+docker build -t golang/builder .
+
+sed "s/env=XXX/env=$HOST_TYPE/" rundockerbuildlet.service > rundockerbuildlet.service.expanded
+sudo cp rundockerbuildlet.service.expanded /etc/systemd/user/rundockerbuildlet.service
+sudo systemctl enable /etc/systemd/user/rundockerbuildlet.service || true
+sudo systemctl daemon-reload || true
+sudo systemctl restart docker.service
+sudo systemctl restart rundockerbuildlet.service

env/linux-ppc64le/osuosl/install-docker.bash

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Just the docker setup, ending at adding our regular user to the docker group.
+# The setup.bash script does everything else.
+
+set -e
+sudo apt-get update
+sudo apt-get --yes upgrade
+sudo apt-get --yes install docker.io
+sudo usermod -aG docker $USER

env/linux-ppc64le/osuosl/rundockerbuildlet.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Run Buildlets in Docker
+After=network.target
+
+[Install]
+WantedBy=network-online.target
+
+[Service]
+Type=simple
+# The (-n * -cpu) values must currently be <= number of host cores.
+# The host has 10 cores, so the -n=5 (five containers) * -cpu=2 (two CPUs per container) == 10.
+ExecStart=/usr/local/bin/rundockerbuildlet -basename=power_ -image=golang/builder -n=5 -cpu=2 -memory=3.9g --env=XXX
+Restart=always
+RestartSec=2
+StartLimitInterval=0

env/linux-ppc64le/osuosl/setup-remote.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# See Makefile for usage
+
+set -e
+
+USER_AT_HOST=$1
+if [ "$USER_AT_HOST" = "" ]; then
+        echo "Missing user@host arg; see Makefile for usage" >&2
+        exit 2
+fi
+HOST_TYPE=$2
+if [ "$HOST_TYPE" = "" ]; then
+        echo "Missing host type arg; see Makefile for usage" >&2
+        exit 2
+fi
+
+GOARCH=ppc64le GOOS=linux go build -o rundockerbuildlet.ppc64le golang.org/x/build/cmd/rundockerbuildlet
+
+rsync -e "ssh -i ~/.ssh/id_ed25519_golang1" -avPW ./ $USER_AT_HOST:./
+scp -i ~/.ssh/id_ed25519_golang1 $HOME/keys/${HOST_TYPE}.buildkey $USER_AT_HOST:.gobuildkey
+
+# Install Docker, including adding our username to the "docker" group:
+ssh -i ~/.ssh/id_ed25519_golang1 $USER_AT_HOST ./install-docker.bash
+
+# Now that we have Docker, "log in" again (with access to the docker
+# group) and do the rest:
+ssh -i ~/.ssh/id_ed25519_golang1 $USER_AT_HOST ./install-buildlet.bash $HOST_TYPE
+
+