Fix gopackagesdriver for Go 1.18 by replicating stdlib and add test for stdliblist.go

.bazelci/presubmit.yml

@@ -264,6 +264,7 @@ tasks:
     - "-@org_golang_x_text//language:language_test"
     - "-@org_golang_x_tools//cmd/splitdwarf/internal/macho:macho_test"
     - "-@test_chdir_remote//sub:go_default_test"
+    - "-//go/tools/builders:stdliblist_test" # fails on Windows due to #2491
     - "-//tests:buildifier_test" # requires bash
     - "-//tests/core/cgo:dylib_client"
     - "-//tests/core/cgo:dylib_test"

go/private/actions/stdlib.bzl

@@ -54,6 +54,7 @@ def _should_use_sdk_stdlib(go):
 def _build_stdlib_list_json(go):
     out = go.declare_file(go, "stdlib.pkg.json")
     args = go.builder_args(go, "stdliblist")
+    args.add("-sdk", go.sdk.root_file.dirname)
     args.add("-out", out)
     go.actions.run(
         inputs = go.sdk_files,

go/tools/builders/BUILD.bazel

@@ -23,6 +23,20 @@ go_test(
     ],
 )
 
+go_test(
+    name = "stdliblist_test",
+    size = "small",
+    srcs = [
+        "env.go",
+        "flags.go",
+        "replicate.go",
+        "stdliblist.go",
+        "stdliblist_test.go",
+    ],
+    data = ["@go_sdk//:files"],
+    rundir = ".",
+)
+
 filegroup(
     name = "builder_srcs",
     srcs = [

go/tools/builders/env.go

@@ -66,7 +66,7 @@ type env struct {
 // configured with those flags.
 func envFlags(flags *flag.FlagSet) *env {
 	env := &env{}
-	flags.StringVar(&env.sdk, "sdk", "", "Path to the Go SDK.")
+	flags.StringVar(&env.sdk, "sdk", "", "Relative path to the Go SDK.")
 	flags.Var(&tagFlag{}, "tags", "List of build tags considered true.")
 	flags.StringVar(&env.installSuffix, "installsuffix", "", "Standard library under GOROOT/pkg")
 	flags.BoolVar(&env.verbose, "v", false, "Whether subprocess command lines should be printed")
@@ -121,6 +121,7 @@ func (e *env) goTool(tool string, args ...string) []string {
 // and additional arguments.
 func (e *env) goCmd(cmd string, args ...string) []string {
 	exe := filepath.Join(e.sdk, "bin", "go")
+
 	if runtime.GOOS == "windows" {
 		exe += ".exe"
 	}

go/tools/builders/stdliblist.go

@@ -18,7 +18,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"flag"
+	"fmt"
 	"go/build"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -109,29 +111,32 @@ func stdlibPackageID(importPath string) string {
 	return "@io_bazel_rules_go//stdlib:" + importPath
 }
 
-func execRootPath(execRoot, p string) string {
-	dir, _ := filepath.Rel(execRoot, p)
+// labelledPath replace the cloneBase with output base label
+func labelledPath(cloneBase, p string) string {
+	dir, _ := filepath.Rel(cloneBase, p)
 	return filepath.Join("__BAZEL_OUTPUT_BASE__", dir)
 }
 
-func absoluteSourcesPaths(execRoot, pkgDir string, srcs []string) []string {
+//  absoluteSourcesPaths replace cloneBase of the absolution
+//  paths with the label for all source files in a package
+func absoluteSourcesPaths(cloneBase, pkgDir string, srcs []string) []string {
 	ret := make([]string, 0, len(srcs))
-	pkgDir = execRootPath(execRoot, pkgDir)
+	pkgDir = labelledPath(cloneBase, pkgDir)
 	for _, src := range srcs {
 		ret = append(ret, filepath.Join(pkgDir, src))
 	}
 	return ret
 }
 
-func flatPackageForStd(execRoot string, pkg *goListPackage) *flatPackage {
+func flatPackageForStd(cloneBase string, pkg *goListPackage) *flatPackage {
 	// Don't use generated files from the stdlib
-	goFiles := absoluteSourcesPaths(execRoot, pkg.Dir, pkg.GoFiles)
+	goFiles := absoluteSourcesPaths(cloneBase, pkg.Dir, pkg.GoFiles)
 
 	newPkg := &flatPackage{
 		ID:              stdlibPackageID(pkg.ImportPath),
 		Name:            pkg.Name,
 		PkgPath:         pkg.ImportPath,
-		ExportFile:      execRootPath(execRoot, pkg.Target),
+		ExportFile:      labelledPath(cloneBase, pkg.Target),
 		Imports:         map[string]string{},
 		Standard:        pkg.Standard,
 		GoFiles:         goFiles,
@@ -145,6 +150,34 @@ func flatPackageForStd(execRoot string, pkg *goListPackage) *flatPackage {
 	return newPkg
 }
 
+// In Go 1.18, the standard library started using go:embed directives.
+// When Bazel runs this action, it does so inside a sandbox where GOROOT points
+// to an external/go_sdk directory that contains a symlink farm of all files in
+// the Go SDK.
+// If we run "go list" with that GOROOT, this action will fail because those
+// go:embed directives will refuse to include the symlinks in the sandbox.
+//
+// To work around this, cloneGoRoot creates a copy of a subset of external/go_sdk
+// that is sufficient to call "go list" into a new cloneBase directory, e.g.
+// "go list" needs to call "compile", which needs "pkg/tool".
+// We also need to retain the same relative path to the root directory, e.g.
+// "$OUTPUT_BASE/external/go_sdk" becomes
+// {cloneBase}/external/go_sdk", which will be set at GOROOT later. This ensures
+// that file paths in the generated JSON are still valid.
+//
+// cloneGoRoot replicate goRoot to newGoRoot and returns an error if any.
+func cloneGoRoot(goRoot, newGoRoot string) error {
+	if err := os.MkdirAll(newGoRoot, 01755); err != nil {
+		return err
+	}
+
+	if err := replicate(goRoot, newGoRoot, replicatePaths("src", "pkg/tool", "pkg/include")); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // stdliblist runs `go list -json` on the standard library and saves it to a file.
 func stdliblist(args []string) error {
 	// process the args
@@ -158,19 +191,32 @@ func stdliblist(args []string) error {
 		return err
 	}
 
+	cloneBase, err := ioutil.TempDir(goenv.workDirPath, "stdlist-*")
+	if err != nil {
+		return err
+	}
+
+	relGoSdk, err := filepath.Rel(".", goenv.sdk)
+	if err != nil {
+		return err
+	}
+
+	newGoRoot := filepath.Join(cloneBase, relGoSdk)
+	err = cloneGoRoot(abs(goenv.sdk), abs(newGoRoot))
+	if err != nil {
+		return fmt.Errorf("failed to clone new go root %v", err)
+	}
 	// Ensure paths are absolute.
 	absPaths := []string{}
 	for _, path := range filepath.SplitList(os.Getenv("PATH")) {
 		absPaths = append(absPaths, abs(path))
 	}
 	os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator)))
-	os.Setenv("GOROOT", abs(os.Getenv("GOROOT")))
+	os.Setenv("GOROOT", newGoRoot)
 	// Make sure we have an absolute path to the C compiler.
 	// TODO(#1357): also take absolute paths of includes and other paths in flags.
 	os.Setenv("CC", abs(os.Getenv("CC")))
 
-	execRoot := abs(".")
-
 	cachePath := abs(*out + ".gocache")
 	defer os.RemoveAll(cachePath)
 	os.Setenv("GOCACHE", cachePath)
@@ -193,15 +239,14 @@ func stdliblist(args []string) error {
 	if err := goenv.runCommandToFile(jsonData, listArgs); err != nil {
 		return err
 	}
-
 	encoder := json.NewEncoder(jsonFile)
 	decoder := json.NewDecoder(jsonData)
 	for decoder.More() {
 		var pkg *goListPackage
 		if err := decoder.Decode(&pkg); err != nil {
 			return err
 		}
-		if err := encoder.Encode(flatPackageForStd(execRoot, pkg)); err != nil {
+		if err := encoder.Encode(flatPackageForStd(cloneBase, pkg)); err != nil {
 			return err
 		}
 	}

go/tools/builders/stdliblist_test.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func Test_stdliblist(t *testing.T) {
+	testDir := t.TempDir()
+	outJSON := filepath.Join(testDir, "out.json")
+
+	test_args := []string{
+		fmt.Sprintf("-out=%s", outJSON),
+		fmt.Sprintf("-sdk=%s", "external/go_sdk"),
+	}
+
+	if err := stdliblist(test_args); err != nil {
+		t.Errorf("calling stdliblist got err: %v", err)
+	}
+	f, err := os.Open(outJSON)
+	if err != nil {
+		t.Errorf("cannot open output json: %v", err)
+	}
+	defer func() { _ = f.Close() }()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		var result flatPackage
+		jsonLineStr := scanner.Text()
+		if err := json.Unmarshal([]byte(jsonLineStr), &result); err != nil {
+			t.Errorf("cannot parse result line %s \n to goListPackage{}: %v\n", err)
+		}
+		if !strings.HasPrefix(result.ID, "@io_bazel_rules_go//stdlib") {
+			t.Errorf("ID should be prefixed with @io_bazel_rules_go//stdlib :%s", jsonLineStr)
+		}
+		if !strings.HasPrefix(result.ExportFile, "__BAZEL_OUTPUT_BASE__") {
+			t.Errorf("export file should be prefixed with __BAZEL_OUTPUT_BASE__ :%s", jsonLineStr)
+		}
+		for _, gofile := range result.GoFiles {
+			if !strings.HasPrefix(gofile, "__BAZEL_OUTPUT_BASE__/external/go_sdk") {
+				t.Errorf("All go files should be prefixed with __BAZEL_OUTPUT_BASE__/go_sdk :%s", jsonLineStr)
+			}
+		}
+	}
+}

tests/core/stdlib/stdlib_files.bzl

@@ -26,10 +26,11 @@ pure_transition = transition(
 def _stdlib_files_impl(ctx):
     # When a transition is used, ctx.attr._stdlib is a list of Target instead
     # of a Target. Possibly a bug?
-    libs = ctx.attr._stdlib[0][GoStdLib].libs
+    stdlib = ctx.attr._stdlib[0][GoStdLib]
+    libs = stdlib.libs
     runfiles = ctx.runfiles(files = libs)
     return [DefaultInfo(
-        files = depset(libs),
+        files = depset(libs + [stdlib._list_json]),
         runfiles = runfiles,
     )]