all: move internal/x to vendor/golang.org/x and revendor using 'go mo…

…d vendor' This also updates the vendored-in versions of several packages: 'go mod vendor' selects a consistent version of each module, but we had previously vendored an ad-hoc selection of packages. Notably, x/crypto/hkdf was previously vendored in at a much newer commit than the rest of x/crypto. Bringing the rest of x/crypto up to that commit introduced an import of golang.org/x/sys/cpu, which broke the js/wasm build, requiring an upgrade of x/sys to pick up CL 165749. Updates #30228 Updates #30241 Updates #25822 Change-Id: I5b3dbc232b7e6a048a158cbd8d36137af1efb711 Reviewed-on: https://go-review.googlesource.com/c/go/+/164623 Reviewed-by: Filippo Valsorda <[email protected]>
golang · Mar 11, 2019 · c5cf662 · c5cf662
1 parent 0fc89a7
commit c5cf662
Show file tree

Hide file tree

Showing 240 changed files with 3,250 additions and 11,361 deletions.
diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go
@@ -192,6 +192,21 @@ func xinit() {
 
 	gogcflags = os.Getenv("BOOT_GO_GCFLAGS")
 
+	// Add -mod=vendor to GOFLAGS so that commands won't try to resolve go.mod
+	// files for vendored external modules.
+	// TODO(golang.org/issue/30240): If the vendor directory contains the go.mod
+	// files, this probably won't be necessary.
+	// TODO(golang.org/issue/26849): Escape spaces in GOFLAGS if needed.
+	goflags := strings.Fields(os.Getenv("GOFLAGS"))
+	for i, flag := range goflags {
+		if strings.HasPrefix(flag, "-mod=") {
+			goflags = append(goflags[0:i], goflags[i+1:]...)
+			break
+		}
+	}
+	goflags = append(goflags, "-mod=vendor")
+	os.Setenv("GOFLAGS", strings.Join(goflags, " "))
+
 	cc, cxx := "gcc", "g++"
 	if defaultclang {
 		cc, cxx = "clang", "clang++"

diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go
@@ -419,9 +419,10 @@ func (t *tester) registerTests() {
 		if !t.race {
 			cmd.Args = append(cmd.Args, "cmd")
 		}
+		cmd.Stderr = new(bytes.Buffer)
 		all, err := cmd.Output()
 		if err != nil {
-			log.Fatalf("Error running go list std cmd: %v, %s", err, all)
+			log.Fatalf("Error running go list std cmd: %v:\n%s", err, cmd.Stderr)
 		}
 		pkgs := strings.Fields(string(all))
 		for _, pkg := range pkgs {

diff --git a/src/cmd/go/testdata/script/gopath_std_vendor.txt b/src/cmd/go/testdata/script/gopath_std_vendor.txt
@@ -9,8 +9,8 @@ stdout $GOPATH[/\\]src[/\\]vendor
 # to the package 'vendor/golang.org/x/net/http2/hpack' within GOROOT.
 cd importnethttp
 go list -deps -f '{{.ImportPath}} {{.Dir}}'
-stdout ^internal/x/net/http2/hpack
-stdout $GOROOT[/\\]src[/\\]internal[/\\]x[/\\]net[/\\]http2[/\\]hpack
+stdout ^vendor/golang.org/x/net/http2/hpack
+stdout $GOROOT[/\\]src[/\\]vendor[/\\]golang.org[/\\]x[/\\]net[/\\]http2[/\\]hpack
 ! stdout $GOPATH[/\\]src[/\\]vendor
 
 # In the presence of $GOPATH/src/vendor/golang.org/x/net/http2/hpack,

diff --git a/src/cmd/go/testdata/script/list_std.txt b/src/cmd/go/testdata/script/list_std.txt
@@ -16,7 +16,7 @@ stdout cmd/compile
 
 # In GOPATH mode, packages vendored into GOROOT should be reported as standard.
 go list -f '{{if .Standard}}{{.ImportPath}}{{end}}' std cmd
-stdout internal/x/net/http2/hpack
+stdout golang.org/x/net/http2/hpack
 stdout cmd/vendor/golang\.org/x/arch/x86/x86asm
 
 # However, vendored packages should not match wildcard patterns beginning with cmd.

diff --git a/src/cmd/go/testdata/script/mod_list_std.txt b/src/cmd/go/testdata/script/mod_list_std.txt
@@ -5,7 +5,7 @@ env GOPROXY=off
 
 # Outside of GOROOT, our vendored packages should be reported as part of the standard library.
 go list -f '{{if .Standard}}{{.ImportPath}}{{end}}' std cmd
-stdout ^internal/x/net/http2/hpack
+stdout ^vendor/golang.org/x/net/http2/hpack
 stdout ^cmd/vendor/golang\.org/x/arch/x86/x86asm
 
 # cmd/... should match the same packages it used to match in GOPATH mode.
@@ -20,15 +20,15 @@ stdout ^cmd/compile
 # Today, they are listed in 'std' but not './...'.
 cd $GOROOT/src
 go list ./...
-stdout ^internal/x
+! stdout ^vendor/golang.org/x  # TODO: should be included, or should be omitted from 'std'.
 
 cp stdout $WORK/listdot.txt
 go list std
-stdout ^internal/x
+stdout ^vendor/golang.org/x  # TODO: remove vendor/ prefix
 # TODO: cmp stdout $WORK/listdot.txt
 
 go list all
-stdout ^internal/x
+stdout ^vendor/golang.org/x  # TODO: remove vendor/ prefix.
 ! stdout ^std/
 
 
@@ -37,11 +37,11 @@ stdout ^internal/x
 # TODO(golang.org/issue/30241): Make that work.
 # Today, they still have the vendor/ prefix.
 go list std
-stdout ^internal/x/net/http2/hpack  # TODO
+stdout ^vendor/golang.org/x/net/http2/hpack  # TODO
 ! stdout ^golang.org/x/net/http2/hpack       # TODO
 
 go list -deps -f '{{if not .Standard}}{{.ImportPath}}{{end}}' std
-# ! stdout ^internal/x/net/http2/hpack  # TODO
+# ! stdout ^vendor/golang.org/x/net/http2/hpack  # TODO
 ! stdout ^golang.org/x/net/http2/hpack         # TODO
 
 

diff --git a/src/cmd/go/testdata/script/mod_std_vendor.txt b/src/cmd/go/testdata/script/mod_std_vendor.txt
@@ -1,17 +1,20 @@
 env GO111MODULE=on
 env GOPROXY=off
 
+[!gc] skip
+
+# 'go list' should report imports from _test.go in the TestImports field.
 go list -f '{{.TestImports}}'
 stdout net/http # from .TestImports
 
 # 'go list' should find standard-vendored packages.
-go list -f '{{.Dir}}' internal/x/net/http2/hpack
-stdout $GOROOT[/\\]src[/\\]internal
+go list -f '{{.Dir}}' vendor/golang.org/x/net/http2/hpack
+stdout $GOROOT[/\\]src[/\\]vendor
 
 # 'go list -test' should report vendored transitive dependencies of _test.go
 # imports in the Deps field.
 go list -test -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}'
-stdout internal/x/crypto # dep of .TestImports
+stdout ^vendor/golang.org/x/crypto # dep of .TestImports
 
 
 # Modules outside the standard library should not use the packages vendored there...
@@ -29,7 +32,7 @@ stderr 'use of vendored package'
 
 cd ../importstd
 ! go build .
-stderr 'use of internal package'
+stderr 'use of vendored package'
 
 
 # When run within the 'std' module, 'go list -test' should report vendored
@@ -38,8 +41,8 @@ stderr 'use of internal package'
 # Today, they're standard packages as long as they exist.
 cd $GOROOT/src
 go list -test -f '{{range .Deps}}{{.}}{{"\n"}}{{end}}' net/http
-! stdout ^vendor/golang.org/x/net/http2/hpack  # TODO: this will exist later
-stdout ^internal/x/net/http2/hpack
+stdout ^vendor/golang.org/x/net/http2/hpack  # TODO: remove vendor/ prefix
+! stdout ^golang.org/x/net/http2/hpack
 
 -- go.mod --
 module m
@@ -74,4 +77,4 @@ module importvendor
 -- importstd/x.go --
 package importstd
 
-import _ "internal/x/net/http2/hpack"
+import _ "vendor/golang.org/x/net/http2/hpack"
diff --git a/src/cmd/go/testdata/script/std_vendor.txt b/src/cmd/go/testdata/script/std_vendor.txt
@@ -7,13 +7,13 @@ go list -f '{{.TestImports}}'
 stdout net/http # from .TestImports
 
 # 'go list' should report standard-vendored packages by path.
-go list -f '{{.Dir}}' internal/x/net/http2/hpack
-stdout $GOROOT[/\\]src[/\\]internal
+go list -f '{{.Dir}}' vendor/golang.org/x/net/http2/hpack
+stdout $GOROOT[/\\]src[/\\]vendor
 
 # 'go list -test' should report vendored transitive dependencies of _test.go
 # imports in the Deps field, with a 'vendor' prefix on their import paths.
 go list -test -f '{{.Deps}}'
-stdout internal/x/crypto # dep of .TestImports
+stdout golang.org/x/crypto # dep of .TestImports
 
 # Packages outside the standard library should not use its copy of vendored packages.
 cd broken

diff --git a/src/cmd/vet/all/whitelist/s390x.txt b/src/cmd/vet/all/whitelist/s390x.txt
@@ -10,3 +10,10 @@ internal/cpu/cpu_s390x.s: [s390x] kmctrQuery: invalid MOVD of ret+0(FP); interna
 internal/cpu/cpu_s390x.s: [s390x] kmaQuery: invalid MOVD of ret+0(FP); internal/cpu.queryResult is 16-byte value
 internal/cpu/cpu_s390x.s: [s390x] kimdQuery: invalid MOVD of ret+0(FP); internal/cpu.queryResult is 16-byte value
 internal/cpu/cpu_s390x.s: [s390x] klmdQuery: invalid MOVD of ret+0(FP); internal/cpu.queryResult is 16-byte value
+vendor/golang.org/x/sys/cpu/cpu_s390x.s: [s390x] stfle: invalid MOVD of ret+0(FP); vendor/golang.org/x/sys/cpu.facilityList is 32-byte value
+vendor/golang.org/x/sys/cpu/cpu_s390x.s: [s390x] kmQuery: invalid MOVD of ret+0(FP); vendor/golang.org/x/sys/cpu.queryResult is 16-byte value
+vendor/golang.org/x/sys/cpu/cpu_s390x.s: [s390x] kmcQuery: invalid MOVD of ret+0(FP); vendor/golang.org/x/sys/cpu.queryResult is 16-byte value
+vendor/golang.org/x/sys/cpu/cpu_s390x.s: [s390x] kmctrQuery: invalid MOVD of ret+0(FP); vendor/golang.org/x/sys/cpu.queryResult is 16-byte value
+vendor/golang.org/x/sys/cpu/cpu_s390x.s: [s390x] kmaQuery: invalid MOVD of ret+0(FP); vendor/golang.org/x/sys/cpu.queryResult is 16-byte value
+vendor/golang.org/x/sys/cpu/cpu_s390x.s: [s390x] kimdQuery: invalid MOVD of ret+0(FP); vendor/golang.org/x/sys/cpu.queryResult is 16-byte value
+vendor/golang.org/x/sys/cpu/cpu_s390x.s: [s390x] klmdQuery: invalid MOVD of ret+0(FP); vendor/golang.org/x/sys/cpu.queryResult is 16-byte value
diff --git a/src/crypto/tls/cipher_suites.go b/src/crypto/tls/cipher_suites.go
@@ -14,8 +14,8 @@ import (
 	"crypto/sha1"
 	"crypto/sha256"
 	"crypto/x509"
+	"golang.org/x/crypto/chacha20poly1305"
 	"hash"
-	"internal/x/crypto/chacha20poly1305"
 )
 
 // a keyAgreement implements the client and server side of a TLS key agreement

diff --git a/src/crypto/tls/handshake_messages.go b/src/crypto/tls/handshake_messages.go
@@ -6,7 +6,7 @@ package tls
 
 import (
 	"fmt"
-	"internal/x/crypto/cryptobyte"
+	"golang.org/x/crypto/cryptobyte"
 	"strings"
 )
 

diff --git a/src/crypto/tls/key_schedule.go b/src/crypto/tls/key_schedule.go
@@ -8,10 +8,10 @@ import (
 	"crypto/elliptic"
 	"crypto/hmac"
 	"errors"
+	"golang.org/x/crypto/cryptobyte"
+	"golang.org/x/crypto/curve25519"
+	"golang.org/x/crypto/hkdf"
 	"hash"
-	"internal/x/crypto/cryptobyte"
-	"internal/x/crypto/curve25519"
-	"internal/x/crypto/hkdf"
 	"io"
 	"math/big"
 )

diff --git a/src/crypto/tls/ticket.go b/src/crypto/tls/ticket.go
@@ -12,7 +12,7 @@ import (
 	"crypto/sha256"
 	"crypto/subtle"
 	"errors"
-	"internal/x/crypto/cryptobyte"
+	"golang.org/x/crypto/cryptobyte"
 	"io"
 )
 

diff --git a/src/crypto/x509/x509.go b/src/crypto/x509/x509.go
@@ -24,8 +24,8 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"internal/x/crypto/cryptobyte"
-	cryptobyte_asn1 "internal/x/crypto/cryptobyte/asn1"
+	"golang.org/x/crypto/cryptobyte"
+	cryptobyte_asn1 "golang.org/x/crypto/cryptobyte/asn1"
 	"io"
 	"math/big"
 	"net"

diff --git a/src/go.mod b/src/go.mod
@@ -1,3 +1,10 @@
 module std
 
 go 1.12
+
+require (
+	golang.org/x/crypto v0.0.0-20181025213731-e84da0312774
+	golang.org/x/net v0.0.0-20181213202711-891ebc4b82d6
+	golang.org/x/sys v0.0.0-20190306220234-b354f8bf4d9e // indirect
+	golang.org/x/text v0.1.1-0.20171102144821-8253218a5ec6 // indirect
+)
diff --git a/src/go.sum b/src/go.sum
@@ -0,0 +1,8 @@
+golang.org/x/crypto v0.0.0-20181025213731-e84da0312774 h1:a4tQYYYuK9QdeO/+kEvNYyuR21S+7ve5EANok6hABhI=
+golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/net v0.0.0-20181213202711-891ebc4b82d6 h1:gT0Y6H7hbVPUtvtk0YGxMXPgN+p8fYlqWkgJeUCZcaQ=
+golang.org/x/net v0.0.0-20181213202711-891ebc4b82d6/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sys v0.0.0-20190306220234-b354f8bf4d9e h1:UndnRDGP/JcdZX1LBubo1fJ3Jt6GnKREteLJvysiiPE=
+golang.org/x/sys v0.0.0-20190306220234-b354f8bf4d9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.1.1-0.20171102144821-8253218a5ec6 h1:j8pkdn+8tJbBXIFRILFAB5MDo/hAZg4TnknVwnhU6bI=
+golang.org/x/text v0.1.1-0.20171102144821-8253218a5ec6/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go
@@ -325,7 +325,7 @@ var pkgDeps = map[string][]string{
 		"context", "math/rand", "os", "reflect", "sort", "syscall", "time",
 		"internal/nettrace", "internal/poll", "internal/syscall/unix",
 		"internal/syscall/windows", "internal/singleflight", "internal/race",
-		"internal/x/net/dns/dnsmessage", "internal/x/net/lif", "internal/x/net/route",
+		"golang.org/x/net/dns/dnsmessage", "golang.org/x/net/lif", "golang.org/x/net/route",
 	},
 
 	// NET enables use of basic network-related packages.
@@ -362,9 +362,9 @@ var pkgDeps = map[string][]string{
 		"crypto/sha1",
 		"crypto/sha256",
 		"crypto/sha512",
-		"internal/x/crypto/chacha20poly1305",
-		"internal/x/crypto/curve25519",
-		"internal/x/crypto/poly1305",
+		"golang.org/x/crypto/chacha20poly1305",
+		"golang.org/x/crypto/curve25519",
+		"golang.org/x/crypto/poly1305",
 	},
 
 	// Random byte, number generation.
@@ -392,13 +392,13 @@ var pkgDeps = map[string][]string{
 
 	// SSL/TLS.
 	"crypto/tls": {
-		"L4", "CRYPTO-MATH", "OS", "internal/x/crypto/cryptobyte", "internal/x/crypto/hkdf",
+		"L4", "CRYPTO-MATH", "OS", "golang.org/x/crypto/cryptobyte", "golang.org/x/crypto/hkdf",
 		"container/list", "crypto/x509", "encoding/pem", "net", "syscall",
 	},
 	"crypto/x509": {
 		"L4", "CRYPTO-MATH", "OS", "CGO",
 		"crypto/x509/pkix", "encoding/pem", "encoding/hex", "net", "os/user", "syscall", "net/url",
-		"internal/x/crypto/cryptobyte", "internal/x/crypto/cryptobyte/asn1",
+		"golang.org/x/crypto/cryptobyte", "golang.org/x/crypto/cryptobyte/asn1",
 	},
 	"crypto/x509/pkix": {"L4", "CRYPTO-MATH", "encoding/hex"},
 
@@ -414,12 +414,12 @@ var pkgDeps = map[string][]string{
 		"context",
 		"crypto/rand",
 		"crypto/tls",
-		"internal/x/net/http/httpguts",
-		"internal/x/net/http/httpproxy",
-		"internal/x/net/http2/hpack",
-		"internal/x/net/idna",
-		"internal/x/text/unicode/norm",
-		"internal/x/text/width",
+		"golang.org/x/net/http/httpguts",
+		"golang.org/x/net/http/httpproxy",
+		"golang.org/x/net/http2/hpack",
+		"golang.org/x/net/idna",
+		"golang.org/x/text/unicode/norm",
+		"golang.org/x/text/width",
 		"internal/nettrace",
 		"mime/multipart",
 		"net/http/httptrace",
@@ -437,9 +437,9 @@ var pkgDeps = map[string][]string{
 	"net/http/fcgi":      {"L4", "NET", "OS", "context", "net/http", "net/http/cgi"},
 	"net/http/httptest": {
 		"L4", "NET", "OS", "crypto/tls", "flag", "net/http", "net/http/internal", "crypto/x509",
-		"internal/x/net/http/httpguts",
+		"golang.org/x/net/http/httpguts",
 	},
-	"net/http/httputil": {"L4", "NET", "OS", "context", "net/http", "net/http/internal", "internal/x/net/http/httpguts"},
+	"net/http/httputil": {"L4", "NET", "OS", "context", "net/http", "net/http/internal", "golang.org/x/net/http/httpguts"},
 	"net/http/pprof":    {"L4", "OS", "html/template", "net/http", "runtime/pprof", "runtime/trace"},
 	"net/rpc":           {"L4", "NET", "encoding/gob", "html/template", "net/http"},
 	"net/rpc/jsonrpc":   {"L4", "NET", "encoding/json", "net/rpc"},
@@ -490,7 +490,7 @@ func listStdPkgs(goroot string) ([]string, error) {
 		}
 
 		name := filepath.ToSlash(path[len(src):])
-		if name == "builtin" || name == "cmd" || strings.Contains(name, "internal/x/") {
+		if name == "builtin" || name == "cmd" || strings.Contains(name, "golang.org/x/") {
 			return filepath.SkipDir
 		}
 

diff --git a/src/go/internal/srcimporter/srcimporter_test.go b/src/go/internal/srcimporter/srcimporter_test.go
@@ -99,7 +99,7 @@ var importedObjectTests = []struct {
 	{"math.Pi", "const Pi untyped float"},
 	{"math.Sin", "func Sin(x float64) float64"},
 	{"math/big.Int", "type Int struct{neg bool; abs nat}"},
-	{"internal/x/text/unicode/norm.MaxSegmentSize", "const MaxSegmentSize untyped int"},
+	{"golang.org/x/text/unicode/norm.MaxSegmentSize", "const MaxSegmentSize untyped int"},
 }
 
 func TestImportedTypes(t *testing.T) {
@@ -108,12 +108,12 @@ func TestImportedTypes(t *testing.T) {
 	}
 
 	for _, test := range importedObjectTests {
-		s := strings.Split(test.name, ".")
-		if len(s) != 2 {
+		i := strings.LastIndex(test.name, ".")
+		if i < 0 {
 			t.Fatal("invalid test data format")
 		}
-		importPath := s[0]
-		objName := s[1]
+		importPath := test.name[:i]
+		objName := test.name[i+1:]
 
 		pkg, err := importer.ImportFrom(importPath, ".", 0)
 		if err != nil {