cmd/go: add preliminary support for vendor directories

When GO15VENDOREXPERIMENT=1 is in the environment,
this CL changes the resolution of import paths according to
the Go 1.5 vendor proposal:

	If there is a source directory d/vendor, then,
	when compiling a source file within the subtree rooted at d,
	import "p" is interpreted as import "d/vendor/p" if that exists.

	When there are multiple possible resolutions,
	the most specific (longest) path wins.

	The short form must always be used: no import path can
	contain “/vendor/” explicitly.

	Import comments are ignored in vendored packages.

The goal of these changes is to allow authors to vendor (copy) external
packages into their source trees without any modifications to the code.
This functionality has been achieved in tools like godep, nut, and gb by
requiring GOPATH manipulation. This alternate directory-based approach
eliminates the need for GOPATH manipulation and in keeping with the
go command's use of directory layout-based configuration.

The flag allows experimentation with these vendoring semantics once
Go 1.5 is released, without forcing them on by default. If the experiment
is deemed a success, the flag will default to true in Go 1.6 and then be
removed in Go 1.7.

For more details, see the original proposal by Keith Rarick at
https://groups.google.com/d/msg/golang-dev/74zjMON9glU/dGhnoi2IMzsJ.

Change-Id: I2c6527e777d14ac6dc43c53e4b3ff24f3279216e
Reviewed-on: https://go-review.googlesource.com/10923
Reviewed-by: Andrew Gerrand <[email protected]>

src/cmd/go/build.go

@@ -2135,6 +2135,14 @@ func (gcToolchain) gc(b *builder, p *Package, archive, obj string, asmhdr bool,
 		gcargs = append(gcargs, "-buildid", p.buildID)
 	}
 
+	for _, path := range p.Imports {
+		if i := strings.LastIndex(path, "/vendor/"); i >= 0 {
+			gcargs = append(gcargs, "-importmap", path[i+len("/vendor/"):]+"="+path)
+		} else if strings.HasPrefix(path, "vendor/") {
+			gcargs = append(gcargs, "-importmap", path[len("vendor/"):]+"="+path)
+		}
+	}
+
 	args := []interface{}{buildToolExec, tool("compile"), "-o", ofile, "-trimpath", b.work, buildGcflags, gcargs, "-D", p.localPrefix, importArgs}
 	if ofile == archive {
 		args = append(args, "-pack")

src/cmd/go/pkg.go

@@ -215,6 +215,12 @@ func reloadPackage(arg string, stk *importStack) *Package {
 	return loadPackage(arg, stk)
 }
 
+// The Go 1.5 vendoring experiment is enabled by setting GO15VENDOREXPERIMENT=1.
+// The variable is obnoxiously long so that years from now when people find it in
+// their profiles and wonder what it does, there is some chance that a web search
+// might answer the question.
+var go15VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") == "1"
+
 // dirToImportPath returns the pseudo-import path we use for a package
 // outside the Go path.  It begins with _/ and then contains the full path
 // to the directory.  If the package lives in c:\home\gopher\my\pkg then
@@ -239,23 +245,33 @@ func makeImportValid(r rune) rune {
 // but possibly a local import path (an absolute file system path or one beginning
 // with ./ or ../).  A local relative path is interpreted relative to srcDir.
 // It returns a *Package describing the package found in that directory.
-func loadImport(path string, srcDir string, stk *importStack, importPos []token.Position) *Package {
+func loadImport(path, srcDir string, parent *Package, stk *importStack, importPos []token.Position) *Package {
 	stk.push(path)
 	defer stk.pop()
 
 	// Determine canonical identifier for this package.
 	// For a local import the identifier is the pseudo-import path
 	// we create from the full directory to the package.
 	// Otherwise it is the usual import path.
+	// For vendored imports, it is the expanded form.
 	importPath := path
+	origPath := path
 	isLocal := build.IsLocalImport(path)
+	var vendorSearch []string
 	if isLocal {
 		importPath = dirToImportPath(filepath.Join(srcDir, path))
+	} else {
+		path, vendorSearch = vendoredImportPath(parent, path)
+		importPath = path
 	}
+
 	if p := packageCache[importPath]; p != nil {
 		if perr := disallowInternal(srcDir, p, stk); perr != p {
 			return perr
 		}
+		if perr := disallowVendor(srcDir, origPath, p, stk); perr != p {
+			return perr
+		}
 		return reusePackage(p, stk)
 	}
 
@@ -271,11 +287,33 @@ func loadImport(path string, srcDir string, stk *importStack, importPos []token.
 	// TODO: After Go 1, decide when to pass build.AllowBinary here.
 	// See issue 3268 for mistakes to avoid.
 	bp, err := buildContext.Import(path, srcDir, build.ImportComment)
+
+	// If we got an error from go/build about package not found,
+	// it contains the directories from $GOROOT and $GOPATH that
+	// were searched. Add to that message the vendor directories
+	// that were searched.
+	if err != nil && len(vendorSearch) > 0 {
+		// NOTE(rsc): The direct text manipulation here is fairly awful,
+		// but it avoids defining new go/build API (an exported error type)
+		// late in the Go 1.5 release cycle. If this turns out to be a more general
+		// problem we could define a real error type when the decision can be
+		// considered more carefully.
+		text := err.Error()
+		if strings.Contains(text, "cannot find package \"") && strings.Contains(text, "\" in any of:\n\t") {
+			old := strings.SplitAfter(text, "\n")
+			lines := []string{old[0]}
+			for _, dir := range vendorSearch {
+				lines = append(lines, "\t"+dir+" (vendor tree)\n")
+			}
+			lines = append(lines, old[1:]...)
+			err = errors.New(strings.Join(lines, ""))
+		}
+	}
 	bp.ImportPath = importPath
 	if gobin != "" {
 		bp.BinDir = gobin
 	}
-	if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path {
+	if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && (!go15VendorExperiment || !strings.Contains(path, "/vendor/")) {
 		err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment)
 	}
 	p.load(stk, bp, err)
@@ -288,10 +326,81 @@ func loadImport(path string, srcDir string, stk *importStack, importPos []token.
 	if perr := disallowInternal(srcDir, p, stk); perr != p {
 		return perr
 	}
+	if perr := disallowVendor(srcDir, origPath, p, stk); perr != p {
+		return perr
+	}
 
 	return p
 }
 
+var isDirCache = map[string]bool{}
+
+func isDir(path string) bool {
+	result, ok := isDirCache[path]
+	if ok {
+		return result
+	}
+
+	fi, err := os.Stat(path)
+	result = err == nil && fi.IsDir()
+	isDirCache[path] = result
+	return result
+}
+
+// vendoredImportPath returns the expansion of path when it appears in parent.
+// If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path,
+// x/vendor/path, vendor/path, or else stay x/y/z if none of those exist.
+// vendoredImportPath returns the expanded path or, if no expansion is found, the original.
+// If no epxansion is found, vendoredImportPath also returns a list of vendor directories
+// it searched along the way, to help prepare a useful error message should path turn
+// out not to exist.
+func vendoredImportPath(parent *Package, path string) (found string, searched []string) {
+	if parent == nil || !go15VendorExperiment {
+		return path, nil
+	}
+	dir := filepath.Clean(parent.Dir)
+	root := filepath.Clean(parent.Root)
+	if !strings.HasPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator {
+		fatalf("invalid vendoredImportPath: dir=%q root=%q separator=%q", dir, root, string(filepath.Separator))
+	}
+	vpath := "vendor/" + path
+	for i := len(dir); i >= len(root); i-- {
+		if i < len(dir) && dir[i] != filepath.Separator {
+			continue
+		}
+		// Note: checking for the vendor directory before checking
+		// for the vendor/path directory helps us hit the
+		// isDir cache more often. It also helps us prepare a more useful
+		// list of places we looked, to report when an import is not found.
+		if !isDir(filepath.Join(dir[:i], "vendor")) {
+			continue
+		}
+		targ := filepath.Join(dir[:i], vpath)
+		if isDir(targ) {
+			// We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy.
+			// We know the import path for parent's dir.
+			// We chopped off some number of path elements and
+			// added vendor\path to produce c:\gopath\src\foo\bar\baz\vendor\path.
+			// Now we want to know the import path for that directory.
+			// Construct it by chopping the same number of path elements
+			// (actually the same number of bytes) from parent's import path
+			// and then append /vendor/path.
+			chopped := len(dir) - i
+			if chopped == len(parent.ImportPath)+1 {
+				// We walked up from c:\gopath\src\foo\bar
+				// and found c:\gopath\src\vendor\path.
+				// We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7).
+				// Use "vendor/path" without any prefix.
+				return vpath, nil
+			}
+			return parent.ImportPath[:len(parent.ImportPath)-chopped] + "/" + vpath, nil
+		}
+		// Note the existence of a vendor directory in case path is not found anywhere.
+		searched = append(searched, targ)
+	}
+	return path, searched
+}
+
 // reusePackage reuses package p to satisfy the import at the top
 // of the import stack stk.  If this use causes an import loop,
 // reusePackage updates p's error information to record the loop.
@@ -384,6 +493,101 @@ func findInternal(path string) (index int, ok bool) {
 	return 0, false
 }
 
+// disallowVendor checks that srcDir is allowed to import p as path.
+// If the import is allowed, disallowVendor returns the original package p.
+// If not, it returns a new package containing just an appropriate error.
+func disallowVendor(srcDir, path string, p *Package, stk *importStack) *Package {
+	if !go15VendorExperiment {
+		return p
+	}
+
+	// The stack includes p.ImportPath.
+	// If that's the only thing on the stack, we started
+	// with a name given on the command line, not an
+	// import. Anything listed on the command line is fine.
+	if len(*stk) == 1 {
+		return p
+	}
+
+	if perr := disallowVendorVisibility(srcDir, p, stk); perr != p {
+		return perr
+	}
+
+	// Paths like x/vendor/y must be imported as y, never as x/vendor/y.
+	if i, ok := findVendor(path); ok {
+		perr := *p
+		perr.Error = &PackageError{
+			ImportStack: stk.copy(),
+			Err:         "must be imported as " + path[i+len("vendor/"):],
+		}
+		perr.Incomplete = true
+		return &perr
+	}
+
+	return p
+}
+
+// disallowVendorVisibility checks that srcDir is allowed to import p.
+// The rules are the same as for /internal/ except that a path ending in /vendor
+// is not subject to the rules, only subdirectories of vendor.
+// This allows people to have packages and commands named vendor,
+// for maximal compatibility with existing source trees.
+func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Package {
+	// The stack includes p.ImportPath.
+	// If that's the only thing on the stack, we started
+	// with a name given on the command line, not an
+	// import. Anything listed on the command line is fine.
+	if len(*stk) == 1 {
+		return p
+	}
+
+	// Check for "vendor" element.
+	i, ok := findVendor(p.ImportPath)
+	if !ok {
+		return p
+	}
+
+	// Vendor is present.
+	// Map import path back to directory corresponding to parent of vendor.
+	if i > 0 {
+		i-- // rewind over slash in ".../vendor"
+	}
+	parent := p.Dir[:i+len(p.Dir)-len(p.ImportPath)]
+	if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) {
+		return p
+	}
+
+	// Vendor is present, and srcDir is outside parent's tree. Not allowed.
+	perr := *p
+	perr.Error = &PackageError{
+		ImportStack: stk.copy(),
+		Err:         "use of vendored package not allowed",
+	}
+	perr.Incomplete = true
+	return &perr
+}
+
+// findVendor looks for the last non-terminating "vendor" path element in the given import path.
+// If there isn't one, findVendor returns ok=false.
+// Otherwise, findInternal returns ok=true and the index of the "vendor".
+//
+// Note that terminating "vendor" elements don't count: "x/vendor" is its own package,
+// not the vendored copy of an import "" (the empty import path).
+// This will allow people to have packages or commands named vendor.
+// This may help reduce breakage, or it may just be confusing. We'll see.
+func findVendor(path string) (index int, ok bool) {
+	// Two cases, depending on internal at start of string or not.
+	// The order matters: we must return the index of the final element,
+	// because the final one is where the effective import path starts.
+	switch {
+	case strings.Contains(path, "/vendor/"):
+		return strings.LastIndex(path, "/vendor/") + 1, true
+	case strings.HasPrefix(path, "vendor/"):
+		return 0, true
+	}
+	return 0, false
+}
+
 type targetDir int
 
 const (
@@ -630,7 +834,7 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package
 		if path == "C" {
 			continue
 		}
-		p1 := loadImport(path, p.Dir, stk, p.build.ImportPos[path])
+		p1 := loadImport(path, p.Dir, p, stk, p.build.ImportPos[path])
 		if p1.Name == "main" {
 			p.Error = &PackageError{
 				ImportStack: stk.copy(),
@@ -652,8 +856,11 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package
 					p.Error.Pos = pos[0].String()
 				}
 			}
-			path = p1.ImportPath
-			importPaths[i] = path
+		}
+		path = p1.ImportPath
+		importPaths[i] = path
+		if i < len(p.Imports) {
+			p.Imports[i] = path
 		}
 		deps[path] = p1
 		imports = append(imports, p1)
@@ -1294,7 +1501,7 @@ func loadPackage(arg string, stk *importStack) *Package {
 		}
 	}
 
-	return loadImport(arg, cwd, stk, nil)
+	return loadImport(arg, cwd, nil, stk, nil)
 }
 
 // packages returns the packages named by the

src/cmd/go/test.go

@@ -573,8 +573,8 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action,
 	var imports, ximports []*Package
 	var stk importStack
 	stk.push(p.ImportPath + " (test)")
-	for _, path := range p.TestImports {
-		p1 := loadImport(path, p.Dir, &stk, p.build.TestImportPos[path])
+	for i, path := range p.TestImports {
+		p1 := loadImport(path, p.Dir, p, &stk, p.build.TestImportPos[path])
 		if p1.Error != nil {
 			return nil, nil, nil, p1.Error
 		}
@@ -589,21 +589,23 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action,
 			}
 			return nil, nil, nil, err
 		}
+		p.TestImports[i] = p1.ImportPath
 		imports = append(imports, p1)
 	}
 	stk.pop()
 	stk.push(p.ImportPath + "_test")
 	pxtestNeedsPtest := false
-	for _, path := range p.XTestImports {
+	for i, path := range p.XTestImports {
 		if path == p.ImportPath {
 			pxtestNeedsPtest = true
 			continue
 		}
-		p1 := loadImport(path, p.Dir, &stk, p.build.XTestImportPos[path])
+		p1 := loadImport(path, p.Dir, p, &stk, p.build.XTestImportPos[path])
 		if p1.Error != nil {
 			return nil, nil, nil, p1.Error
 		}
 		ximports = append(ximports, p1)
+		p.XTestImports[i] = p1.ImportPath
 	}
 	stk.pop()
 
@@ -728,7 +730,7 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action,
 		if dep == ptest.ImportPath {
 			pmain.imports = append(pmain.imports, ptest)
 		} else {
-			p1 := loadImport(dep, "", &stk, nil)
+			p1 := loadImport(dep, "", nil, &stk, nil)
 			if p1.Error != nil {
 				return nil, nil, nil, p1.Error
 			}

src/cmd/go/testdata/src/vend/bad.go

@@ -0,0 +1,3 @@
+package vend
+
+import _ "r"

src/cmd/go/testdata/src/vend/good.go

@@ -0,0 +1,3 @@
+package vend
+
+import _ "p"

src/cmd/go/testdata/src/vend/hello/hello.go

@@ -0,0 +1,10 @@
+package main
+
+import (
+	"fmt"
+	"strings" // really ../vendor/strings
+)
+
+func main() {
+	fmt.Printf("%s\n", strings.Msg)
+}

src/cmd/go/testdata/src/vend/hello/hello_test.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"strings" // really ../vendor/strings
+	"testing"
+)
+
+func TestMsgInternal(t *testing.T) {
+	if strings.Msg != "hello, world" {
+		t.Fatal("unexpected msg: %v", strings.Msg)
+	}
+}