Skip to content

Commit

Permalink
cmd/go: resolve non-standard imports from within GOROOT/src using ven…
Browse files Browse the repository at this point in the history 
…dor directories

Updates #30228
Fixes #26924

Change-Id: Ie625c64721559c7633396342320536396cd1fcf5
Reviewed-on: https://go-review.googlesource.com/c/go/+/164621
Run-TryBot: Bryan C. Mills <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Jay Conrod <[email protected]>
  • Loading branch information
Bryan C. Mills committed Mar 11, 2019
1 parent 1c2d4da commit fd080ea
Show file tree
Hide file tree
Showing 13 changed files with 276 additions and 56 deletions.
2 changes: 1 addition & 1 deletion src/cmd/go/internal/cfg/cfg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ var (
BuildWork bool // -work flag
BuildX bool // -x flag

CmdName string // "build", "install", "list", etc.
CmdName string // "build", "install", "list", "mod tidy", etc.

DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable)
)
Expand Down
38 changes: 18 additions & 20 deletions src/cmd/go/internal/load/pkg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,14 @@ var (
ModInit func()

// module hooks; nil if module use is disabled
ModBinDir func() string // return effective bin directory
ModLookup func(path string) (dir, realPath string, err error) // lookup effective meaning of import
ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct
ModImportPaths func(args []string) []*search.Match // expand import paths
ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary
ModInfoProg func(info string) []byte // wrap module info in .go code for binary
ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files
ModDirImportPath func(string) string // return effective import path for directory
ModBinDir func() string // return effective bin directory
ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import
ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct
ModImportPaths func(args []string) []*search.Match // expand import paths
ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary
ModInfoProg func(info string) []byte // wrap module info in .go code for binary
ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files
ModDirImportPath func(string) string // return effective import path for directory
)

var IgnoreImports bool // control whether we ignore imports in packages
Expand Down Expand Up @@ -483,8 +483,10 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
}

parentPath := ""
parentIsStd := false
if parent != nil {
parentPath = parent.ImportPath
parentIsStd = parent.Standard
}

// Determine canonical identifier for this package.
Expand All @@ -501,7 +503,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
importPath = dirToImportPath(filepath.Join(srcDir, path))
} else if cfg.ModulesEnabled {
var p string
modDir, p, modErr = ModLookup(path)
modDir, p, modErr = ModLookup(parentPath, parentIsStd, path)
if modErr == nil {
importPath = p
}
Expand Down Expand Up @@ -641,7 +643,13 @@ func isDir(path string) bool {
// Go 1.11 module legacy conversion (golang.org/issue/25069).
func ResolveImportPath(parent *Package, path string) (found string) {
if cfg.ModulesEnabled {
if _, p, e := ModLookup(path); e == nil {
parentPath := ""
parentIsStd := false
if parent != nil {
parentPath = parent.ImportPath
parentIsStd = parent.Standard
}
if _, p, e := ModLookup(parentPath, parentIsStd, path); e == nil {
return p
}
return path
Expand Down Expand Up @@ -1401,16 +1409,6 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
continue
}
p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil {
p.Error = &PackageError{
ImportStack: stk.Copy(),
Err: fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath),
}
pos := p.Internal.Build.ImportPos[path]
if len(pos) > 0 {
p.Error.Pos = pos[0].String()
}
}

path = p1.ImportPath
importPaths[i] = path
Expand Down
3 changes: 0 additions & 3 deletions src/cmd/go/internal/modload/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,6 @@ func findStandardImportPath(path string) string {
if goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) {
return filepath.Join(cfg.GOROOT, "src", path)
}
if goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, "vendor/"+path) {
return filepath.Join(cfg.GOROOT, "src/vendor", path)
}
}
return ""
}
Expand Down
13 changes: 13 additions & 0 deletions src/cmd/go/internal/modload/import.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,19 @@ func Import(path string) (m module.Version, dir string, err error) {
if search.IsStandardImportPath(path) {
if goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) {
dir := filepath.Join(cfg.GOROOT, "src", path)

// If the main module is in the standard library, attribute its packages
// to that module.
switch Target.Path {
case "cmd":
if strings.HasPrefix(path, "cmd") {
return Target, dir, nil
}
case "std":
if !strings.HasPrefix(path, "cmd") {
return Target, dir, nil
}
}
return module.Version{}, dir, nil
}
}
Expand Down
6 changes: 6 additions & 0 deletions src/cmd/go/internal/modload/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"cmd/go/internal/mvs"
"cmd/go/internal/renameio"
"cmd/go/internal/search"
"cmd/go/internal/str"
"encoding/json"
"fmt"
"go/build"
Expand Down Expand Up @@ -380,6 +381,11 @@ func InitMod() {
// modFileToBuildList initializes buildList from the modFile.
func modFileToBuildList() {
Target = modFile.Module.Mod
if (str.HasPathPrefix(Target.Path, "std") || str.HasPathPrefix(Target.Path, "cmd")) &&
search.InDir(cwd, cfg.GOROOTsrc) == "" {
base.Fatalf("go: reserved module path %s not allow outside of GOROOT/src", Target.Path)
}

list := []module.Version{Target}
for _, r := range modFile.Require {
list = append(list, r.Mod)
Expand Down
93 changes: 81 additions & 12 deletions src/cmd/go/internal/modload/load.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"io/ioutil"
"os"
"path"
pathpkg "path"
"path/filepath"
"sort"
"strings"
Expand Down Expand Up @@ -90,7 +91,9 @@ func ImportPaths(patterns []string) []*search.Match {
// the exact version of a particular module increases during
// the loader iterations.
m.Pkgs = str.StringList(fsDirs[i])
for j, pkg := range m.Pkgs {
pkgs := m.Pkgs
m.Pkgs = m.Pkgs[:0]
for _, pkg := range pkgs {
dir := pkg
if !filepath.IsAbs(dir) {
dir = filepath.Join(cwd, pkg)
Expand All @@ -108,6 +111,16 @@ func ImportPaths(patterns []string) []*search.Match {
if strings.HasPrefix(suffix, "/vendor/") {
// TODO getmode vendor check
pkg = strings.TrimPrefix(suffix, "/vendor/")
} else if Target.Path == "std" {
// Don't add the prefix "std/" to packages in the "std" module.
// It's the one module path that isn't a prefix of its packages.
pkg = strings.TrimPrefix(suffix, "/")
if pkg == "builtin" {
// "builtin" is a pseudo-package with a real source file.
// It's not included in "std", so it shouldn't be included in
// "./..." within module "std" either.
continue
}
} else {
pkg = Target.Path + suffix
}
Expand All @@ -129,10 +142,10 @@ func ImportPaths(patterns []string) []*search.Match {
// After loader is done iterating, we still need to return the
// path, so that "go list -e" produces valid output.
if iterating {
pkg = ""
continue
}
}
m.Pkgs[j] = pkg
m.Pkgs = append(m.Pkgs, pkg)
}

case strings.Contains(m.Pattern, "..."):
Expand Down Expand Up @@ -163,9 +176,7 @@ func ImportPaths(patterns []string) []*search.Match {
updateMatches(true)
for _, m := range matches {
for _, pkg := range m.Pkgs {
if pkg != "" {
roots = append(roots, pkg)
}
roots = append(roots, pkg)
}
}
return roots
Expand Down Expand Up @@ -394,13 +405,17 @@ func ModuleUsedDirectly(path string) bool {
}

// Lookup returns the source directory, import path, and any loading error for
// the package at path.
// the package at path as imported from the package in parentDir.
// Lookup requires that one of the Load functions in this package has already
// been called.
func Lookup(path string) (dir, realPath string, err error) {
func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) {
if path == "" {
panic("Lookup called with empty package path")
}

if parentIsStd {
path = loaded.stdVendor(parentPath, path)
}
pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
if !ok {
// The loader should have found all the relevant paths.
Expand Down Expand Up @@ -434,10 +449,11 @@ func Lookup(path string) (dir, realPath string, err error) {
// TODO(rsc): It might be nice to make the loader take and return
// a buildList rather than hard-coding use of the global.
type loader struct {
tags map[string]bool // tags for scanDir
testRoots bool // include tests for roots
isALL bool // created with LoadALL
testAll bool // include tests for all packages
tags map[string]bool // tags for scanDir
testRoots bool // include tests for roots
isALL bool // created with LoadALL
testAll bool // include tests for all packages
forceStdVendor bool // if true, load standard-library dependencies from the vendor subtree

// reset on each iteration
roots []*loadPkg
Expand All @@ -457,6 +473,17 @@ func newLoader() *loader {
ld := new(loader)
ld.tags = imports.Tags()
ld.testRoots = LoadTests

switch Target.Path {
case "std", "cmd":
// Inside the "std" and "cmd" modules, we prefer to use the vendor directory
// unless the command explicitly changes the module graph.
// TODO(golang.org/issue/30240): Remove this special case.
if cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ") {
ld.forceStdVendor = true
}
}

return ld
}

Expand Down Expand Up @@ -631,7 +658,11 @@ func (ld *loader) doPkg(item interface{}) {
}
}

inStd := (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "")
for _, path := range imports {
if inStd {
path = ld.stdVendor(pkg.path, path)
}
pkg.imports = append(pkg.imports, ld.pkg(path, false))
}

Expand All @@ -642,6 +673,30 @@ func (ld *loader) doPkg(item interface{}) {
}
}

// stdVendor returns the canonical import path for the package with the given
// path when imported from the standard-library package at parentPath.
func (ld *loader) stdVendor(parentPath, path string) string {
if search.IsStandardImportPath(path) {
return path
}

if str.HasPathPrefix(parentPath, "cmd") && (Target.Path != "cmd" || ld.forceStdVendor) {
vendorPath := pathpkg.Join("cmd", "vendor", path)
if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil {
return vendorPath
}
}
if Target.Path != "std" || ld.forceStdVendor {
vendorPath := pathpkg.Join("vendor", path)
if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil {
return vendorPath
}
}

// Not vendored: resolve from modules.
return path
}

// computePatternAll returns the list of packages matching pattern "all",
// starting with a list of the import paths for the packages in the main module.
func (ld *loader) computePatternAll(paths []string) []string {
Expand Down Expand Up @@ -932,6 +987,20 @@ func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) {
return vendorList, nil
}

switch Target.Path {
case "std", "cmd":
// When inside "std" or "cmd", only fetch and read go.mod files if we're
// explicitly running a command that can change the module graph. If we have
// to resolve a new dependency, we might pick the wrong version, but 'go mod
// tidy' will fix it — and new standard-library dependencies should be rare
// anyway.
//
// TODO(golang.org/issue/30240): Drop this special-case.
if cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ") {
return nil, nil
}
}

origPath := mod.Path
if repl := Replacement(mod); repl.Path != "" {
if repl.Version == "" {
Expand Down
5 changes: 5 additions & 0 deletions src/cmd/go/internal/modload/query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"cmd/go/internal/modfetch/codehost"
"cmd/go/internal/module"
"cmd/go/internal/semver"
"cmd/go/internal/str"
"fmt"
pathpkg "path"
"strings"
Expand Down Expand Up @@ -131,6 +132,10 @@ func Query(path, query string, allowed func(module.Version) bool) (*modfetch.Rev
return &modfetch.RevInfo{Version: Target.Version}, nil
}

if str.HasPathPrefix(path, "std") || str.HasPathPrefix(path, "cmd") {
return nil, fmt.Errorf("explicit requirement on standard-library module %s not allowed", path)
}

// Load versions and execute query.
repo, err := modfetch.Lookup(path)
if err != nil {
Expand Down
Loading

0 comments on commit fd080ea

Please sign in to comment.