From ae2d4b3cee100c078c5502d94d87b0498deee2b9 Mon Sep 17 00:00:00 2001 From: deads2k Date: Thu, 5 Jan 2017 16:20:00 -0500 Subject: [PATCH] bump(k8s.io/gengo): 8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21 --- Godeps/Godeps.json | 20 +- .../deepcopy-gen/generators/deepcopy.go | 123 ++++-- .../defaulter-gen/generators/defaulter.go | 1 - .../gengo/examples/set-gen/sets/byte.go | 2 +- .../k8s.io/gengo/examples/set-gen/sets/doc.go | 2 +- .../gengo/examples/set-gen/sets/empty.go | 2 +- .../k8s.io/gengo/examples/set-gen/sets/int.go | 2 +- .../gengo/examples/set-gen/sets/int64.go | 2 +- .../gengo/examples/set-gen/sets/string.go | 2 +- vendor/k8s.io/gengo/parser/parse.go | 372 ++++++++++-------- 10 files changed, 298 insertions(+), 230 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 68ee62d0a7078..9339ca65269a1 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -2669,43 +2669,43 @@ }, { "ImportPath": "k8s.io/gengo/args", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/gengo/examples/deepcopy-gen/generators", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/gengo/examples/defaulter-gen/generators", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/gengo/examples/import-boss/generators", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/gengo/examples/set-gen/generators", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/gengo/examples/set-gen/sets", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/gengo/generator", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/gengo/namer", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/gengo/parser", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/gengo/types", - "Rev": "6a1c24d7f08e671c244023ca9367d2dfbfaf57fc" + "Rev": "8dd9c9e5e82c3cca687497c3cd7ac90e702c7c21" }, { "ImportPath": "k8s.io/heapster/metrics/api/v1/types", diff --git a/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/deepcopy.go b/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/deepcopy.go index 384ec5ca36203..707111bc64d9f 100644 --- a/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/deepcopy.go +++ b/vendor/k8s.io/gengo/examples/deepcopy-gen/generators/deepcopy.go @@ -124,14 +124,16 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat inputs := sets.NewString(context.Inputs...) packages := generator.Packages{} header := append([]byte(fmt.Sprintf("// +build !%s\n\n", arguments.GeneratedBuildTag)), boilerplate...) - header = append(header, []byte( - ` -// This file was autogenerated by deepcopy-gen. Do not edit it manually! + header = append(header, []byte(` + // This file was autogenerated by deepcopy-gen. Do not edit it manually! -`)...) + `)...) boundingDirs := []string{} if customArgs, ok := arguments.CustomArgs.(*CustomArgs); ok { + if customArgs.BoundingDirs == nil { + customArgs.BoundingDirs = context.Inputs + } for i := range customArgs.BoundingDirs { // Strip any trailing slashes - they are not exactly "correct" but // this is friendlier. @@ -140,7 +142,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat } for i := range inputs { - glog.V(5).Infof("considering pkg %q", i) + glog.V(5).Infof("Considering pkg %q", i) pkg := context.Universe[i] if pkg == nil { // If the input had no Go files, for example. @@ -181,16 +183,16 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat } if pkgNeedsGeneration { + glog.V(3).Infof("Package %q needs generation", i) packages = append(packages, &generator.DefaultPackage{ PackageName: strings.Split(filepath.Base(pkg.Path), ".")[0], PackagePath: pkg.Path, HeaderText: header, GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) { - generators = []generator.Generator{} - generators = append( - generators, NewGenDeepCopy(arguments.OutputFileBaseName, pkg.Path, boundingDirs, (ptagValue == tagValuePackage), ptagRegister)) - return generators + return []generator.Generator{ + NewGenDeepCopy(arguments.OutputFileBaseName, pkg.Path, boundingDirs, (ptagValue == tagValuePackage), ptagRegister), + } }, FilterFunc: func(c *generator.Context, t *types.Type) bool { return t.Name.Package == pkg.Path @@ -202,7 +204,6 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat } const ( - apiPackagePath = "k8s.io/kubernetes/pkg/api" conversionPackagePath = "k8s.io/kubernetes/pkg/conversion" runtimePackagePath = "k8s.io/kubernetes/pkg/runtime" ) @@ -253,11 +254,16 @@ func (g *genDeepCopy) Filter(c *generator.Context, t *types.Type) bool { enabled = true } } - copyable := enabled && copyableType(t) - if copyable { - g.typesForInit = append(g.typesForInit, t) + if !enabled { + return false } - return copyable + if !copyableType(t) { + glog.V(2).Infof("Type %v is not copyable", t) + return false + } + glog.V(4).Infof("Type %v is copyable", t) + g.typesForInit = append(g.typesForInit, t) + return true } func (g *genDeepCopy) copyableAndInBounds(t *types.Type) bool { @@ -368,12 +374,20 @@ func (g *genDeepCopy) Init(c *generator.Context, w io.Writer) error { cloner := c.Universe.Type(types.Name{Package: conversionPackagePath, Name: "Cloner"}) g.imports.AddType(cloner) if !g.registerTypes { - // TODO: We should come up with a solution to register all generated - // deep-copy functions. However, for now, to avoid import cycles - // we register only those explicitly requested. - return nil + sw := generator.NewSnippetWriter(w, c, "$", "$") + sw.Do("// GetGeneratedDeepCopyFuncs returns the generated funcs, since we aren't registering them.\n", nil) + sw.Do("func GetGeneratedDeepCopyFuncs() []conversion.GeneratedDeepCopyFunc{\n", nil) + sw.Do("return []conversion.GeneratedDeepCopyFunc{\n", nil) + for _, t := range g.typesForInit { + args := argsFromType(t). + With("typeof", c.Universe.Package("reflect").Function("TypeOf")) + sw.Do("{Fn: $.type|dcFnName$, InType: $.typeof|raw$(&$.type|raw${})},\n", args) + } + sw.Do("}\n", nil) + sw.Do("}\n\n", nil) + return sw.Error() } - glog.V(5).Infof("registering types in pkg %q", g.targetPackage) + glog.V(5).Infof("Registering types in pkg %q", g.targetPackage) sw := generator.NewSnippetWriter(w, c, "$", "$") sw.Do("func init() {\n", nil) @@ -410,12 +424,12 @@ func (g *genDeepCopy) needsGeneration(t *types.Type) bool { } if g.allTypes && tv == "false" { // The whole package is being generated, but this type has opted out. - glog.V(5).Infof("not generating for type %v because type opted out", t) + glog.V(5).Infof("Not generating for type %v because type opted out", t) return false } if !g.allTypes && tv != "true" { // The whole package is NOT being generated, and this type has NOT opted in. - glog.V(5).Infof("not generating for type %v because type did not opt in", t) + glog.V(5).Infof("Not generating for type %v because type did not opt in", t) return false } return true @@ -425,7 +439,7 @@ func (g *genDeepCopy) GenerateType(c *generator.Context, t *types.Type, w io.Wri if !g.needsGeneration(t) { return nil } - glog.V(5).Infof("generating for type %v", t) + glog.V(5).Infof("Generating deepcopy function for type %v", t) sw := generator.NewSnippetWriter(w, c, "$", "$") args := argsFromType(t). @@ -535,12 +549,18 @@ func (g *genDeepCopy) doSlice(t *types.Type, sw *generator.SnippetWriter) { } func (g *genDeepCopy) doStruct(t *types.Type, sw *generator.SnippetWriter) { - if len(t.Members) == 0 { - // at least do something with in/out to avoid "declared and not used" errors - sw.Do("_ = in\n_ = out\n", nil) + if hasDeepCopyMethod(t) { + sw.Do("*out = in.DeepCopy()\n", nil) + return } + + // Simple copy covers a lot of cases. + sw.Do("*out = *in\n", nil) + + // Now fix-up fields as needed. for _, m := range t.Members { t := m.Type + hasMethod := hasDeepCopyMethod(t) if t.Kind == types.Alias { copied := *t.Underlying copied.Name = t.Name @@ -548,28 +568,40 @@ func (g *genDeepCopy) doStruct(t *types.Type, sw *generator.SnippetWriter) { } args := generator.Args{ "type": t, + "kind": t.Kind, "name": m.Name, } switch t.Kind { case types.Builtin: - sw.Do("out.$.name$ = in.$.name$\n", args) + if hasMethod { + sw.Do("out.$.name$ = in.$.name$.DeepCopy()\n", args) + } case types.Map, types.Slice, types.Pointer: - sw.Do("if in.$.name$ != nil {\n", args) - sw.Do("in, out := &in.$.name$, &out.$.name$\n", args) - g.generateFor(t, sw) - sw.Do("} else {\n", nil) - sw.Do("out.$.name$ = nil\n", args) - sw.Do("}\n", nil) + if hasMethod { + sw.Do("if in.$.name$ != nil {\n", args) + sw.Do("out.$.name$ = in.$.name$.DeepCopy()\n", args) + sw.Do("}\n", nil) + } else { + // Fixup non-nil reference-sematic types. + sw.Do("if in.$.name$ != nil {\n", args) + sw.Do("in, out := &in.$.name$, &out.$.name$\n", args) + g.generateFor(t, sw) + sw.Do("}\n", nil) + } case types.Struct: - if hasDeepCopyMethod(t) { + if hasMethod { sw.Do("out.$.name$ = in.$.name$.DeepCopy()\n", args) } else if t.IsAssignable() { - sw.Do("out.$.name$ = in.$.name$\n", args) + // Nothing else needed. } else if g.copyableAndInBounds(t) { + // Not assignable but should have a deepcopy function. + // TODO: do a topological sort of packages and ensure that this works, else inline it. sw.Do("if err := $.type|dcFnName$(&in.$.name$, &out.$.name$, c); err != nil {\n", args) sw.Do("return err\n", nil) sw.Do("}\n", nil) } else { + // Fall back on the slow-path and hope it works. + // TODO: don't depend on kubernetes code for this sw.Do("if newVal, err := c.DeepCopy(&in.$.name$); err != nil {\n", args) sw.Do("return err\n", nil) sw.Do("} else {\n", nil) @@ -577,13 +609,22 @@ func (g *genDeepCopy) doStruct(t *types.Type, sw *generator.SnippetWriter) { sw.Do("}\n", nil) } default: - sw.Do("if in.$.name$ == nil {\n", args) - sw.Do("out.$.name$ = nil\n", args) - sw.Do("} else if newVal, err := c.DeepCopy(&in.$.name$); err != nil {\n", args) - sw.Do("return err\n", nil) - sw.Do("} else {\n", nil) - sw.Do("out.$.name$ = *newVal.(*$.type|raw$)\n", args) - sw.Do("}\n", nil) + // Interfaces, Arrays, and other Kinds we don't understand. + sw.Do("// in.$.name$ is kind '$.kind$'\n", args) + if hasMethod { + sw.Do("if in.$.name$ != nil {\n", args) + sw.Do("out.$.name$ = in.$.name$.DeepCopy()\n", args) + sw.Do("}\n", args) + } else { + // TODO: don't depend on kubernetes code for this + sw.Do("if in.$.name$ != nil {\n", args) + sw.Do("if newVal, err := c.DeepCopy(&in.$.name$); err != nil {\n", args) + sw.Do("return err\n", nil) + sw.Do("} else {\n", nil) + sw.Do("out.$.name$ = *newVal.(*$.type|raw$)\n", args) + sw.Do("}\n", nil) + sw.Do("}\n", nil) + } } } } diff --git a/vendor/k8s.io/gengo/examples/defaulter-gen/generators/defaulter.go b/vendor/k8s.io/gengo/examples/defaulter-gen/generators/defaulter.go index 0895192fc28fd..1724e83a95ce8 100644 --- a/vendor/k8s.io/gengo/examples/defaulter-gen/generators/defaulter.go +++ b/vendor/k8s.io/gengo/examples/defaulter-gen/generators/defaulter.go @@ -323,7 +323,6 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat if len(newDefaulters) == 0 { glog.V(5).Infof("no defaulters in package %s", pkg.Name) - continue } packages = append(packages, diff --git a/vendor/k8s.io/gengo/examples/set-gen/sets/byte.go b/vendor/k8s.io/gengo/examples/set-gen/sets/byte.go index 3d6d0dfe43bd9..a460e4b1f55df 100644 --- a/vendor/k8s.io/gengo/examples/set-gen/sets/byte.go +++ b/vendor/k8s.io/gengo/examples/set-gen/sets/byte.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/k8s.io/gengo/examples/set-gen/sets/doc.go b/vendor/k8s.io/gengo/examples/set-gen/sets/doc.go index c5e541621f408..28a6a7d5c72b7 100644 --- a/vendor/k8s.io/gengo/examples/set-gen/sets/doc.go +++ b/vendor/k8s.io/gengo/examples/set-gen/sets/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/k8s.io/gengo/examples/set-gen/sets/empty.go b/vendor/k8s.io/gengo/examples/set-gen/sets/empty.go index 5654edd773022..cd22b953aa630 100644 --- a/vendor/k8s.io/gengo/examples/set-gen/sets/empty.go +++ b/vendor/k8s.io/gengo/examples/set-gen/sets/empty.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/k8s.io/gengo/examples/set-gen/sets/int.go b/vendor/k8s.io/gengo/examples/set-gen/sets/int.go index 6d32f84c7891b..0614e9fb000ab 100644 --- a/vendor/k8s.io/gengo/examples/set-gen/sets/int.go +++ b/vendor/k8s.io/gengo/examples/set-gen/sets/int.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/k8s.io/gengo/examples/set-gen/sets/int64.go b/vendor/k8s.io/gengo/examples/set-gen/sets/int64.go index 1de18319b7ed1..82e1ba7821241 100644 --- a/vendor/k8s.io/gengo/examples/set-gen/sets/int64.go +++ b/vendor/k8s.io/gengo/examples/set-gen/sets/int64.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/k8s.io/gengo/examples/set-gen/sets/string.go b/vendor/k8s.io/gengo/examples/set-gen/sets/string.go index da66eaf8e745a..baef7a6a2bf3e 100644 --- a/vendor/k8s.io/gengo/examples/set-gen/sets/string.go +++ b/vendor/k8s.io/gengo/examples/set-gen/sets/string.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/vendor/k8s.io/gengo/parser/parse.go b/vendor/k8s.io/gengo/parser/parse.go index 58c9c7d65e2c3..5a3440068add4 100644 --- a/vendor/k8s.io/gengo/parser/parse.go +++ b/vendor/k8s.io/gengo/parser/parse.go @@ -33,30 +33,38 @@ import ( "k8s.io/gengo/types" ) +// This clarifies when a pkg path has been canonicalized. +type importPathString string + // Builder lets you add all the go files in all the packages that you care // about, then constructs the type source data. type Builder struct { - context *build.Context - buildInfo map[string]*build.Package + context *build.Context + + // Map of package names to more canonical information about the package. + // This might hold the same value for multiple names, e.g. if someone + // referenced ./pkg/name or in the case of vendoring, which canonicalizes + // differently that what humans would type. + buildPackages map[string]*build.Package fset *token.FileSet - // map of package id to list of parsed files - parsed map[string][]parsedFile - // map of package id to absolute path (to prevent overlap) - absPaths map[string]string + // map of package path to list of parsed files + parsed map[importPathString][]parsedFile + // map of package path to absolute path (to prevent overlap) + absPaths map[importPathString]string - // Set by makePackage(), used by importer() and friends. - pkgs map[string]*tc.Package + // Set by typeCheckPackage(), used by importPackage() and friends. + typeCheckedPackages map[importPathString]*tc.Package // Map of package path to whether the user requested it or it was from // an import. - userRequested map[string]bool + userRequested map[importPathString]bool // All comments from everywhere in every parsed file. endLineToCommentGroup map[fileLine]*ast.CommentGroup // map of package to list of packages it imports. - importGraph map[string]map[string]struct{} + importGraph map[importPathString]map[string]struct{} } // parsedFile is for tracking files with name @@ -87,13 +95,14 @@ func New() *Builder { c.CgoEnabled = false return &Builder{ context: &c, - buildInfo: map[string]*build.Package{}, + buildPackages: map[string]*build.Package{}, + typeCheckedPackages: map[importPathString]*tc.Package{}, fset: token.NewFileSet(), - parsed: map[string][]parsedFile{}, - absPaths: map[string]string{}, - userRequested: map[string]bool{}, + parsed: map[importPathString][]parsedFile{}, + absPaths: map[importPathString]string{}, + userRequested: map[importPathString]bool{}, endLineToCommentGroup: map[fileLine]*ast.CommentGroup{}, - importGraph: map[string]map[string]struct{}{}, + importGraph: map[importPathString]map[string]struct{}{}, } } @@ -105,71 +114,73 @@ func (b *Builder) AddBuildTags(tags ...string) { // Get package information from the go/build package. Automatically excludes // e.g. test files and files for other platforms-- there is quite a bit of // logic of that nature in the build package. -func (b *Builder) buildPackage(pkgPath string) (*build.Package, error) { - // This is a bit of a hack. The srcDir argument to Import() should - // properly be the dir of the file which depends on the package to be - // imported, so that vendoring can work properly. We assume that there is - // only one level of vendoring, and that the CWD is inside the GOPATH, so - // this should be safe. - cwd, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("unable to get current directory: %v", err) +func (b *Builder) importBuildPackage(dir string) (*build.Package, error) { + if buildPkg, ok := b.buildPackages[dir]; ok { + return buildPkg, nil } - - // First, find it, so we know what path to use. - pkg, err := b.context.Import(pkgPath, cwd, build.FindOnly) - if err != nil { - return nil, fmt.Errorf("unable to *find* %q: %v", pkgPath, err) - } - - pkgPath = pkg.ImportPath - - if pkg, ok := b.buildInfo[pkgPath]; ok { - return pkg, nil - } - pkg, err = b.context.Import(pkgPath, cwd, build.ImportComment) + // This validates the `package foo // github.com/bar/foo` comments. + buildPkg, err := b.importWithMode(dir, build.ImportComment) if err != nil { if _, ok := err.(*build.NoGoError); !ok { - return nil, fmt.Errorf("unable to import %q: %v", pkgPath, err) + return nil, fmt.Errorf("unable to import %q: %v", dir, err) } } - b.buildInfo[pkgPath] = pkg - - if b.importGraph[pkgPath] == nil { - b.importGraph[pkgPath] = map[string]struct{}{} + if buildPkg == nil { + // Might be an empty directory. Try to just find the dir. + buildPkg, err = b.importWithMode(dir, build.FindOnly) + if err != nil { + return nil, err + } } - for _, p := range pkg.Imports { - b.importGraph[pkgPath][p] = struct{}{} + + // Remember it under the user-provided name. + glog.V(5).Infof("saving buildPackage %s", dir) + b.buildPackages[dir] = buildPkg + if dir != buildPkg.ImportPath { + // Since `dir` is not the canonical name, see if we knew it under another name. + if buildPkg, ok := b.buildPackages[buildPkg.ImportPath]; ok { + return buildPkg, nil + } + // Must be new, save it under the canonical name, too. + glog.V(5).Infof("saving buildPackage %s", buildPkg.ImportPath) + b.buildPackages[buildPkg.ImportPath] = buildPkg } - return pkg, nil + + return buildPkg, nil } -// AddFile adds a file to the set. The pkg must be of the form -// "canonical/pkg/path" and the path must be the absolute path to the file. -func (b *Builder) AddFile(pkg string, path string, src []byte) error { - return b.addFile(pkg, path, src, true) +// AddFileForTest adds a file to the set, without verifying that the provided +// pkg actually exists on disk. The pkg must be of the form "canonical/pkg/path" +// and the path must be the absolute path to the file. Because this bypasses +// the normal recursive finding of package dependencies (on disk), test should +// sort their test files topologically first, so all deps are resolved by the +// time we need them. +func (b *Builder) AddFileForTest(pkg string, path string, src []byte) error { + if err := b.addFile(importPathString(pkg), path, src, true); err != nil { + return err + } + if _, err := b.typeCheckPackage(importPathString(pkg)); err != nil { + return err + } + return nil } -// addFile adds a file to the set. The pkg must be of the form +// addFile adds a file to the set. The pkgPath must be of the form // "canonical/pkg/path" and the path must be the absolute path to the file. A // flag indicates whether this file was user-requested or just from following // the import graph. -func (b *Builder) addFile(pkg string, path string, src []byte, userRequested bool) error { +func (b *Builder) addFile(pkgPath importPathString, path string, src []byte, userRequested bool) error { + glog.V(6).Infof("addFile %s %s", pkgPath, path) p, err := parser.ParseFile(b.fset, path, src, parser.DeclarationErrors|parser.ParseComments) if err != nil { return err } - dirPath := filepath.Dir(path) - if prev, found := b.absPaths[pkg]; found { - if dirPath != prev { - return fmt.Errorf("package %q (%s) previously resolved to %s", pkg, dirPath, prev) - } - } else { - b.absPaths[pkg] = dirPath - } - b.parsed[pkg] = append(b.parsed[pkg], parsedFile{path, p}) - b.userRequested[pkg] = userRequested + // This is redundant with addDir, but some tests call AddFileForTest, which + // call into here without calling addDir. + b.userRequested[pkgPath] = userRequested || b.userRequested[pkgPath] + + b.parsed[pkgPath] = append(b.parsed[pkgPath], parsedFile{path, p}) for _, c := range p.Comments { position := b.fset.Position(c.End()) b.endLineToCommentGroup[fileLine{position.Filename, position.Line}] = c @@ -177,12 +188,12 @@ func (b *Builder) addFile(pkg string, path string, src []byte, userRequested boo // We have to get the packages from this specific file, in case the // user added individual files instead of entire directories. - if b.importGraph[pkg] == nil { - b.importGraph[pkg] = map[string]struct{}{} + if b.importGraph[pkgPath] == nil { + b.importGraph[pkgPath] = map[string]struct{}{} } for _, im := range p.Imports { importedPath := strings.Trim(im.Path.Value, `"`) - b.importGraph[pkg][importedPath] = struct{}{} + b.importGraph[pkgPath][importedPath] = struct{}{} } return nil } @@ -191,45 +202,40 @@ func (b *Builder) addFile(pkg string, path string, src []byte, userRequested boo // a single go package in it. GOPATH, GOROOT, and the location of your go // binary (`which go`) will all be searched if dir doesn't literally resolve. func (b *Builder) AddDir(dir string) error { - return b.addDir(dir, true) + _, err := b.importPackage(dir, true) + return err } // AddDirRecursive is just like AddDir, but it also recursively adds // subdirectories; it returns an error only if the path couldn't be resolved; // any directories recursed into without go source are ignored. func (b *Builder) AddDirRecursive(dir string) error { - // This is a bit of a hack. The srcDir argument to Import() should - // properly be the dir of the file which depends on the package to be - // imported, so that vendoring can work properly. We assume that there is - // only one level of vendoring, and that the CWD is inside the GOPATH, so - // this should be safe. - cwd, err := os.Getwd() - if err != nil { - return fmt.Errorf("unable to get current directory: %v", err) - } - - // First, find it, so we know what path to use. - pkg, err := b.context.Import(dir, cwd, build.FindOnly) - if err != nil { - return fmt.Errorf("unable to *find* %q: %v", dir, err) - } - - if err := b.addDir(dir, true); err != nil { + // Add the root. + if _, err := b.importPackage(dir, true); err != nil { glog.Warningf("Ignoring directory %v: %v", dir, err) } - prefix := strings.TrimSuffix(pkg.Dir, strings.TrimSuffix(dir, "/")) - filepath.Walk(pkg.Dir, func(path string, info os.FileInfo, err error) error { + // filepath.Walk includes the root dir, but we already did that, so we'll + // remove that prefix and rebuild a package import path. + prefix := b.buildPackages[dir].Dir + fn := func(path string, info os.FileInfo, err error) error { if info != nil && info.IsDir() { - trimmed := strings.TrimPrefix(path, prefix) - if trimmed != "" { - if err := b.addDir(trimmed, true); err != nil { - glog.Warningf("Ignoring child directory %v: %v", trimmed, err) + rel := strings.TrimPrefix(path, prefix) + if rel != "" { + // Make a pkg path. + pkg := filepath.Join(b.buildPackages[dir].ImportPath, rel) + + // Add it. + if _, err := b.importPackage(pkg, true); err != nil { + glog.Warningf("Ignoring child directory %v: %v", pkg, err) } } } return nil - }) + } + if err := filepath.Walk(b.buildPackages[dir].Dir, fn); err != nil { + return err + } return nil } @@ -239,44 +245,46 @@ func (b *Builder) AddDirRecursive(dir string) error { // GOPATH, GOROOT, and the location of your go binary (`which go`) will all be // searched if dir doesn't literally resolve. func (b *Builder) AddDirTo(dir string, u *types.Universe) error { - if _, found := b.parsed[dir]; !found { - // We want all types from this package, as if they were directly added - // by the user. They WERE added by the user, in effect. - if err := b.addDir(dir, true); err != nil { - return err - } - } else { - // We already had this package, but we want it to be considered as if - // the user addid it directly. - b.userRequested[dir] = true + // We want all types from this package, as if they were directly added + // by the user. They WERE added by the user, in effect. + if _, err := b.importPackage(dir, true); err != nil { + return err } - return b.findTypesIn(dir, u) + return b.findTypesIn(importPathString(b.buildPackages[dir].ImportPath), u) } // The implementation of AddDir. A flag indicates whether this directory was // user-requested or just from following the import graph. func (b *Builder) addDir(dir string, userRequested bool) error { - pkg, err := b.buildPackage(dir) + glog.V(5).Infof("addDir %s", dir) + buildPkg, err := b.importBuildPackage(dir) if err != nil { return err } - // Check in case this package was added (maybe dir was not canonical) - if wasRequested, wasAdded := b.userRequested[dir]; wasAdded { - if !userRequested || userRequested == wasRequested { - return nil + pkgPath := importPathString(buildPkg.ImportPath) + if dir != buildPkg.ImportPath { + glog.V(5).Infof("addDir %s, canonical path is %s", dir, pkgPath) + } + + // Sanity check the pkg dir has not changed. + if prev, found := b.absPaths[pkgPath]; found { + if buildPkg.Dir != prev { + return fmt.Errorf("package %q (%s) previously resolved to %s", pkgPath, buildPkg.Dir, prev) } + } else { + b.absPaths[pkgPath] = buildPkg.Dir } - for _, n := range pkg.GoFiles { + for _, n := range buildPkg.GoFiles { if !strings.HasSuffix(n, ".go") { continue } - absPath := filepath.Join(pkg.Dir, n) + absPath := filepath.Join(buildPkg.Dir, n) data, err := ioutil.ReadFile(absPath) if err != nil { return fmt.Errorf("while loading %q: %v", absPath, err) } - err = b.addFile(dir, absPath, data, userRequested) + err = b.addFile(pkgPath, absPath, data, userRequested) if err != nil { return fmt.Errorf("while parsing %q: %v", absPath, err) } @@ -284,32 +292,53 @@ func (b *Builder) addDir(dir string, userRequested bool) error { return nil } -// importer is a function that will be called by the type check package when it -// needs to import a go package. 'path' is the import path. go1.5 changes the -// interface, and importAdapter below implements the new interface in terms of -// the old one. -func (b *Builder) importer(imports map[string]*tc.Package, path string) (*tc.Package, error) { - if pkg, ok := imports[path]; ok { - return pkg, nil +// importPackage is a function that will be called by the type check package when it +// needs to import a go package. 'path' is the import path. +func (b *Builder) importPackage(dir string, userRequested bool) (*tc.Package, error) { + glog.V(5).Infof("importPackage %s", dir) + var pkgPath = importPathString(dir) + + // Get the canonical path if we can. + if buildPkg := b.buildPackages[dir]; buildPkg != nil { + glog.V(5).Infof("importPackage %s, canonical path is %s", dir, buildPkg.ImportPath) + pkgPath = importPathString(buildPkg.ImportPath) } + + // If we have not seen this before, process it now. ignoreError := false - if _, ours := b.parsed[path]; !ours { + if _, found := b.parsed[pkgPath]; !found { // Ignore errors in paths that we're importing solely because // they're referenced by other packages. ignoreError = true - if err := b.addDir(path, false); err != nil { + + // Add it. + if err := b.addDir(dir, userRequested); err != nil { return nil, err } + + // Get the canonical path now that it has been added. + if buildPkg := b.buildPackages[dir]; buildPkg != nil { + glog.V(5).Infof("importPackage %s, canonical path is %s", dir, buildPkg.ImportPath) + pkgPath = importPathString(buildPkg.ImportPath) + } } - pkg, err := b.typeCheckPackage(path) + + // If it was previously known, just check that the user-requestedness hasn't + // changed. + b.userRequested[pkgPath] = userRequested || b.userRequested[pkgPath] + + // Run the type checker. We may end up doing this to pkgs that are already + // done, or are in the queue to be done later, but it will short-circuit, + // and we can't miss pkgs that are only depended on. + pkg, err := b.typeCheckPackage(pkgPath) if err != nil { if ignoreError && pkg != nil { - glog.V(2).Infof("type checking encountered some errors in %q, but ignoring.\n", path) + glog.V(2).Infof("type checking encountered some errors in %q, but ignoring.\n", pkgPath) } else { return nil, err } } - imports[path] = pkg + return pkg, nil } @@ -318,85 +347,58 @@ type importAdapter struct { } func (a importAdapter) Import(path string) (*tc.Package, error) { - return a.b.importer(a.b.pkgs, path) + return a.b.importPackage(path, false) } // typeCheckPackage will attempt to return the package even if there are some // errors, so you may check whether the package is nil or not even if you get // an error. -func (b *Builder) typeCheckPackage(id string) (*tc.Package, error) { - if pkg, ok := b.pkgs[id]; ok { +func (b *Builder) typeCheckPackage(pkgPath importPathString) (*tc.Package, error) { + glog.V(5).Infof("typeCheckPackage %s", pkgPath) + if pkg, ok := b.typeCheckedPackages[pkgPath]; ok { if pkg != nil { + glog.V(6).Infof("typeCheckPackage %s already done", pkgPath) return pkg, nil } // We store a nil right before starting work on a package. So // if we get here and it's present and nil, that means there's // another invocation of this function on the call stack // already processing this package. - return nil, fmt.Errorf("circular dependency for %q", id) + return nil, fmt.Errorf("circular dependency for %q", pkgPath) } - parsedFiles, ok := b.parsed[id] + parsedFiles, ok := b.parsed[pkgPath] if !ok { - return nil, fmt.Errorf("No files for pkg %q: %#v", id, b.parsed) + return nil, fmt.Errorf("No files for pkg %q: %#v", pkgPath, b.parsed) } files := make([]*ast.File, len(parsedFiles)) for i := range parsedFiles { files[i] = parsedFiles[i].file } - b.pkgs[id] = nil + b.typeCheckedPackages[pkgPath] = nil c := tc.Config{ IgnoreFuncBodies: true, - // Note that importAdater can call b.import which calls this + // Note that importAdapter can call b.importPackage which calls this // method. So there can't be cycles in the import graph. Importer: importAdapter{b}, Error: func(err error) { glog.V(2).Infof("type checker error: %v\n", err) }, } - pkg, err := c.Check(id, b.fset, files, nil) - b.pkgs[id] = pkg // record the result whether or not there was an error + pkg, err := c.Check(string(pkgPath), b.fset, files, nil) + b.typeCheckedPackages[pkgPath] = pkg // record the result whether or not there was an error return pkg, err } -func (b *Builder) makeAllPackages() error { - // Take a snapshot to iterate, since this will recursively mutate b.parsed. - keys := []string{} - for id := range b.parsed { - keys = append(keys, id) - } - for _, id := range keys { - if _, err := b.makePackage(id); err != nil { - return err - } - } - return nil -} - -func (b *Builder) makePackage(id string) (*tc.Package, error) { - if b.pkgs == nil { - b.pkgs = map[string]*tc.Package{} - } - - // We have to check here even though we made a new one above, - // because typeCheckPackage follows the import graph, which may - // cause a package to be filled before we get to it in this - // loop. - if pkg, done := b.pkgs[id]; done { - return pkg, nil - } - return b.typeCheckPackage(id) -} - // FindPackages fetches a list of the user-imported packages. // Note that you need to call b.FindTypes() first. func (b *Builder) FindPackages() []string { result := []string{} - for pkgPath := range b.pkgs { + for pkgPath := range b.typeCheckedPackages { if b.userRequested[pkgPath] { // Since walkType is recursive, all types that are in packages that // were directly mentioned will be included. We don't need to // include all types in all transitive packages, though. - result = append(result, pkgPath) + result = append(result, string(pkgPath)) } } return result @@ -405,13 +407,15 @@ func (b *Builder) FindPackages() []string { // FindTypes finalizes the package imports, and searches through all the // packages for types. func (b *Builder) FindTypes() (types.Universe, error) { - if err := b.makeAllPackages(); err != nil { - return nil, err - } - u := types.Universe{} + // Take a snapshot of pkgs to iterate, since this will recursively mutate + // b.parsed. + keys := []importPathString{} for pkgPath := range b.parsed { + keys = append(keys, pkgPath) + } + for _, pkgPath := range keys { if err := b.findTypesIn(pkgPath, &u); err != nil { return nil, err } @@ -421,22 +425,28 @@ func (b *Builder) FindTypes() (types.Universe, error) { // findTypesIn finalizes the package import and searches through the package // for types. -func (b *Builder) findTypesIn(pkgPath string, u *types.Universe) error { - pkg, err := b.makePackage(pkgPath) - if err != nil { - return err +func (b *Builder) findTypesIn(pkgPath importPathString, u *types.Universe) error { + glog.V(5).Infof("findTypesIn %s", pkgPath) + pkg := b.typeCheckedPackages[pkgPath] + if pkg == nil { + return fmt.Errorf("findTypesIn(%s): package is not known", pkgPath) } if !b.userRequested[pkgPath] { // Since walkType is recursive, all types that the // packages they asked for depend on will be included. // But we don't need to include all types in all // *packages* they depend on. + glog.V(5).Infof("findTypesIn %s: package is not user requested", pkgPath) return nil } + // We're keeping this package. This call will create the record. + u.Package(string(pkgPath)).Name = pkg.Name() + u.Package(string(pkgPath)).Path = pkg.Path() + for _, f := range b.parsed[pkgPath] { if strings.HasSuffix(f.name, "/doc.go") { - tp := u.Package(pkgPath) + tp := u.Package(string(pkgPath)) for i := range f.file.Comments { tp.Comments = append(tp.Comments, splitLines(f.file.Comments[i].Text())...) } @@ -480,12 +490,30 @@ func (b *Builder) findTypesIn(pkgPath string, u *types.Universe) error { } } for p := range b.importGraph[pkgPath] { - u.AddImports(pkgPath, p) + u.AddImports(string(pkgPath), p) } - u.Package(pkgPath).Name = pkg.Name() return nil } +func (b *Builder) importWithMode(dir string, mode build.ImportMode) (*build.Package, error) { + // This is a bit of a hack. The srcDir argument to Import() should + // properly be the dir of the file which depends on the package to be + // imported, so that vendoring can work properly and local paths can + // resolve. We assume that there is only one level of vendoring, and that + // the CWD is inside the GOPATH, so this should be safe. Nobody should be + // using local (relative) paths except on the CLI, so CWD is also + // sufficient. + cwd, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("unable to get current directory: %v", err) + } + buildPkg, err := b.context.Import(dir, cwd, mode) + if err != nil { + return nil, err + } + return buildPkg, nil +} + // if there's a comment on the line `lines` before pos, return its text, otherwise "". func (b *Builder) priorCommentLines(pos token.Pos, lines int) *ast.CommentGroup { position := b.fset.Position(pos)