From 6cbfe964d71eccc91fa9c89b99c01f001f130d8b Mon Sep 17 00:00:00 2001 From: Dave Protasowski Date: Fri, 1 May 2020 10:18:26 -0400 Subject: [PATCH] [modules] Dependent command packages can now be built with ko (#154) Thus if you have a (in)direct command package as a dependency say `myhost.com/go/package/cmd/run` you can now publish this with the following ko command ko publish myhost.com/go/package/cmd/run --- pkg/build/gobuild.go | 79 ++++++++++++++++++++++++++++++++------- pkg/build/gobuild_test.go | 35 +++++++++++++++-- pkg/build/options.go | 11 +++++- 3 files changed, 105 insertions(+), 20 deletions(-) diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index ca12ca0ae8..231e4ee5e5 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -46,12 +46,17 @@ const ( type GetBase func(string) (v1.Image, error) type builder func(context.Context, string, v1.Platform, bool) (string, error) +type buildContext interface { + Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error) +} + type gobuild struct { getBase GetBase creationTime v1.Time build builder disableOptimizations bool - mod *modInfo + mod *modules + buildContext buildContext } // Option is a functional option for NewGo. @@ -62,7 +67,8 @@ type gobuildOpener struct { creationTime v1.Time build builder disableOptimizations bool - mod *modInfo + mod *modules + buildContext buildContext } func (gbo *gobuildOpener) Open() (Interface, error) { @@ -75,38 +81,80 @@ func (gbo *gobuildOpener) Open() (Interface, error) { build: gbo.build, disableOptimizations: gbo.disableOptimizations, mod: gbo.mod, + buildContext: gbo.buildContext, }, nil } // https://golang.org/pkg/cmd/go/internal/modinfo/#ModulePublic +type modules struct { + main *modInfo + deps map[string]*modInfo +} + type modInfo struct { Path string Dir string + Main bool } // moduleInfo returns the module path and module root directory for a project // using go modules, otherwise returns nil. // // Related: https://github.com/golang/go/issues/26504 -func moduleInfo() *modInfo { - output, err := exec.Command("go", "list", "-mod=readonly", "-m", "-json").Output() +func moduleInfo() (*modules, error) { + modules := modules{ + deps: make(map[string]*modInfo), + } + + // TODO we read all the output as a single byte array - it may + // be possible & more efficient to stream it + output, err := exec.Command("go", "list", "-mod=readonly", "-json", "-m", "all").Output() if err != nil { - return nil + return nil, nil } - var info modInfo - if err := json.Unmarshal(output, &info); err != nil { - return nil + + dec := json.NewDecoder(bytes.NewReader(output)) + + for { + var info modInfo + + err := dec.Decode(&info) + if err == io.EOF { + // all done + break + } + + modules.deps[info.Path] = &info + + if info.Main { + modules.main = &info + } + + if err != nil { + return nil, fmt.Errorf("error reading module data %w", err) + } + } + + if modules.main == nil { + return nil, fmt.Errorf("couldn't find main module") } - return &info + + return &modules, nil } // NewGo returns a build.Interface implementation that: // 1. builds go binaries named by importpath, // 2. containerizes the binary on a suitable base, func NewGo(options ...Option) (Interface, error) { + module, err := moduleInfo() + if err != nil { + return nil, err + } + gbo := &gobuildOpener{ - build: build, - mod: moduleInfo(), + build: build, + mod: module, + buildContext: &gb.Default, } for _, option := range options { @@ -141,7 +189,7 @@ func (g *gobuild) IsSupportedReference(s string) bool { // Note that we will fall back to GOPATH if the project isn't using go modules. func (g *gobuild) importPackage(ref reference) (*gb.Package, error) { if g.mod == nil { - return gb.Import(ref.Path(), gb.Default.GOPATH, gb.ImportComment) + return g.buildContext.Import(ref.Path(), gb.Default.GOPATH, gb.ImportComment) } // If we're inside a go modules project, try to use the module's directory @@ -149,8 +197,11 @@ func (g *gobuild) importPackage(ref reference) (*gb.Package, error) { // * any strict reference we get // * paths that match module path prefix (they should be in this project) // * relative paths (they should also be in this project) - if ref.IsStrict() || strings.HasPrefix(ref.Path(), g.mod.Path) || gb.IsLocalImport(ref.Path()) { - return gb.Import(ref.Path(), g.mod.Dir, gb.ImportComment) + // * path is a module + + _, isDep := g.mod.deps[ref.Path()] + if ref.IsStrict() || strings.HasPrefix(ref.Path(), g.mod.main.Path) || gb.IsLocalImport(ref.Path()) || isDep { + return g.buildContext.Import(ref.Path(), g.mod.main.Dir, gb.ImportComment) } return nil, fmt.Errorf("unmatched importPackage %q with gomodules", ref.String()) diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go index db32048100..0c236f502d 100644 --- a/pkg/build/gobuild_test.go +++ b/pkg/build/gobuild_test.go @@ -17,6 +17,7 @@ package build import ( "archive/tar" "context" + gb "go/build" "io" "io/ioutil" "path" @@ -69,12 +70,31 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) { if err != nil { t.Fatalf("random.Image() = %v", err) } - mod := &modInfo{ - Path: "github.com/google/ko/cmd/ko/test", - Dir: ".", + + mods := &modules{ + main: &modInfo{ + Path: "github.com/google/ko/cmd/ko/test", + Dir: ".", + }, + deps: map[string]*modInfo{ + "github.com/some/module/cmd": &modInfo{ + Path: "github.com/some/module/cmd", + Dir: ".", + }, + }, + } + + opts := []Option{ + WithBaseImages(func(string) (v1.Image, error) { return base, nil }), + withModuleInfo(mods), + withBuildContext(stubBuildContext{ + // make all referenced deps commands + "github.com/google/ko/cmd/ko/test": &gb.Package{Name: "main"}, + "github.com/some/module/cmd": &gb.Package{Name: "main"}, + }), } - ng, err := NewGo(WithBaseImages(func(string) (v1.Image, error) { return base, nil }), withModuleInfo(mod)) + ng, err := NewGo(opts...) if err != nil { t.Fatalf("NewGo() = %v", err) } @@ -82,6 +102,7 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) { // Supported import paths. for _, importpath := range []string{ "github.com/google/ko/cmd/ko/test", // ko can build the test package. + "github.com/some/module/cmd", // ko can build commands in dependent modules } { t.Run(importpath, func(t *testing.T) { if !ng.IsSupportedReference(importpath) { @@ -376,3 +397,9 @@ func TestGoBuild(t *testing.T) { } }) } + +type stubBuildContext map[string]*gb.Package + +func (s stubBuildContext) Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error) { + return s[path], nil +} diff --git a/pkg/build/options.go b/pkg/build/options.go index 4bcee0dbf4..66ddf4cad6 100644 --- a/pkg/build/options.go +++ b/pkg/build/options.go @@ -57,9 +57,16 @@ func withBuilder(b builder) Option { // withModulePath is a functional option for overriding the module path for // the current ko invocation. // This is exposed for testing. -func withModuleInfo(mi *modInfo) Option { +func withModuleInfo(m *modules) Option { return func(gbo *gobuildOpener) error { - gbo.mod = mi + gbo.mod = m + return nil + } +} + +func withBuildContext(b buildContext) Option { + return func(gbo *gobuildOpener) error { + gbo.buildContext = b return nil } }