(apro) Merge pull request #258 from datawire/lukeshu/make-fast

build-aux: migrate to go.mod, rewrite go-opensource in Go

.gitignore

@@ -1,10 +1,9 @@
+# prelude.mk
+/bin/
+
 # teleproxy.mk
-/teleproxy
 /teleproxy.log
 
-# kubeapply.mk
-/kubeapply
-
 # kubernaut.mk
 *.knaut
 *.knaut.claim
@@ -15,14 +14,21 @@
 /docker-registry.yaml.o
 
 # go-mod.mk
-/golangci-lint
 /go-test.tap
 /go1*.src.tar.gz
 
+# go-mod.mk
+/gopath/
+
 # --- Files ignored by older versions ---
 # Remove the tail of this list when the commit making the change gets
 # far enough in to the past.
 
+# 2019-07-01
+/teleproxy
+/kubeapply
+/golangci-lint
+
 # 2019-02-06
 /patter.go
 /patter.go.tmp

CHANGELOG.md

@@ -0,0 +1,85 @@
+<!-- -*- fill-column: 100 -*- -->
+# Datawire build-aux CHANGELOG
+
+ - 2019-07-05: `build-aux-push`: Work around problem with `git subtree`; avoid accidentally pushing
+               proprietary code to build-aux.git.
+
+ - 2019-07-03: `go-mod.mk`: `.opensource.tar.gz` files are still part of `make build`, but no longer
+               part of `make go-build`.
+
+ - 2019-07-03: Rewrite the `go-opensource` Bash script as `go-mkopensource` in Go.
+
+ - 2019-07-03: Migrate from `curl` to `go.mod`.
+ - 2019-07-03: BREAKING CHANGE: Move executables to be in `./build-aux/bin/` instead of directly in
+               `./build-aux/`.  Each of these programs now has a variable to refer to it by, instead
+               of having to hard-code the path.  It is also no longer valid to use one of those
+               programs without depending on it first.
+
+ - 2019-06-20: `go-mod.mk`: Bump golangci-lint version 1.15.0→1.17.1.
+ - 2019-06-20: `go-mod.mk`: For each binary, generate a `BIN.opensource.tar.gz` file.
+ - 2019-06-20: `go-workspace.mk`: Remove.
+
+ - 2019-05-31: `go-modmk`: Add `go doc` target to run `godoc -http`.
+
+ - 2019-05-01: BREAKING CHANGE: `docker.mk`: Don't include `kuberanut-ui.mk`.
+ - 2019-05-01: BREAKING CHANGE: `teleproxy.mk`: Don't include `kuberanut-ui.mk`.
+ - 2019-05-01: Go over documented inputs/outputs; differentiate between "eager" and "lazy inputs".
+ - 2019-05-01: BREAKING CHANGE: `help.mk`: Don't include `common.mk`.
+ - 2019-05-01: BREAKING CHANGE: `kubeapply.mk`: Don't include `common.mk`.
+ - 2019-05-01: BREAKING CHANGE: `kubernaut.mk`: Don't include `common.mk`.
+ - 2019-05-01: BREAKING CHANGE: `flock.mk`: Delete; merge in to `prelude.mk`.
+ - 2019-05-01: Drop dependency on Go for setting `GO{HOST,}{OS,ARCH}`.
+ - 2019-05-01: BREAKING CHANGE: Use GOHOSTOS/GOHOSTARCH instead of GOOS/GOARCH as appropriate.
+ - 2019-05-01: `prelude.mk`: Add `$(call lazyonce,…)`.
+ - 2019-05-01: `prelude.mk`: Introduce, steal code from `common.mk`.
+
+ - 2019-02-15: `kubernaut-ui.mk`: Avoid warnings from `make` inside of `makeh shell`.
+
+ - 2019-02-13: `_go-common.mk`: Bump golangci-lint version 1.13.1→1.15.0.
+
+ - 2019-02-08: `common.mk`: Slight rework of how test dependencies work.
+
+ - 2019-01-30: `common.mk`: Add TAP-based `check` infrastructure.
+
+ - 2019-01-23: `docker.mk`: Fix `.knaut.push` on macOS.
+ - 2019-01-18: `docker.mk`: Robustness improvements.
+
+ - 2019-01-15: `go.mk`: Remove this symlink to `go-workspace.mk`.
+
+ - 2019-01-10: `teleproxy.mk`: Go to simple 1s polling, instead of exponential backoff.
+
+ - 2019-01-05: `build-aux-push`: Rejoin afer the split.
+
+ - 2018-12-23: `flock.mk`: Introduce.
+
+ - 2018-12-22: `go-mod.mk`: Enable parallel builds for Go 1.12+
+ - 2018-12-22: `go-workspace.mk`: Enable parallel builds for Go 1.10+
+ - 2018-12-22: Almost exclusively use `$(KUBECONFIG)` to refer to `cluster.knaut`.  This allows the
+               caller to override the value by passing a `KUBECONFIG=…` argument to `make`.  This is
+               an important property for running `make check` inside of `testbench`, so that
+               testbench can provide the Kubernaut file.  The exception is that we still hard-code
+               the value in `clean: cluster.knaut.clean` so that `make clean` doesn't remove the
+               user's Kubernaut file.
+
+ - 2018-12-22: `_go-common.mk`: Add a variable to disable `go-test`'s automatic definition
+
+ - 2018-12-20: BREAKING CHANGE: `common.mk`: The argument order of `$(call joinlist,…)` has changed.
+
+ - 2018-12-20: BREAKING CHANGE: `go-{mod,workspace}.mk`: `$(pkg)` is now `$(go.module)`, and is no
+               longer necessary to manually set; `go-{mod,workspace}.mk` are both smart enough to
+               set it automatically!
+ - 2018-12-20: BREAKING CHANGE: `go-{mod,workspace}.mk`: `$(bins)` is now `$(notdir $(go.bins))`,
+               and is no longer nescessary to manually set; `go-{mod,workspace}.mk` are both smart
+               enough to set it automatically!
+ - 2018-12-20: BREAKING CHANGE: `go-{mod,workspace}.mk`: Go binaries now go in
+               `./bin_$(GOOS)_$(GOARCH)/` by default, instead of the old default of `./`
+ - 2018-12-20: `go-{mod,workspace}.mk`: The `Makefile` doesn't need to export `GOPATH` or anything
+               like that, the .mk snippets now export things.
+ - 2018-12-20: `go-{mod,workspace}.mk`: It's no longer necessary to use `$(GO)`, just write
+               `go`--but don't don't use it inside of `$(shell …)` if you can avoid it, exported
+               variables don't affect `$(shell …)`.
+ - 2018-12-20: `common.mk`: It's no longer necessary to clean up `bin_*/`, in your `clean:` rule,
+               `common.mk` takes care of that for you.
+ - 2018-12-20: `common.mk` goes ahead and declares several common targets as `.PHONY`
+
+ - 2018-12-06: Rename `go.mk` to `go-workspace.mk`, introduce a new `go-mod.mk`.

HACKING.md

@@ -2,8 +2,6 @@
 
 ## Misc notes
 
- - Any `.go` files should say `// +build ignore` to prevent `go list
-   ./...` from picking them up.
  - If you have a dependency on another `.mk` file includes, include it
    with `include $(dir $(lastword $(MAKEFILE_LIST)))common.mk`.
  - `.PHONY` targets that you wish to be user-visible should have a `##
@@ -16,6 +14,12 @@
 
    include guards to make sure they are only included once; similar to
    how you would with a C header file.
+ - The Make `export` directive *only* affects recipes, and does *not*
+   affect `$(shell …)`.  Because of this, you shoud not call `go`
+   inside of `$(shell …)`, as `$GO111MODULE` may not have the correct
+   value.
+ - Make sure to pass `--fail` to `curl` when downloading things;
+   otherwise it will silently save 404 HTML/XML pages.
 
 ## Naming conventions
 
@@ -43,10 +47,13 @@
    (including macOS).  Therefore, `--` working is a reasonable base
    assumption.  Known exceptions:
     * macOS `chmod`
+ - Prefer `curl` to `wget`; macOS ships with `curl`, it doesn't ship
+   with `wget`.
 
 ## Style guide
 
- - (see "Naming conventions")
+ - See "Naming conventions"..
+ - See [`docs/conventions.md`](./docs/conventions.md).
  - Place `.PHONY:` immediately *after* the rule definition.
  - Use pattern rules instead of "old-fashioned suffix rules" (as the
    GNU Make manual refers to them).

flock.go → bin-go/flock/flock.go

@@ -1,7 +1,5 @@
 // Copyright 2018 Datawire. All rights reserved.
 
-// +build ignore
-
 // flock.go is a minimal implementation of flock(1) (from util-linux)
 // for systems that don't have flock(1) but do have flock(2).
 //

bin-go/go-mkopensource/internal/golist/golist.go

@@ -0,0 +1,28 @@
+package golist
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"os/exec"
+)
+
+func GoList(pkg string, flags ...string) ([]Package, error) {
+	stdoutBytes, err := exec.Command("go", append([]string{"list"}, append(flags, "-json", "--", pkg)...)...).Output()
+	if err != nil {
+		return nil, err
+	}
+	stdoutDecoder := json.NewDecoder(bytes.NewReader(stdoutBytes))
+	var ret []Package
+	for {
+		var pkg Package
+		if err := stdoutDecoder.Decode(&pkg); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		ret = append(ret, pkg)
+	}
+	return ret, nil
+}

bin-go/go-mkopensource/internal/golist/types.go

@@ -0,0 +1,90 @@
+package golist
+
+import (
+	"time"
+)
+
+// These definitions are copied verbatim from `go help list`.
+
+type Package struct {
+	Dir           string   // directory containing package sources
+	ImportPath    string   // import path of package in dir
+	ImportComment string   // path in import comment on package statement
+	Name          string   // package name
+	Doc           string   // package documentation string
+	Target        string   // install path
+	Shlib         string   // the shared library that contains this package (only set when -linkshared)
+	Goroot        bool     // is this package in the Go root?
+	Standard      bool     // is this package part of the standard Go library?
+	Stale         bool     // would 'go install' do anything for this package?
+	StaleReason   string   // explanation for Stale==true
+	Root          string   // Go root or Go path dir containing this package
+	ConflictDir   string   // this directory shadows Dir in $GOPATH
+	BinaryOnly    bool     // binary-only package: cannot be recompiled from sources
+	ForTest       string   // package is only for use in named test
+	Export        string   // file containing export data (when using -export)
+	Module        *Module  // info about package's containing module, if any (can be nil)
+	Match         []string // command-line patterns matching this package
+	DepOnly       bool     // package is only a dependency, not explicitly listed
+
+	// Source files
+	GoFiles         []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
+	CgoFiles        []string // .go source files that import "C"
+	CompiledGoFiles []string // .go files presented to compiler (when using -compiled)
+	IgnoredGoFiles  []string // .go source files ignored due to build constraints
+	CFiles          []string // .c source files
+	CXXFiles        []string // .cc, .cxx and .cpp source files
+	MFiles          []string // .m source files
+	HFiles          []string // .h, .hh, .hpp and .hxx source files
+	FFiles          []string // .f, .F, .for and .f90 Fortran source files
+	SFiles          []string // .s source files
+	SwigFiles       []string // .swig files
+	SwigCXXFiles    []string // .swigcxx files
+	SysoFiles       []string // .syso object files to add to archive
+	TestGoFiles     []string // _test.go files in package
+	XTestGoFiles    []string // _test.go files outside package
+
+	// Cgo directives
+	CgoCFLAGS    []string // cgo: flags for C compiler
+	CgoCPPFLAGS  []string // cgo: flags for C preprocessor
+	CgoCXXFLAGS  []string // cgo: flags for C++ compiler
+	CgoFFLAGS    []string // cgo: flags for Fortran compiler
+	CgoLDFLAGS   []string // cgo: flags for linker
+	CgoPkgConfig []string // cgo: pkg-config names
+
+	// Dependency information
+	Imports      []string          // import paths used by this package
+	ImportMap    map[string]string // map from source import to ImportPath (identity entries omitted)
+	Deps         []string          // all (recursively) imported dependencies
+	TestImports  []string          // imports from TestGoFiles
+	XTestImports []string          // imports from XTestGoFiles
+
+	// Error information
+	Incomplete bool            // this package or a dependency has an error
+	Error      *PackageError   // error loading package
+	DepsErrors []*PackageError // errors loading dependencies
+}
+
+type PackageError struct {
+	ImportStack []string // shortest path from package named on command line to this one
+	Pos         string   // position of error (if present, file:line:col)
+	Err         string   // the error itself
+}
+
+type Module struct {
+	Path     string       // module path
+	Version  string       // module version
+	Versions []string     // available module versions (with -versions)
+	Replace  *Module      // replaced by this module
+	Time     *time.Time   // time version was created
+	Update   *Module      // available update, if any (with -u)
+	Main     bool         // is this the main module?
+	Indirect bool         // is this module only an indirect dependency of main module?
+	Dir      string       // directory holding files for this module, if any
+	GoMod    string       // path to go.mod file for this module, if any
+	Error    *ModuleError // error loading module
+}
+
+type ModuleError struct {
+	Err string // the error itself
+}