[WIP, DNM] multi-worker daemon

- [X] put multiples workers in a single binary ("-tags containerd standalone")
- [X] add worker selector to LLB vertex metadata
- [X] s/worker/executor/g
- [WIP] introduce the new "worker" concept moby#176 (comment)
- [X] fix up CLI
- [ ] fix up tests
- [ ] allow using multiples workers (requires inter-vertex cache copier, HUGE!) --> will be separate PR

Implementation notes:
- "Workers" are renamed to "executors" now
- The new "worker" instance holds an "executor" instance and its
related stuffs such as the snapshotter
- For containerd, we have separate workers for each of the available
snapshotters: containerd-overlay, containerd-btrfs, ... However, only
the default one is used, unless `containerd-worker-multiple-snapshotter`
is specified.
- The default worker is "runc-overlay"

Signed-off-by: Akihiro Suda <[email protected]>

Makefile

@@ -1,5 +1,5 @@
 
-BINARIES=bin/buildd-standalone bin/buildd-containerd bin/buildctl	bin/buildctl-darwin bin/buildd.exe bin/buildctl.exe
+BINARIES=bin/buildd bin/buildd-standalone bin/buildd-containerd bin/buildctl	bin/buildctl-darwin bin/buildd.exe bin/buildctl.exe
 
 binaries: $(BINARIES)
 

cmd/buildd/main.go

@@ -5,13 +5,19 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/containerd/containerd/sys"
 	"github.com/docker/go-connections/sockets"
+	"github.com/moby/buildkit/control"
+	"github.com/moby/buildkit/frontend"
+	"github.com/moby/buildkit/frontend/dockerfile"
+	"github.com/moby/buildkit/frontend/gateway"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/moby/buildkit/util/appdefaults"
 	"github.com/moby/buildkit/util/profiler"
+	"github.com/moby/buildkit/worker"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -20,6 +26,12 @@ import (
 	"google.golang.org/grpc"
 )
 
+var (
+	appFlags []cli.Flag
+	// key: priority (+: less preferred, -: more preferred)
+	workerCtors = make(map[int]func(c *cli.Context, root string) ([]*worker.Worker, error), 0)
+)
+
 func main() {
 	app := cli.NewApp()
 	app.Name = "buildd"
@@ -47,7 +59,7 @@ func main() {
 		},
 	}
 
-	app.Flags = appendFlags(app.Flags)
+	app.Flags = append(app.Flags, appFlags...)
 
 	app.Action = func(c *cli.Context) error {
 		ctx, cancel := context.WithCancel(appcontext.Context())
@@ -179,3 +191,53 @@ func unaryInterceptor(globalCtx context.Context) grpc.ServerOption {
 		return
 	})
 }
+
+func newController(c *cli.Context, root string) (*control.Controller, error) {
+	wc, err := workerController(c, root)
+	if err != nil {
+		return nil, err
+	}
+	frontends := map[string]frontend.Frontend{}
+	frontends["dockerfile.v0"] = dockerfile.NewDockerfileFrontend()
+	frontends["gateway.v0"] = gateway.NewGatewayFrontend()
+	return control.NewController(control.Opt{
+		WorkerController: wc,
+		Frontends:        frontends,
+	})
+}
+
+func workerController(c *cli.Context, root string) (*worker.Controller, error) {
+	wc := &worker.Controller{}
+	type ctorEntry struct {
+		priority int
+		ctor     func(c *cli.Context, root string) ([]*worker.Worker, error)
+	}
+	var ctors []ctorEntry
+	for p, ctor := range workerCtors {
+		ctors = append(ctors, ctorEntry{priority: p, ctor: ctor})
+	}
+	sort.Slice(ctors, func(i, j int) bool { return ctors[i].priority < ctors[j].priority })
+	for _, e := range ctors {
+		ws, err := e.ctor(c, root)
+		if err != nil {
+			return nil, err
+		}
+		for _, w := range ws {
+			logrus.Infof("Found worker %q", w.Name)
+			if err = wc.Add(w); err != nil {
+				return nil, err
+			}
+		}
+	}
+	nWorkers := len(wc.GetAll())
+	if nWorkers == 0 {
+		return nil, errors.New("no worker found, build the buildkit daemon with tags? (e.g. \"standalone\", \"containerd\")")
+	}
+	defaultWorker, err := wc.GetDefault()
+	if err != nil {
+		return nil, err
+	}
+	logrus.Infof("Found %d workers, default=%q", nWorkers, defaultWorker.Name)
+	logrus.Warn("Currently, only the default worker can be used.")
+	return wc, nil
+}

cmd/buildd/main_containerd.go

@@ -1,25 +1,73 @@
-// +build containerd,!standalone
+// +build containerd
 
 package main
 
 import (
-	"github.com/moby/buildkit/control"
+	"os"
+	"strings"
+
+	"github.com/moby/buildkit/worker"
+	"github.com/moby/buildkit/worker/containerd"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-func appendFlags(f []cli.Flag) []cli.Flag {
-	return append(f, []cli.Flag{
+func init() {
+	appFlags = append(appFlags,
+		cli.StringFlag{
+			Name:  "containerd-worker",
+			Usage: "enable containerd workers (true/false/auto)",
+		},
+		cli.BoolFlag{
+			Name:  "containerd-worker-multiple-snapshotters",
+			Usage: "use multiple snapshotters. useful for some applications that does not work with the default snapshotter (overlay)",
+		},
 		cli.StringFlag{
-			Name:  "containerd",
+			Name:  "containerd-worker-addr",
 			Usage: "containerd socket",
 			Value: "/run/containerd/containerd.sock",
-		},
-	}...)
+		})
+	// 1 is less preferred than 0 (runcCtor)
+	workerCtors[1] = containerdCtor
 }
 
-// root must be an absolute path
-func newController(c *cli.Context, root string) (*control.Controller, error) {
-	socket := c.GlobalString("containerd")
+func containerdCtor(c *cli.Context, root string) ([]*worker.Worker, error) {
+	socket := c.GlobalString("containerd-worker-addr")
+	boolOrAuto, err := parseBoolOrAuto(c.GlobalString("containerd-worker"))
+	if err != nil {
+		return nil, err
+	}
+	if (boolOrAuto == nil && skipContainerd(socket)) || (boolOrAuto != nil && !*boolOrAuto) {
+		return nil, nil
+	}
+	opts, err := containerd.NewWorkerOpts(root, socket)
+	if err != nil {
+		return nil, err
+	}
+	if !c.GlobalBool("containerd-worker-multiple-snapshotters") {
+		opts = opts[0:1]
+	}
+	var ws []*worker.Worker
+	for _, opt := range opts {
+		w, err := worker.NewWorker(opt)
+		if err != nil {
+			return ws, err
+		}
+		ws = append(ws, w)
+	}
+	return ws, nil
+}
 
-	return control.NewContainerd(root, socket)
+func skipContainerd(socket string) bool {
+	if strings.HasPrefix(socket, "tcp://") {
+		// FIXME(AkihiroSuda): prohibit tcp?
+		return false
+	}
+	socketPath := strings.TrimPrefix(socket, "unix://")
+	if _, err := os.Stat(socketPath); os.IsNotExist(err) {
+		// FIXME(AkihiroSuda): add more conditions
+		logrus.Warnf("skipping containerd worker, as %q does not exist", socketPath)
+		return true
+	}
+	return false
 }

cmd/buildd/main_standalone.go

@@ -1,17 +1,57 @@
-// +build standalone,!containerd
+// +build standalone
+
+// TODO(AkihiroSuda): s/standalone/oci/g
 
 package main
 
 import (
-	"github.com/moby/buildkit/control"
+	"os/exec"
+
+	"github.com/moby/buildkit/worker"
+	"github.com/moby/buildkit/worker/runc"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-func appendFlags(f []cli.Flag) []cli.Flag {
-	return f
+func init() {
+	appFlags = append(appFlags,
+		cli.StringFlag{
+			Name:  "oci-worker",
+			Usage: "enable oci workers (true/false/auto)",
+		},
+	)
+	// TODO: allow multiple oci runtimes and snapshotters
+	workerCtors[0] = runcCtor
+}
+
+func runcCtor(c *cli.Context, root string) ([]*worker.Worker, error) {
+	boolOrAuto, err := parseBoolOrAuto(c.GlobalString("oci-worker"))
+	if err != nil {
+		return nil, err
+	}
+	if (boolOrAuto == nil && skipRunc()) || (boolOrAuto != nil && !*boolOrAuto) {
+		return nil, nil
+	}
+	opts, err := runc.NewWorkerOpts(root)
+	if err != nil {
+		return nil, err
+	}
+	var ws []*worker.Worker
+	for _, opt := range opts {
+		w, err := worker.NewWorker(opt)
+		if err != nil {
+			return ws, err
+		}
+		ws = append(ws, w)
+	}
+	return ws, nil
 }
 
-// root must be an absolute path
-func newController(c *cli.Context, root string) (*control.Controller, error) {
-	return control.NewStandalone(root)
+func skipRunc() bool {
+	_, err := exec.LookPath("runc")
+	if err != nil {
+		logrus.Warnf("skipping oci worker, as runc does not exist")
+		return true
+	}
+	return false
 }

cmd/buildd/util.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"strconv"
+	"strings"
+)
+
+// parseBoolOrAuto returns (nil, nil) if s is "auto"
+func parseBoolOrAuto(s string) (*bool, error) {
+	if strings.ToLower(s) == "auto" {
+		return nil, nil
+	}
+	b, err := strconv.ParseBool(s)
+	return &b, err
+}