add example/kube-consistent-hash

examples/kube-consistent-hash/README.md

@@ -0,0 +1,26 @@
+# Distributed Build with Consistent Hashing
+
+Demo for efficiently using BuildKit daemon-local cache with multi-node cluster
+
+## Deploy
+
+```console
+$ kubectl apply -f buildkitd-rootless-statefulset.yaml
+$ kubectl scale --replicas=10 statefulset/buildkitd
+```
+
+## Consistent hashing
+
+Define the key string for consistent hashing.
+
+For example, the key can be defined as `<REPO NAME>:<CONTEXT PATH>`, e.g.
+`github.com/example/project:some/directory`.
+
+
+Then determine the pod that corresponds to the key:
+```console
+$ go get ./consistenthash
+$ pod=$(./show-running-pods.sh | consistenthash $key)
+```
+
+You can connect to the pod using `export BUILDKIT_HOST=kube-pod://$pod`.

examples/kube-consistent-hash/buildkitd-rootless-statefulset.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app: buildkitd
+  name: buildkitd
+spec:
+  serviceName: buildkitd
+  replicas: 1
+  selector:
+    matchLabels:
+      app: buildkitd
+  template:
+    metadata:
+      labels:
+        app: buildkitd
+      annotations:
+        container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined
+        container.seccomp.security.alpha.kubernetes.io/buildkitd: unconfined
+# see buildkit/docs/rootless.md for caveats of rootless mode
+    spec:
+      containers:
+      - name: buildkitd
+        image: moby/buildkit:master-rootless
+        args:
+        - --addr
+        - unix:///run/user/1000/buildkit/buildkitd.sock
+        - --addr
+        - tcp://0.0.0.0:1234
+        - --oci-worker-no-process-sandbox
+        ports:
+        - containerPort: 1234
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: buildkitd
+  name: buildkitd
+spec:
+  ports:
+  - port: 1234
+    protocol: TCP
+  selector:
+    app: buildkitd 

examples/kube-consistent-hash/consistenthash/main.go

@@ -0,0 +1,67 @@
+// Package main provides simple consistenthash commandline utility.
+/*
+Demo:
+
+  $ seq -f node-%.f 1 100 > /tmp/nodes
+  $ consistenthash < /tmp/nodes apple
+  node-42
+  $ consistenthash < /tmp/nodes banana
+  node-48
+  $ consistenthash < /tmp/nodes chocolate
+  node-2
+
+Modify /tmp/nodes and confirm that the result rarely changes.
+*/
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/serialx/hashring"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	if err := xmain(); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+func xmain() error {
+	if len(os.Args) != 2 {
+		fmt.Fprintf(os.Stderr, "Usage: %s KEY\nNode list needs to be provided via stdin\n", os.Args[0])
+		os.Exit(1)
+		return errors.New("should not reach here")
+	}
+	key := os.Args[1]
+	stdin, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		return err
+	}
+	var nodes []string
+	for _, s := range strings.Split(string(stdin), "\n") {
+		s = strings.TrimSpace(s)
+		if s != "" {
+			nodes = append(nodes, s)
+		}
+	}
+	chosen, err := doConsistentHash(nodes, key)
+	if err != nil {
+		return err
+	}
+	fmt.Println(chosen)
+	return nil
+}
+
+func doConsistentHash(nodes []string, key string) (string, error) {
+	ring := hashring.New(nodes)
+	x, ok := ring.GetNode(key)
+	if !ok {
+		return "", errors.Errorf("no node found for key %q", key)
+	}
+	return x, nil
+}

examples/kube-consistent-hash/show-running-pods.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -eu
+selector="app=buildkitd"
+kubectl get pods --selector=$selector --field-selector=status.phase=Running -o=go-template --template="{{range .items}}{{.metadata.name}}{{\"\n\"}}{{end}}" --sort-by="{.metadata.name}"

go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/opentracing/opentracing-go v0.0.0-20171003133519-1361b9cd60be
 	github.com/pkg/errors v0.8.1
 	github.com/pkg/profile v1.2.1
+	github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
 	github.com/sirupsen/logrus v1.3.0
 	github.com/stretchr/testify v1.3.0
 	github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 // indirect

go.sum

@@ -117,6 +117,8 @@ github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE=
 github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 h1:ka9QPuQg2u4LGipiZGsgkg3rJCo4iIUCy75FddM0GRQ=
+github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
 github.com/sirupsen/logrus v1.0.3 h1:B5C/igNWoiULof20pKfY4VntcIPqKuwEmoLZrabbUrc=
 github.com/sirupsen/logrus v1.0.3/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=