Merge pull request #107 from andrewsykim/external-snapshotter-leader-…

…election

Add Leader Election Support

Gopkg.toml

@@ -11,8 +11,7 @@ required = [
 
 [[constraint]]
   name = "github.com/container-storage-interface/spec"
-  version = "=1.0.0" # Set this to exactly 1.0.0 until we can update test code to handle 1.1.0
-
+  version = "=1.1.0"
 
 # The dependency on external-provisioner should be removed with #60.
 [[constraint]]
@@ -37,7 +36,7 @@ required = [
 
 [[constraint]]
   name = "github.com/kubernetes-csi/csi-lib-utils"
-  version = ">=0.4.0"
+  version = ">=v0.6.1"
 
 [prune]
   non-go = true

README.md

@@ -48,6 +48,13 @@ It is necessary to create a new service account and give it enough privileges to
 for i in $(find deploy/kubernetes -name '*.yaml'); do kubectl create -f $i; done
 ```
 
+### Running with Leader Election
+
+If you want to run external-snapshotter with higher availability, you can enable resource based leader election. To enable this, set the following flags:
+```bash
+--leader-election=true
+```
+
 ## Testing
 
 Running Unit Tests:

cmd/csi-snapshotter/main.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/kubernetes-csi/csi-lib-utils/connection"
+	"github.com/kubernetes-csi/csi-lib-utils/leaderelection"
 	csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc"
 	"github.com/kubernetes-csi/external-snapshotter/pkg/controller"
 	"github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter"
@@ -65,10 +66,14 @@ var (
 	snapshotNamePrefix              = flag.String("snapshot-name-prefix", "snapshot", "Prefix to apply to the name of a created snapshot")
 	snapshotNameUUIDLength          = flag.Int("snapshot-name-uuid-length", -1, "Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate.")
 	showVersion                     = flag.Bool("version", false, "Show version.")
+
+	leaderElection          = flag.Bool("leader-election", false, "Enables leader election.")
+	leaderElectionNamespace = flag.String("leader-election-namespace", "", "The namespace where the leader election resource exists. Defaults to the pod namespace if not set.")
 )
 
 var (
-	version = "unknown"
+	version                = "unknown"
+	leaderElectionLockName = "external-snapshotter-leader-election"
 )
 
 func main() {
@@ -192,17 +197,31 @@ func main() {
 		*snapshotNameUUIDLength,
 	)
 
-	// run...
-	stopCh := make(chan struct{})
-	factory.Start(stopCh)
-	coreFactory.Start(stopCh)
-	go ctrl.Run(threads, stopCh)
-
-	// ...until SIGINT
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt)
-	<-c
-	close(stopCh)
+	run := func(context.Context) {
+		// run...
+		stopCh := make(chan struct{})
+		factory.Start(stopCh)
+		coreFactory.Start(stopCh)
+		go ctrl.Run(threads, stopCh)
+
+		// ...until SIGINT
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, os.Interrupt)
+		<-c
+		close(stopCh)
+	}
+
+	if !*leaderElection {
+		run(context.TODO())
+	} else {
+		le := leaderelection.NewLeaderElection(kubeClient, leaderElectionLockName, run)
+		if *leaderElectionNamespace != "" {
+			le.WithNamespace(*leaderElectionNamespace)
+		}
+		if err := le.Run(); err != nil {
+			klog.Fatalf("failed to initialize leader election: %v", err)
+		}
+	}
 }
 
 func buildConfig(kubeconfig string) (*rest.Config, error) {

deploy/kubernetes/rbac-external-provisioner.yaml

@@ -59,7 +59,7 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
 
 ---
-# Provisioner must be able to work with endpoints in current namespace
+# Provisioner must be able to work with endpoints and leases in current namespace
 # if (and only if) leadership election is enabled
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
@@ -71,6 +71,9 @@ rules:
 - apiGroups: [""]
   resources: ["endpoints"]
   verbs: ["get", "watch", "list", "delete", "update", "create"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
 
 ---
 kind: RoleBinding

deploy/kubernetes/rbac.yaml

@@ -63,3 +63,30 @@ roleRef:
   # change the name also here if the ClusterRole gets renamed
   name: external-snapshotter-runner
   apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: default # TODO: replace with the namespace you want for your sidecar
+  name: external-snapshotter-leaderelection
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: external-snapshotter-leaderelection
+  namespace: default # TODO: replace with the namespace you want for your sidecar
+subjects:
+  - kind: ServiceAccount
+    name: csi-snapshotter
+    namespace: default # TODO: replace with the namespace you want for your sidecar
+roleRef:
+  kind: Role
+  name: external-snapshotter-leaderelection
+  apiGroup: rbac.authorization.k8s.io
+

deploy/kubernetes/setup-csi-snapshotter.yaml

@@ -89,6 +89,7 @@ spec:
           args:
             - "--csi-address=$(ADDRESS)"
             - "--connection-timeout=15s"
+            - "--leader-election=false"
           env:
             - name: ADDRESS
               value: /csi/csi.sock