From cc7d39d79d394a26433de7a8295e3980541949ee Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Thu, 19 Mar 2020 15:27:13 -0700 Subject: [PATCH 1/7] tentative support for an external controlling coordinator --- pkg/apis/m3dboperator/v1alpha1/cluster.go | 9 +++++++++ pkg/k8sops/m3db/generators.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/pkg/apis/m3dboperator/v1alpha1/cluster.go b/pkg/apis/m3dboperator/v1alpha1/cluster.go index 303339bf..91eefaba 100644 --- a/pkg/apis/m3dboperator/v1alpha1/cluster.go +++ b/pkg/apis/m3dboperator/v1alpha1/cluster.go @@ -277,6 +277,15 @@ type ClusterSpec struct { // conjunction with HostNetwork.+optional // +optional DNSPolicy *corev1.DNSPolicy `json:"dnsPolicy,omitEmpty"` + + // Specify a "controlling" coordinator for the cluster + // It is expected that there is a separate standalone coordinator cluster + // It is externally managed - not managed by this operator + // It is expected to have a service endpoint + // Setup this db cluster, but do not assume a co-located coordinator + // Instead provide a selector here so we can point to a separate coordinator service + // Specify here the labels required for the selector + ExternalCoordinatorSelector map[string]string `json:"externalCoordinatorSelector,omitempty"` } // NodeAffinityTerm represents a node label and a set of label values, any of diff --git a/pkg/k8sops/m3db/generators.go b/pkg/k8sops/m3db/generators.go index c578ab1d..6afff29b 100644 --- a/pkg/k8sops/m3db/generators.go +++ b/pkg/k8sops/m3db/generators.go @@ -236,6 +236,9 @@ func GenerateCoordinatorService(cluster *myspec.M3DBCluster) (*v1.Service, error selectorLabels := labels.BaseLabels(cluster) selectorLabels[labels.Component] = labels.ComponentM3DBNode + if len(cluster.Spec.ExternalCoordinatorSelector) > 0 { + selectorLabels = cluster.Spec.ExternalCoordinatorSelector + } serviceLabels := labels.BaseLabels(cluster) serviceLabels[labels.Component] = labels.ComponentCoordinator From 43fc7214af1198da323cf5d7765a9fffc35f21f3 Mon Sep 17 00:00:00 2001 From: Matt Schallert Date: Tue, 7 Apr 2020 10:22:28 -0400 Subject: [PATCH 2/7] docs gen --- docs/api.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api.md b/docs/api.md index de116f9b..831950e2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -62,6 +62,7 @@ ClusterSpec defines the desired state for a M3 cluster to be converge to. | nodeEndpointFormat | NodeEndpointFormat allows overriding of the endpoint used for a node in the M3DB placement. Defaults to \"{{ .PodName }}.{{ .M3DBService }}:{{ .Port }}\". Useful if access to the cluster from other namespaces is desired. See \"Node Endpoint\" docs for full variables available. | string | false | | hostNetwork | HostNetwork indicates whether M3DB pods should run in the same network namespace as the node its on. This option should be used sparingly due to security concerns outlined in the linked documentation. https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces | bool | false | | dnsPolicy | DNSPolicy allows the user to set the pod's DNSPolicy. This is often used in conjunction with HostNetwork.+optional | *corev1.DNSPolicy | false | +| externalCoordinatorSelector | Specify a \"controlling\" coordinator for the cluster It is expected that there is a separate standalone coordinator cluster It is externally managed - not managed by this operator It is expected to have a service endpoint Setup this db cluster, but do not assume a co-located coordinator Instead provide a selector here so we can point to a separate coordinator service Specify here the labels required for the selector | map[string]string | false | [Back to TOC](#table-of-contents) From 5bd44d08efdf7b803be7272403e6b36e506b2cfd Mon Sep 17 00:00:00 2001 From: Matt Schallert Date: Tue, 7 Apr 2020 10:30:35 -0400 Subject: [PATCH 3/7] tests --- pkg/k8sops/m3db/generators_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/k8sops/m3db/generators_test.go b/pkg/k8sops/m3db/generators_test.go index 4203fd4b..d0ee6696 100644 --- a/pkg/k8sops/m3db/generators_test.go +++ b/pkg/k8sops/m3db/generators_test.go @@ -542,4 +542,10 @@ func TestGenerateCoordinatorService(t *testing.T) { } assert.Equal(t, expSvc, svc) + + cluster.Spec.ExternalCoordinatorSelector = map[string]string{"foo": "bar"} + expSvc.Spec.Selector = map[string]string{"foo": "bar"} + svc, err = GenerateCoordinatorService(cluster) + assert.NoError(t, err) + assert.Equal(t, expSvc, svc) } From 6939987fd4fcc99bb77a1734c24742ba6096a487 Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Tue, 7 Apr 2020 19:49:37 -0700 Subject: [PATCH 4/7] initcontainers start add volumes for the initcontainers add initContainer support for custom dbnode setup --- bundle.yaml | 90 ----------------------- pkg/apis/m3dboperator/v1alpha1/cluster.go | 9 +++ pkg/k8sops/m3db/generators.go | 10 +++ 3 files changed, 19 insertions(+), 90 deletions(-) delete mode 100644 bundle.yaml diff --git a/bundle.yaml b/bundle.yaml deleted file mode 100644 index 5f9a0c72..00000000 --- a/bundle.yaml +++ /dev/null @@ -1,90 +0,0 @@ ---- -# Source: m3db-operator/templates/service_account.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: m3db-operator - namespace: default ---- -# Source: m3db-operator/templates/cluster_role.yaml -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: m3db-operator -rules: -- apiGroups: ["extensions"] - resources: ["deployments", "replicasets", "daemonsets"] - verbs: ["create", "get", "update", "delete", "list"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "get", "update", "delete", "list"] -- apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "create", "delete", "deletecollection"] -- apiGroups: [""] - resources: ["persistentvolumes", "persistentvolumeclaims", "services", "secrets", "configmaps"] - verbs: ["create", "get", "update", "delete", "list"] -- apiGroups: ["batch"] - resources: ["cronjobs", "jobs"] - verbs: ["create", "get", "deletecollection", "delete"] -- apiGroups: [""] - resources: ["pods"] - verbs: ["list", "get", "watch", "update", "patch"] -- apiGroups: ["apps"] - resources: ["statefulsets", "deployments"] - verbs: ["*"] -- apiGroups: ["operator.m3db.io"] - resources: ["*"] - verbs: ["*"] -- apiGroups: [""] - resources: ["events"] - verbs: ["create", "patch"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] ---- -# Source: m3db-operator/templates/cluster_role_binding.yaml -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: m3db-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: m3db-operator -subjects: -- kind: ServiceAccount - name: m3db-operator - namespace: default ---- -# Source: m3db-operator/templates/stateful_set.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: m3db-operator - namespace: default -spec: - serviceName: m3db-operator - replicas: 1 - selector: - matchLabels: - name: m3db-operator - template: - metadata: - labels: - name: m3db-operator - spec: - securityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - containers: - - name: m3db-operator - image: quay.io/m3db/m3db-operator:v0.6.0 - command: - - m3db-operator - imagePullPolicy: Always - env: - - name: ENVIRONMENT - value: production - serviceAccount: m3db-operator diff --git a/pkg/apis/m3dboperator/v1alpha1/cluster.go b/pkg/apis/m3dboperator/v1alpha1/cluster.go index 91eefaba..66b044f5 100644 --- a/pkg/apis/m3dboperator/v1alpha1/cluster.go +++ b/pkg/apis/m3dboperator/v1alpha1/cluster.go @@ -286,6 +286,15 @@ type ClusterSpec struct { // Instead provide a selector here so we can point to a separate coordinator service // Specify here the labels required for the selector ExternalCoordinatorSelector map[string]string `json:"externalCoordinatorSelector,omitempty"` + + // Custom setup for db nodes can be done via initContainers + // Provide the complete spec for the initContainer here + // If any storage volumes are needed in the initContainer see InitVolumes below + InitContainers []corev1.Container `json:"initContainers,omitempty"` + + // If the InitContainers require any storage volumes + // Provide the complete specification for the required Volumes here + InitVolumes []corev1.Volume `json:"initVolumes,omitempty"` } // NodeAffinityTerm represents a node label and a set of label values, any of diff --git a/pkg/k8sops/m3db/generators.go b/pkg/k8sops/m3db/generators.go index 6afff29b..331e518e 100644 --- a/pkg/k8sops/m3db/generators.go +++ b/pkg/k8sops/m3db/generators.go @@ -195,6 +195,16 @@ func GenerateStatefulSet( m3dbContainer.Env = append(m3dbContainer.Env, cluster.Spec.EnvVars...) } + if cluster.Spec.InitContainers != nil && len(cluster.Spec.InitContainers) > 0 { + cluster := cluster.DeepCopy() + statefulSet.Spec.Template.Spec.InitContainers = append(statefulSet.Spec.Template.Spec.InitContainers, cluster.Spec.InitContainers...) + } + + if cluster.Spec.InitVolumes != nil && len(cluster.Spec.InitVolumes) > 0 { + cluster := cluster.DeepCopy() + statefulSet.Spec.Template.Spec.Volumes = append(statefulSet.Spec.Template.Spec.Volumes, cluster.Spec.InitVolumes...) + } + return statefulSet, nil } From 0f453c30df66a1b0d0568a40922c3cabc5e2fef7 Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Tue, 7 Apr 2020 20:08:45 -0700 Subject: [PATCH 5/7] restored a file bundle.yaml i accidentally deleted --- bundle.yaml | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 bundle.yaml diff --git a/bundle.yaml b/bundle.yaml new file mode 100644 index 00000000..5f9a0c72 --- /dev/null +++ b/bundle.yaml @@ -0,0 +1,90 @@ +--- +# Source: m3db-operator/templates/service_account.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: m3db-operator + namespace: default +--- +# Source: m3db-operator/templates/cluster_role.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: m3db-operator +rules: +- apiGroups: ["extensions"] + resources: ["deployments", "replicasets", "daemonsets"] + verbs: ["create", "get", "update", "delete", "list"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "update", "delete", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "create", "delete", "deletecollection"] +- apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims", "services", "secrets", "configmaps"] + verbs: ["create", "get", "update", "delete", "list"] +- apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["create", "get", "deletecollection", "delete"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "get", "watch", "update", "patch"] +- apiGroups: ["apps"] + resources: ["statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["operator.m3db.io"] + resources: ["*"] + verbs: ["*"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +# Source: m3db-operator/templates/cluster_role_binding.yaml +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: m3db-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: m3db-operator +subjects: +- kind: ServiceAccount + name: m3db-operator + namespace: default +--- +# Source: m3db-operator/templates/stateful_set.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: m3db-operator + namespace: default +spec: + serviceName: m3db-operator + replicas: 1 + selector: + matchLabels: + name: m3db-operator + template: + metadata: + labels: + name: m3db-operator + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + containers: + - name: m3db-operator + image: quay.io/m3db/m3db-operator:v0.6.0 + command: + - m3db-operator + imagePullPolicy: Always + env: + - name: ENVIRONMENT + value: production + serviceAccount: m3db-operator From 6fb9d66f4959e60330f6d47582bbbd4cd290a7f0 Mon Sep 17 00:00:00 2001 From: Matt Schallert Date: Wed, 8 Apr 2020 15:47:16 -0400 Subject: [PATCH 6/7] gen docs --- docs/api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api.md b/docs/api.md index 831950e2..08251968 100644 --- a/docs/api.md +++ b/docs/api.md @@ -63,6 +63,8 @@ ClusterSpec defines the desired state for a M3 cluster to be converge to. | hostNetwork | HostNetwork indicates whether M3DB pods should run in the same network namespace as the node its on. This option should be used sparingly due to security concerns outlined in the linked documentation. https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces | bool | false | | dnsPolicy | DNSPolicy allows the user to set the pod's DNSPolicy. This is often used in conjunction with HostNetwork.+optional | *corev1.DNSPolicy | false | | externalCoordinatorSelector | Specify a \"controlling\" coordinator for the cluster It is expected that there is a separate standalone coordinator cluster It is externally managed - not managed by this operator It is expected to have a service endpoint Setup this db cluster, but do not assume a co-located coordinator Instead provide a selector here so we can point to a separate coordinator service Specify here the labels required for the selector | map[string]string | false | +| initContainers | Custom setup for db nodes can be done via initContainers Provide the complete spec for the initContainer here If any storage volumes are needed in the initContainer see InitVolumes below | []corev1.Container | false | +| initVolumes | If the InitContainers require any storage volumes Provide the complete specification for the required Volumes here | []corev1.Volume | false | [Back to TOC](#table-of-contents) From d9a7f22d13719bae90c15651c6ecdb4465af7e72 Mon Sep 17 00:00:00 2001 From: Matt Schallert Date: Wed, 8 Apr 2020 15:52:33 -0400 Subject: [PATCH 7/7] tests --- pkg/k8sops/m3db/generators_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pkg/k8sops/m3db/generators_test.go b/pkg/k8sops/m3db/generators_test.go index d0ee6696..771652db 100644 --- a/pkg/k8sops/m3db/generators_test.go +++ b/pkg/k8sops/m3db/generators_test.go @@ -466,6 +466,28 @@ func TestGenerateStatefulSet(t *testing.T) { diff, _ := messagediff.PrettyDiff(ss, newSS) t.Log(diff) } + + // Test init containers + fixture = getFixture("testM3DBCluster.yaml", t) + fixture.Spec.InitContainers = []v1.Container{ + { + Name: "init0", + }, + } + + ss = baseSS.DeepCopy() + ss.Spec.Template.Spec.InitContainers = []v1.Container{ + { + Name: "init0", + }, + } + newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount) + assert.NoError(t, err) + assert.NotNil(t, newSS) + if !assert.Equal(t, ss, newSS) { + diff, _ := messagediff.PrettyDiff(ss, newSS) + t.Log(diff) + } } func TestGenerateM3DBService(t *testing.T) {