From d062ee84793337386c5c07b13886a0a85df63678 Mon Sep 17 00:00:00 2001 From: Matt Schallert Date: Thu, 25 Feb 2021 20:19:16 +0000 Subject: [PATCH 1/2] [ci] Update kind; fix node image Also make test manifests work better out of the box with kind. --- example/etcd/etcd-kind.yaml | 103 +++++++++++++++++++++++++++++++++ example/m3db-kind.yaml | 50 ++++++++++++++++ integration/harness/etcd.go | 2 +- scripts/kind-create-cluster.sh | 28 ++++++--- 4 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 example/etcd/etcd-kind.yaml create mode 100644 example/m3db-kind.yaml diff --git a/example/etcd/etcd-kind.yaml b/example/etcd/etcd-kind.yaml new file mode 100644 index 00000000..4fdb2985 --- /dev/null +++ b/example/etcd/etcd-kind.yaml @@ -0,0 +1,103 @@ +apiVersion: v1 +kind: Service +metadata: + name: etcd + labels: + app: etcd +spec: + ports: + - port: 2379 + name: client + - port: 2380 + name: peer + clusterIP: None + selector: + app: etcd +--- +apiVersion: v1 +kind: Service +metadata: + name: etcd-cluster + labels: + app: etcd +spec: + selector: + app: etcd + ports: + - port: 2379 + protocol: TCP + type: ClusterIP +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: etcd + labels: + app: etcd +spec: + serviceName: "etcd" + replicas: 3 + selector: + matchLabels: + app: etcd + template: + metadata: + labels: + app: etcd + spec: + containers: + - name: etcd + # 3.3.18 + image: gcr.io/etcd-development/etcd@sha256:e80582131febde2862a530660cb74c6b1cb2e1d2b443e6c563139cedc636f2e7 + imagePullPolicy: IfNotPresent + command: + - "etcd" + - "--name" + - "$(MY_POD_NAME)" + - "--listen-peer-urls" + - "http://$(MY_IP):2380" + - "--listen-client-urls" + - "http://$(MY_IP):2379,http://127.0.0.1:2379" + - "--advertise-client-urls" + - "http://$(MY_POD_NAME).etcd:2379" + - "--initial-cluster-token" + - "etcd-cluster-1" + - "--initial-advertise-peer-urls" + - "http://$(MY_POD_NAME).etcd:2380" + - "--initial-cluster" + - "etcd-0=http://etcd-0.etcd:2380,etcd-1=http://etcd-1.etcd:2380,etcd-2=http://etcd-2.etcd:2380" + - "--initial-cluster-state" + - "new" + - "--data-dir" + - "/var/lib/etcd" + ports: + - containerPort: 2379 + name: client + - containerPort: 2380 + name: peer + volumeMounts: + - name: etcd-data + mountPath: /var/lib/etcd + env: + - name: MY_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: ETCDCTL_API + value: "3" + # Kind has a default storageclass. No need to set storageClassName. + volumeClaimTemplates: + - metadata: + name: etcd-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi + limits: + storage: 4Gi diff --git a/example/m3db-kind.yaml b/example/m3db-kind.yaml new file mode 100644 index 00000000..4c5fc2bb --- /dev/null +++ b/example/m3db-kind.yaml @@ -0,0 +1,50 @@ +apiVersion: operator.m3db.io/v1alpha1 +kind: M3DBCluster +metadata: + name: m3db-cluster +spec: + image: quay.io/m3db/m3dbnode:latest + replicationFactor: 3 + numberOfShards: 256 + isolationGroups: + - name: group1 + numInstances: 1 + nodeAffinityTerms: + - key: failure-domain.beta.kubernetes.io/zone + values: + - us-east1-b + - name: group2 + numInstances: 1 + nodeAffinityTerms: + - key: failure-domain.beta.kubernetes.io/zone + values: + - us-east1-c + - name: group3 + numInstances: 1 + nodeAffinityTerms: + - key: failure-domain.beta.kubernetes.io/zone + values: + - us-east1-d + namespaces: + - name: metrics-10s:2d + preset: 10s:2d + etcdEndpoints: + - http://etcd-0.etcd:2379 + - http://etcd-1.etcd:2379 + - http://etcd-2.etcd:2379 + containerResources: + requests: + memory: 4Gi + cpu: '1' + limits: + memory: 4Gi + # Kind has a default storageclass. No need to set storageClassName. + dataDirVolumeClaimTemplate: + metadata: + name: m3db-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi diff --git a/integration/harness/etcd.go b/integration/harness/etcd.go index 3db27919..7ec5a141 100644 --- a/integration/harness/etcd.go +++ b/integration/harness/etcd.go @@ -46,7 +46,7 @@ const ( ) func (h *Harness) createEtcdCluster() error { - f, err := os.Open("../manifests/etcd.yaml") + f, err := os.Open("../manifests/etcd-kind.yaml") if err != nil { return err } diff --git a/scripts/kind-create-cluster.sh b/scripts/kind-create-cluster.sh index 2fa5980a..70261e9b 100755 --- a/scripts/kind-create-cluster.sh +++ b/scripts/kind-create-cluster.sh @@ -4,22 +4,30 @@ set -exo pipefail echo "--- :kubernetes: Installing kind" -KUBE_VERSION=${KUBE_VERSION:-v1.15.7} +KUBE_VERSION=${KUBE_VERSION:-v1.16.15} +KIND_VERSION=${KIND_VERSION:-0.10.0} CLUSTER_NAME=kind L_UNAME=$(uname | tr "[:upper:]" "[:lower:]") mkdir -p "$HOME/bin" +export PATH="$HOME/bin:$PATH" + +# Use command -v so that user can have kind in PATH but outside of "$HOME/bin". +if [[ ! $(command -v kind) || "$BUILDKITE" == "true" ]]; then + curl -sL -o "$HOME/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-${L_UNAME}-amd64" + chmod +x "$HOME/bin/kind" +fi -if [[ ! -x "$HOME/bin/kind" || "$BUILDKITE" == "true" ]]; then - curl -sL -o "$HOME/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-${L_UNAME}-amd64" +if [[ "$(kind --version)" != "kind version ${KIND_VERSION}" ]]; then + echo "expected kind version ${KIND_VERSION}, got $(kind --version)" + exit 1 fi -if [[ ! -x "$HOME/bin/kubectl" || "$BUILDKITE" == "true" ]]; then +if [[ ! $(command -v kubectl) || "$BUILDKITE" == "true" ]]; then curl -sL -o "$HOME/bin/kubectl" "https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/${L_UNAME}/amd64/kubectl" + chmod +x "$HOME/bin/kubectl" fi -chmod +x "$HOME/bin/kind" "$HOME/bin/kubectl" -export PATH="$HOME/bin:$PATH" echo "--- :kubernetes: Deleting existing kind clusters" kind get clusters -q | while read -r CLUSTER; do @@ -28,6 +36,10 @@ done echo "--- :kubernetes: Creating kind cluster" +# NB(schallert): starting in 1.17, we'll want to move away from the deprecated +# "failure-domain" labels: +# https://kubernetes.io/docs/reference/labels-annotations-taints/#failure-domainbetakubernetesiozone. + cat > cluster.yaml < Date: Thu, 25 Feb 2021 20:28:49 +0000 Subject: [PATCH 2/2] fix e2e --- integration/harness/etcd.go | 2 +- integration/manifests/etcd.yaml | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/integration/harness/etcd.go b/integration/harness/etcd.go index 7ec5a141..3db27919 100644 --- a/integration/harness/etcd.go +++ b/integration/harness/etcd.go @@ -46,7 +46,7 @@ const ( ) func (h *Harness) createEtcdCluster() error { - f, err := os.Open("../manifests/etcd-kind.yaml") + f, err := os.Open("../manifests/etcd.yaml") if err != nil { return err } diff --git a/integration/manifests/etcd.yaml b/integration/manifests/etcd.yaml index bfdc4683..4fdb2985 100644 --- a/integration/manifests/etcd.yaml +++ b/integration/manifests/etcd.yaml @@ -47,8 +47,9 @@ spec: spec: containers: - name: etcd - image: quay.io/coreos/etcd:v3.3.3 - imagePullPolicy: Always + # 3.3.18 + image: gcr.io/etcd-development/etcd@sha256:e80582131febde2862a530660cb74c6b1cb2e1d2b443e6c563139cedc636f2e7 + imagePullPolicy: IfNotPresent command: - "etcd" - "--name" @@ -74,6 +75,9 @@ spec: name: client - containerPort: 2380 name: peer + volumeMounts: + - name: etcd-data + mountPath: /var/lib/etcd env: - name: MY_IP valueFrom: @@ -83,3 +87,17 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: ETCDCTL_API + value: "3" + # Kind has a default storageclass. No need to set storageClassName. + volumeClaimTemplates: + - metadata: + name: etcd-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 4Gi + limits: + storage: 4Gi