From 155095523d3e468ebc7c00dd3a9cf711aecfd0df Mon Sep 17 00:00:00 2001 From: Derek Perkins Date: Tue, 1 Jan 2019 16:16:24 -0700 Subject: [PATCH 1/2] helm: run orchestrator refresh before downtime end Signed-off-by: Derek Perkins --- helm/vitess/templates/_vttablet.tpl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/helm/vitess/templates/_vttablet.tpl b/helm/vitess/templates/_vttablet.tpl index d526642ddf9..4f00a785677 100644 --- a/helm/vitess/templates/_vttablet.tpl +++ b/helm/vitess/templates/_vttablet.tpl @@ -359,6 +359,9 @@ spec: /vt/bin/vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC PlannedReparentShard -keyspace_shard={{ $keyspace.name }}/{{ $shard.name }} -avoid_master=$current_alias {{ if $orc.enabled }} + # tell orchestrator to refresh its view of this tablet + wget -q -S -O - "http://orchestrator.{{ $namespace }}/api/refresh/$hostname.vttablet/3306" + # let orchestrator attempt recoveries now wget -q -S -O - "http://orchestrator.{{ $namespace }}/api/end-downtime/$hostname.vttablet/3306" {{ end }} @@ -379,11 +382,6 @@ spec: done -{{ if $orc.enabled }} - # tell orchestrator to refresh its view of this tablet - wget -q -S -O - "http://orchestrator.{{ $namespace }}/api/refresh/$hostname.vttablet/3306" -{{ end }} - # delete the current tablet from topology. Not strictly necessary, but helps to prevent # edge cases where there are two masters /vt/bin/vtctlclient ${VTCTL_EXTRA_FLAGS[@]} -server $VTCTLD_SVC DeleteTablet $current_alias From 71a2b0c644485649d43e5a5142e760aa1922b3c2 Mon Sep 17 00:00:00 2001 From: Derek Perkins Date: Tue, 1 Jan 2019 16:30:57 -0700 Subject: [PATCH 2/2] helm: bump chart version to 1.0.4 Signed-off-by: Derek Perkins --- helm/release.sh | 2 +- helm/vitess/CHANGELOG.md | 9 +++++++++ helm/vitess/Chart.yaml | 2 +- helm/vitess/templates/_orchestrator.tpl | 4 ++-- helm/vitess/templates/_pmm.tpl | 2 +- helm/vitess/templates/_vttablet.tpl | 8 ++++---- helm/vitess/values.yaml | 10 +++++----- 7 files changed, 23 insertions(+), 14 deletions(-) diff --git a/helm/release.sh b/helm/release.sh index 7e0f3a1642a..a9f1a17d646 100755 --- a/helm/release.sh +++ b/helm/release.sh @@ -1,6 +1,6 @@ #!/bin/bash -version_tag=1.0.3 +version_tag=1.0.4 docker pull vitess/k8s:latest docker tag vitess/k8s:latest vitess/k8s:helm-$version_tag diff --git a/helm/vitess/CHANGELOG.md b/helm/vitess/CHANGELOG.md index 992437e922f..aca78299575 100644 --- a/helm/vitess/CHANGELOG.md +++ b/helm/vitess/CHANGELOG.md @@ -1,3 +1,12 @@ +## 1.0.4 - 2019-01-01 + +### Changes +* Use the [Orchestrator API](https://github.com/github/orchestrator/blob/master/docs/using-the-web-api.md) +to call `begin-downtime` before running `PlannedReparentShard` in the `preStopHook`, to make sure that Orchestrator +doesn't try to run an external failover while Vitess is reparenting. When it is complete, it calls `end-downtime`. +Also call `forget` on the instance after calling `vtctlclient DeleteTablet`. It will be rediscovered if/when +the tablet comes back up. This eliminates most possible race conditions that could cause split brain. + ## 1.0.3 - 2018-12-20 ### Changes diff --git a/helm/vitess/Chart.yaml b/helm/vitess/Chart.yaml index b65208e7bf0..021aa8a39bd 100644 --- a/helm/vitess/Chart.yaml +++ b/helm/vitess/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: vitess -version: 1.0.3 +version: 1.0.4 description: Single-Chart Vitess Cluster keywords: - vitess diff --git a/helm/vitess/templates/_orchestrator.tpl b/helm/vitess/templates/_orchestrator.tpl index d0babc41a59..99f88de1e21 100644 --- a/helm/vitess/templates/_orchestrator.tpl +++ b/helm/vitess/templates/_orchestrator.tpl @@ -123,7 +123,7 @@ spec: value: "15999" - name: recovery-log - image: vitess/logtail:helm-1.0.3 + image: vitess/logtail:helm-1.0.4 imagePullPolicy: IfNotPresent env: - name: TAIL_FILEPATH @@ -133,7 +133,7 @@ spec: mountPath: /tmp - name: audit-log - image: vitess/logtail:helm-1.0.3 + image: vitess/logtail:helm-1.0.4 imagePullPolicy: IfNotPresent env: - name: TAIL_FILEPATH diff --git a/helm/vitess/templates/_pmm.tpl b/helm/vitess/templates/_pmm.tpl index 9ff20aa94a5..32e8d70fce8 100644 --- a/helm/vitess/templates/_pmm.tpl +++ b/helm/vitess/templates/_pmm.tpl @@ -218,7 +218,7 @@ spec: trap : TERM INT; sleep infinity & wait - name: pmm-client-metrics-log - image: vitess/logtail:helm-1.0.3 + image: vitess/logtail:helm-1.0.4 imagePullPolicy: IfNotPresent env: - name: TAIL_FILEPATH diff --git a/helm/vitess/templates/_vttablet.tpl b/helm/vitess/templates/_vttablet.tpl index 4f00a785677..636393efa34 100644 --- a/helm/vitess/templates/_vttablet.tpl +++ b/helm/vitess/templates/_vttablet.tpl @@ -533,7 +533,7 @@ spec: {{ define "cont-logrotate" }} - name: logrotate - image: vitess/logrotate:helm-1.0.3 + image: vitess/logrotate:helm-1.0.4 imagePullPolicy: IfNotPresent volumeMounts: - name: vtdataroot @@ -547,7 +547,7 @@ spec: {{ define "cont-mysql-errorlog" }} - name: error-log - image: vitess/logtail:helm-1.0.3 + image: vitess/logtail:helm-1.0.4 imagePullPolicy: IfNotPresent env: @@ -565,7 +565,7 @@ spec: {{ define "cont-mysql-slowlog" }} - name: slow-log - image: vitess/logtail:helm-1.0.3 + image: vitess/logtail:helm-1.0.4 imagePullPolicy: IfNotPresent env: @@ -583,7 +583,7 @@ spec: {{ define "cont-mysql-generallog" }} - name: general-log - image: vitess/logtail:helm-1.0.3 + image: vitess/logtail:helm-1.0.4 imagePullPolicy: IfNotPresent env: diff --git a/helm/vitess/values.yaml b/helm/vitess/values.yaml index 03d200e2a71..9d9c9a371c9 100644 --- a/helm/vitess/values.yaml +++ b/helm/vitess/values.yaml @@ -177,7 +177,7 @@ etcd: # Default values for vtctld resources defined in 'topology' vtctld: serviceType: ClusterIP - vitessTag: helm-1.0.3 + vitessTag: helm-1.0.4 resources: # requests: # cpu: 100m @@ -188,7 +188,7 @@ vtctld: # Default values for vtgate resources defined in 'topology' vtgate: serviceType: ClusterIP - vitessTag: helm-1.0.3 + vitessTag: helm-1.0.4 resources: # requests: # cpu: 500m @@ -207,13 +207,13 @@ vtgate: # Default values for vtctlclient resources defined in 'topology' vtctlclient: - vitessTag: helm-1.0.3 + vitessTag: helm-1.0.4 extraFlags: {} secrets: [] # secrets are mounted under /vt/usersecrets/{secretname} # Default values for vtworker resources defined in 'jobs' vtworker: - vitessTag: helm-1.0.3 + vitessTag: helm-1.0.4 extraFlags: {} resources: # requests: @@ -224,7 +224,7 @@ vtworker: # Default values for vttablet resources defined in 'topology' vttablet: - vitessTag: helm-1.0.3 + vitessTag: helm-1.0.4 # valid values are # - mysql56 (for MySQL 8.0)