-
Notifications
You must be signed in to change notification settings - Fork 65
470 lines (426 loc) · 20.8 KB
/
deploy-hubs.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
# The use of ::set-output in this workflow file was deprecated as per:
# https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/
# We have instead pivoted to writing to the GITHUB_ENV file, either by the
# method suggested in the above blog post, or by following the below Stack
# Overflow answer for python code: https://stackoverflow.com/a/70123641
# More info on GITHUB_ENV: https://docs.github.com/en/actions/learn-github-actions/environment-variables
name: Deploy and test hubs
on:
push:
branches:
- main
paths:
# Include changes to the deployer script's folder, but exclude some parts
- deployer/**
- "!deployer/README.md"
- "!deployer/health_check_tests/**"
- "!deployer/commands/generate/billing/**"
- "!deployer/commands/generate/dedicated_cluster/**"
- "!deployer/commands/generate/hub_asset/**"
- "!deployer/commands/generate/resource_allocation/**"
- requirements.txt
- .github/actions/setup-deploy/**
- helm-charts/**
- config/clusters/**
# Exclude the template configuration files
- "!config/clusters/templates/**"
- "!terraform/aws/projects/template.tfvars"
- "!terraform/gcp/projects/cluster.tfvars.template"
- "!eksctl/template.jsonnet"
pull_request:
branches:
- main
paths:
# Include changes to the deployer script's folder, but exclude some parts
- deployer/**
- "!deployer/README.md"
- "!deployer/health_check_tests/**"
- "!deployer/commands/generate/billing/**"
- "!deployer/commands/generate/dedicated_cluster/**"
- "!deployer/commands/generate/hub_asset/**"
- "!deployer/commands/generate/resource_allocation/**"
- requirements.txt
- .github/actions/setup-deploy/**
- helm-charts/**
- config/clusters/**
# Exclude the template configuration files
- "!config/clusters/templates/**"
- "!terraform/aws/projects/template.tfvars"
- "!terraform/gcp/projects/cluster.tfvars.template"
- "!eksctl/template.jsonnet"
# Queue triggered executions of this workflow stemming from pushes to avoid
# deployment conflicts.
#
# By using a different concurrency groups for pull requests and pushes, we
# reduce the risk of cancelling a queued but not started workflow as discussed
# in https://github.com/2i2c-org/infrastructure/issues/3214.
#
# github.head_ref is used to create PR unique concurrency groups, and for
# workflow executions not triggered by a PR we get a dedicated group.
#
# ref: https://docs.github.com/en/actions/using-jobs/using-concurrency
#
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || 'not-a-pr' }}
cancel-in-progress: false
# This environment variable triggers the deployer to colourise print statements in the
# GitHug Actions logs for easy reading
env:
TERM: xterm
# This may not be required any more, and it may depend on the kubectl version
# we use etc as well. For now, we have it added to avoid issues.
USE_GKE_GCLOUD_AUTH_PLUGIN: "True"
jobs:
# This job runs in Pull Requests and on pushes to the default branch. It identifies
# which files have been added or modified by recent GitHub activity and parsed a list
# to the `deployer generate helm-upgrade-job`s command of the deployer. This command generates
# two lists of dictionaries, which can be read by GitHub Actions as matrix jobs. The
# first set of jobs describes which clusters need their support chart and/or staging
# hub upgraded; and the second set of jobs describe which production hubs require
# upgrading. These two lists are set as job outputs via GITHUB_ENV to be consumed
# by the later jobs. They are also pretty-printed in a human-readable format to the
# logs, and converted into Markdown tables for posting into GitHub comments.
generate-jobs:
runs-on: ubuntu-latest
outputs:
support-and-staging-matrix-jobs: ${{ env.support-and-staging-matrix-jobs }}
prod-hub-matrix-jobs: ${{ env.prod-hub-matrix-jobs }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.11"
# There will almost never be a cache hit on the cache key when this job is
# run, as it is the first of all jobs in this workflow. An exception is if
# this job is re-attempted as part of the same workflow run after
# succeeding previously.
- name: Save pip's install cache on job completion
uses: actions/cache@v4
with:
path: ~/.cache/pip
# key determines if we define or reuse an existing cache or not. Our
# key ensure we cache within a workflow run and its attempts, but not
# between workflow runs.
key: "${{ github.run_id }}"
- name: Install deployer script's Python dependencies
run: |
pip install --editable .
pip list
- name: Get merged/open PR labels
uses: actions/github-script@v7
id: pr-labels
with:
# Both pull_request and push can have triggered this job to run. A
# context with PR info (including its labels) is available when this
# job is triggered via pull_request, but not when triggered via push -
# a push can be triggered by other things than merged PRs. Due to
# this, we check if a pushed commit is matching some PRs merge commit,
# and if so gets its labels.
script: |
let labels = null;
if (context.eventName === 'pull_request') {
labels = context.payload.pull_request.labels;
}
else if (context.eventName === 'push') {
// api ref: https://octokit.github.io/rest.js/v20#pulls-list
const resp = await github.rest.pulls.list(
{
owner: context.repo.owner,
repo: context.repo.repo,
state: 'closed',
sort: 'updated',
direction: 'desc',
per_page: 100,
}
);
const merged_pr = resp.data.find(pr => pr.merge_commit_sha === context.sha);
labels = merged_pr?.labels;
}
label_names = (labels || []).map(l => l.name);
return label_names
- name: Identify files that have been added or modified
# Action repo: https://github.com/dorny/paths-filter
uses: dorny/paths-filter@v3
id: changed-files
with:
token: ""
list-files: csv
filters: |
changed:
- added|modified: deployer/**
- added|modified: requirements.txt
- added|modified: .github/actions/setup-deploy/**
- added|modified: helm-charts/basehub/**
- added|modified: helm-charts/daskhub/**
- added|modified: helm-charts/support/**
- added|modified: config/clusters/**
# This step will create a comment-body.txt file containing the jobs to be run in a
# Markdown table format to be posted on a Pull Request, if this job is triggered
# by one
- name: Generate matrix jobs
run: |
deployer generate helm-upgrade-jobs "${{ steps.changed-files.outputs.changed_files }}" '${{ steps.pr-labels.outputs.result }}'
# The comment-deployment-plan-pr.yaml workflow won't have the correct context to
# know the PR number, so we save it to a file to pass to that workflow
- name: Save Pull Request number to a file
if: github.event_name == 'pull_request'
run: |
echo "${{ github.event.number }}" > pr-number.txt
# Upload the pr-number.txt and comment-body.txt files as artifacts for the
# comment-deployment-plan-pr.yaml workflow to access
- name: Upload artifacts
if: >
github.event_name == 'pull_request' &&
(env.support-and-staging-matrix-jobs != '[]' ||
env.prod-hub-matrix-jobs != '[]')
uses: actions/upload-artifact@v4
with:
name: pr
path: |
pr-number.txt
comment-body.txt
# https://github.com/ravsamhq/notify-slack-action
# Needs to be added per job
# When https://github.com/integrations/slack/issues/1563 gets implemented,
# we can use that instead
- name: Report Status
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
# Warning: there are multiple "Report Status" steps in this file (one per job).
# Make sure they are all updated
notify_when: "failure"
status: ${{ job.status }} # required
# Message should look like: "Hey @author! Deploy and test hubs failed for "Add new hub - 9305e08".
message_format: '{emoji} Hey @${{ github.event.head_commit.author.name }}! *{workflow}* {status_message} for "${{ github.event.head_commit.message }} - <{commit_url}|{commit_sha}>". Checkout the run at {run_url}.'
footer: "<{run_url}|Failing Run>"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GHA_FAILURES_WEBHOOK_URL }}
# This job upgrades the support chart, staging hub, and dask-staging hub (if present)
# for clusters in parallel, if those upgrades are required. This job needs the
# `generate-jobs` job to have completed and set an output to the
# `support-and-staging-matrix-jobs` variable name. It's inputs are a list of
# dictionaries with the keys cluster_name, provider, upgrade_support, and
# upgrade_staging for each cluster that requires it.
upgrade-support-and-staging:
runs-on: ubuntu-latest
needs: [generate-jobs]
# We declare outputs indicating the job failed status of a specific job
# variation. We are currently required to do this in a hardcoded fashion,
# see this post for feature requests for this to be improved:
# https://github.community/t/bug-jobs-output-should-return-a-list-for-a-matrix-job/128626/32?u=consideratio
#
# Warning: names can include alphanumerics, '-', and '_', but not '.', so
# we replace '.' for '-' in cluster names.
#
# If you are adding a new cluster, please remember to list it here!
outputs:
failure_2i2c-aws-us: "${{ env.failure_2i2c-aws-us }}"
failure_2i2c-uk: "${{ env.failure_2i2c-uk }}"
failure_2i2c: "${{ env.failure_2i2c }}"
failure_awi-ciroh: "${{ env.failure_awi-ciroh }}"
failure_catalystproject-africa: "${{ env.failure_catalystproject-africa }}"
failure_catalystproject-latam: "${{ env.failure_catalystproject-latam }}"
failure_cloudbank: "${{ env.failure_cloudbank }}"
failure_dubois: "${{ env.failure_dubois }}"
failure_earthscope: "${{ env.failure_earthscope }}"
failure_gridsst: "${{ env.failure_gridsst }}"
failure_hhmi: "${{ env.failure_hhmi }}"
failure_jupyter-health: "${{ env.failure_jupyter-health }}"
failure_jupyter-meets-the-earth: "${{ env.failure_jupyter-meets-the-earth }}"
failure_kitware: "${{ env.failure_kitware }}"
failure_leap: "${{ env.failure_leap }}"
failure_nasa-cryo: "${{ env.failure_nasa-cryo }}"
failure_nasa-ghg: "${{ env.failure_nasa-ghg }}"
failure_nasa-veda: "${{ env.failure_nasa-veda }}"
failure_nmfs-openscapes: "${{ env.failure_nmfs-openscapes }}"
failure_openscapes: "${{ env.failure_openscapes }}"
failure_opensci: "${{ env.failure_opensci }}"
failure_pangeo-hubs: "${{ env.failure_pangeo-hubs }}"
failure_projectpythia: "${{ env.failure_projectpythia }}"
failure_queensu: "${{ env.failure_queensu }}"
failure_smithsonian: "${{ env.failure_smithsonian }}"
failure_strudel: "${{ env.failure_strudel }}"
failure_ubc-eoas: "${{ env.failure_ubc-eoas }}"
failure_utoronto: "${{ env.failure_utoronto }}"
failure_victor: "${{ env.failure_victor }}"
if: |
(github.event_name == 'push' && contains(github.ref, 'main')) &&
needs.generate-jobs.result == 'success' &&
needs.generate-jobs.outputs.support-and-staging-matrix-jobs != '[]'
strategy:
# Don't stop other deployments if one fails
fail-fast: false
matrix:
jobs: ${{ fromJson(needs.generate-jobs.outputs.support-and-staging-matrix-jobs) }}
steps:
- uses: actions/checkout@v4
- name: Setup deploy for ${{ matrix.jobs.cluster_name }}
uses: ./.github/actions/setup-deploy
with:
provider: ${{ matrix.jobs.provider }}
GCP_KMS_DECRYPTOR_KEY: ${{ secrets.GCP_KMS_DECRYPTOR_KEY }}
- name: Upgrade support chart on cluster ${{ matrix.jobs.cluster_name }}
if: matrix.jobs.upgrade_support
run: |
deployer deploy-support ${{ matrix.jobs.cluster_name }}
- name: Upgrade staging hub on cluster ${{ matrix.jobs.cluster_name }}
if: matrix.jobs.upgrade_staging
run: |
deployer deploy ${{ matrix.jobs.cluster_name }} staging
# Retry action: https://github.com/marketplace/actions/retry-step
- name: Run health check for staging hub on cluster ${{ matrix.jobs.cluster_name }}
if: matrix.jobs.upgrade_staging
uses: nick-fields/retry@v3
with:
timeout_minutes: 10
max_attempts: 2
command: |
deployer run-hub-health-check ${{ matrix.jobs.cluster_name }} staging
- name: Upgrade dask-staging hub on cluster ${{ matrix.jobs.cluster_name }} if it exists
if: matrix.jobs.upgrade_staging && matrix.jobs.cluster_name == '2i2c'
run: |
deployer deploy ${{ matrix.jobs.cluster_name }} dask-staging
# Retry action: https://github.com/marketplace/actions/retry-step
- name: Run health check for dask-staging hub on cluster ${{ matrix.jobs.cluster_name }} if it exists
if: matrix.jobs.upgrade_staging && matrix.jobs.cluster_name == '2i2c'
uses: nick-fields/retry@v3
with:
timeout_minutes: 10
max_attempts: 2
command: |
deployer run-hub-health-check ${{ matrix.jobs.cluster_name }} dask-staging
- name: Declare failure status
if: always()
shell: python
run: |
import os
name = "${{ matrix.jobs.cluster_name }}".replace(".", "-")
failure = "${{ job.status == 'failure' }}"
env_file = os.getenv("GITHUB_ENV")
with open(env_file, "a") as f:
f.write(f"failure_{name}={failure}")
# https://github.com/ravsamhq/notify-slack-action
# Needs to be added per job
# FIXME: when https://github.com/integrations/slack/issues/1563 gets implemented,
# we can use that instead
- name: Report Status
if: always()
uses: ravsamhq/notify-slack-action@v2
# Warning: there are multiple "Report Status" steps in this file (one per job).
# Make sure they are all updated
with:
notify_when: "failure"
status: ${{ job.status }} # required
# Message should look like: "Hey @author! Deploy and test hubs failed for "Add new hub - 9305e08".
message_format: '{emoji} Hey @${{ github.event.head_commit.author.name }}! *{workflow}* {status_message} for "${{ github.event.head_commit.message }} - <{commit_url}|{commit_sha}>". Checkout the run at {run_url}.'
footer: "<{run_url}|Failing Run>"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GHA_FAILURES_WEBHOOK_URL }}
# This jobs reduces the initially planned prod-hub-matrix-jobs deployments by
# filtering out any deployment to a cluster with a failed support-and-staging
# job.
filter-generate-jobs:
runs-on: ubuntu-latest
needs: [generate-jobs, upgrade-support-and-staging]
if: |
!cancelled() &&
(github.event_name == 'push' && contains(github.ref, 'main')) &&
needs.generate-jobs.result == 'success' &&
needs.generate-jobs.outputs.prod-hub-matrix-jobs != '[]'
outputs:
filtered-prod-hub-matrix-jobs: ${{ env.filtered-prod-hub-matrix-jobs }}
steps:
# This Python script filters out any prod hub deployment job from running
# later based on if its part of a cluster where support/staging upgrade
# just failed. Data is injected to the script before its executed via
# string literals as rendered GitHub workflow expressions.
- name: Filter prod deploy jobs to run based on failures in support/staging
shell: python
run: |
import os
import json
jobs = json.loads(r"""${{ needs.generate-jobs.outputs.prod-hub-matrix-jobs }}""")
outputs = json.loads(r"""${{ toJson(needs.upgrade-support-and-staging.outputs) }}""")
try:
filtered_jobs = [
job
for job in jobs
if outputs[f"failure_{job['cluster_name'].replace('.', '-')}"] != "true"
]
except KeyError:
print(f"The {cluster_name} cluster wasn't found in the `upgrade-support-and-staging.outputs` list. Please add it before continuing!")
env_file = os.getenv("GITHUB_ENV")
with open(env_file, "a") as f:
f.write(f"filtered-prod-hub-matrix-jobs={json.dumps(filtered_jobs)}")
# https://github.com/ravsamhq/notify-slack-action
# Needs to be added per job
# When https://github.com/integrations/slack/issues/1563 gets implemented,
# we can use that instead
- name: Report Status
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
# Warning: there are multiple "Report Status" steps in this file (one per job).
# Make sure they are all updated
notify_when: "failure"
status: ${{ job.status }} # required
# Message should look like: "Hey @author! Deploy and test hubs failed for "Add new hub - 9305e08".
message_format: '{emoji} Hey @${{ github.event.head_commit.author.name }}! *{workflow}* {status_message} for "${{ github.event.head_commit.message }} - <{commit_url}|{commit_sha}>". Checkout the run at {run_url}.'
footer: "<{run_url}|Failing Run>"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GHA_FAILURES_WEBHOOK_URL }}
# This job upgrades production hubs on clusters in parallel, if required. This
# job needs both the `filter-generate-jobs` to have completed to provide its
# output `filtered-prod-hub-matrix-jobs`. It is a list of dictionaries with
# the keys cluster_name, provider, and hub_name for each production hub that
# requires an upgrade and didn't have a failed support-and-staging-upgrade job
# run as part of this workflow.
upgrade-prod-hubs:
runs-on: ubuntu-latest
needs: [filter-generate-jobs]
if: |
!cancelled() &&
(github.event_name == 'push' && contains(github.ref, 'main')) &&
needs.filter-generate-jobs.result == 'success' &&
needs.filter-generate-jobs.outputs.filtered-prod-hub-matrix-jobs != '[]'
strategy:
# Don't stop other deployments if one fails
fail-fast: false
matrix:
jobs: ${{ fromJson(needs.filter-generate-jobs.outputs.filtered-prod-hub-matrix-jobs) }}
steps:
- uses: actions/checkout@v4
- name: Setup deploy for ${{ matrix.jobs.cluster_name }} cluster
uses: ./.github/actions/setup-deploy
with:
provider: ${{ matrix.jobs.provider }}
GCP_KMS_DECRYPTOR_KEY: ${{ secrets.GCP_KMS_DECRYPTOR_KEY }}
- name: Upgrade ${{ matrix.jobs.hub_name }} hub on cluster ${{ matrix.jobs.cluster_name }}
run: |
deployer deploy ${{ matrix.jobs.cluster_name }} ${{ matrix.jobs.hub_name }}
# Retry action: https://github.com/marketplace/actions/retry-step
- name: Run health check against ${{ matrix.jobs.hub_name }} hub on cluster ${{ matrix.jobs.cluster_name}}
uses: nick-fields/retry@v3
with:
timeout_minutes: 10
max_attempts: 3
command: |
deployer run-hub-health-check ${{ matrix.jobs.cluster_name }} ${{ matrix.jobs.hub_name }}
# https://github.com/ravsamhq/notify-slack-action
# Needs to be added per job
# When https://github.com/integrations/slack/issues/1563 gets implemented,
# we can use that instead
- name: Report Status
if: always()
uses: ravsamhq/notify-slack-action@v2
with:
# Warning: there are multiple "Report Status" steps in this file (one per job).
# Make sure they are all updated
notify_when: "failure"
status: ${{ job.status }} # required
# Message should look like: "Hey @author! Deploy and test hubs failed for "Add new hub - 9305e08".
message_format: '{emoji} Hey @${{ github.event.head_commit.author.name }}! *{workflow}* {status_message} for "${{ github.event.head_commit.message }} - <{commit_url}|{commit_sha}>". Checkout the run at {run_url}.'
footer: "<{run_url}|Failing Run>"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GHA_FAILURES_WEBHOOK_URL }}