Skip to content
CI

feat: move runner scaling to the provisioner #9408

Sign in to view logs

feat: move runner scaling to the provisioner

feat: move runner scaling to the provisioner #9408

Workflow file for this run

.github/workflows/ci.yml at 71a976b
on:
pull_request:
merge_group:
name: CI
concurrency:
group: ${{ github.ref }}-ci
cancel-in-progress: true
jobs:
test:
name: Test Go
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: Docker Compose
run: docker compose up -d --wait
- name: Build Language Plugins
run: just build-language-plugins
- name: Test
run: |
go-test-annotate
# go-test-annotate ${{ (github.event_name == 'pull_request' && github.event.action != 'enqueued' && !contains( github.event.pull_request.labels.*.name, 'run-all')) && '-short' || '' }}
test-readme:
name: Test README
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: Docker Compose
run: docker compose up -d --wait
- name: Build Language Plugins
run: just build-language-plugins
- name: Test README
run: just test-readme
test-scripts:
name: Test Scripts
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Init Hermit
uses: cashapp/[email protected]
- name: Test Scripts
run: just test-scripts
sql:
name: SQL
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: Docker Compose
run: docker compose up -d --wait
- name: Initialise database
run: just init-db
- name: Vet SQL
run: sqlc vet
ensure-frozen-migrations:
name: Ensure Frozen Migrations
# if: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'skip-ensure-frozen-migrations') }}
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Init Hermit
uses: cashapp/[email protected]
- name: Freeze Migrations
run: just ensure-frozen-migrations
lint:
name: Lint
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: golangci-lint
run: |
golangci-lint --version
golangci-lint run --new-from-rev="$(git merge-base origin/main HEAD)" --out-format github-actions ./...
- name: lint-commit-or-rollback
run: lint-commit-or-rollback ./backend/... 2>&1 | to-annotation
- name: go-check-sumtype
shell: bash
run: go-check-sumtype ./... 2>&1 | to-annotation
- name: actionlint
shell: bash
run: actionlint --oneline | to-annotation
# Too annoying to disable individual warnings
# - name: staticcheck
# run: staticcheck ./...
- name: lint-scripts
run: just lint-scripts
proto-breaking:
name: Proto Breaking Change Check
if: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'skip-proto-breaking') }}
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Init Hermit
uses: cashapp/[email protected]
- name: Proto Breaking Change Check
shell: bash
run: |
set -o pipefail
buf breaking --against 'https://github.com/TBD54566975/ftl.git#branch=main' | to-annotation || true
console:
name: Console
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: Console pnpm install
run: just pnpm-install
- name: Console Lint
run: just lint-frontend
- name: Console Build
run: just build-frontend
- name: Console Test
run: just test-frontend
extension:
name: VSCode Extension
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: VSCode extension pnpm install
run: just pnpm-install
- name: VSCode extension lint
working-directory: frontend/vscode
run: pnpm run lint
- name: VSCode extension pnpm build and package
run: just package-extension
plugin:
name: Intellij Plugin
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: Install Java
run: java -version
- name: Build Intellij Plugin
run: just build-intellij-plugin
build-all:
name: Rebuild All
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: Docker Compose
run: docker compose up -d --wait
- name: Init DB
run: just init-db
- name: Rebuild All
run: just build-all
- name: Check No SCM Changes
run: |
if [[ -n $(git status -s) ]]
then
echo "Running the build resulted in changes to git controlled files:"
git status -s
git --no-pager diff
exit 1
fi
docs:
name: Build Docs
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: cashapp/[email protected]
- run: cd docs && zola build
arch-lint:
name: Lint Architecture
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: cashapp/[email protected]
- run: go-arch-lint check || true
docker-build-controller:
name: Build Controller Docker Image
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: cashapp/[email protected]
- uses: ./.github/actions/build-cache
- run: just build-docker controller
docker-build-provisioner:
name: Build Provisioner Docker Image
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: cashapp/[email protected]
- uses: ./.github/actions/build-cache
- run: just build-docker provisioner
docker-build-cron:
name: Build Cron Docker Image
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: cashapp/[email protected]
- uses: ./.github/actions/build-cache
- run: just build-docker cron
docker-build-http-ingress:
name: Build HTTP Ingress Docker Image
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: cashapp/[email protected]
- uses: ./.github/actions/build-cache
- run: just build-docker http-ingress
docker-build-runner:
name: Build Runner Docker Images
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: cashapp/[email protected]
- uses: ./.github/actions/build-cache
- name: Build Runner Docker Image
run: just build-docker runner
docker-build-jvm-runners:
name: Build JVM Runner Docker Images
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: cashapp/[email protected]
- uses: ./.github/actions/build-cache
- name: Build JVM Docker Imag
run: just build-docker runner-jvm
console-e2e:
name: Console e2e
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: Restore playwright browser cache
id: cache-playwright
uses: actions/cache/restore@v4
with:
path: ~/.cache/ms-playwright
key: ${{ runner.os }}-playwright-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-playwright-
- name: Console pnpm install
run: just pnpm-install
- name: Build Language Plugins
run: just build-language-plugins
- name: Console e2e
run: just e2e-frontend
integration-shard:
name: Shard Integration Tests
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.extract-tests.outputs.matrix }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Extract test cases
id: extract-tests
run: |
set -euo pipefail
# shellcheck disable=SC2046
echo "matrix={\"test\":$(jq -c -n '$ARGS.positional' --args $(git grep -l '^//go:build integration' | xargs grep '^func Test' | awk '{print $2}' | cut -d'(' -f1))}" >> "$GITHUB_OUTPUT"
integration-run:
name: Integration Test
# if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
needs: integration-shard
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix: ${{fromJson(needs.integration-shard.outputs.matrix)}}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: Docker Compose
run: docker compose --profile integration up -d --wait
- name: Create DB
run: just init-db
- name: Download Go Modules
run: go mod download
- name: Build Language Plugins
run: just build-language-plugins
- name: Run ${{ matrix.test }}
run: |
set -euo pipefail
# shellcheck disable=SC2046
go test -v -race -tags integration -run '^${{ matrix.test }}$' $(git grep -l '^//go:build integration' | xargs grep -l '^func ${{ matrix.test }}' | xargs -I {} dirname ./{})
infrastructure-shard:
name: Shard Infrastructure Tests
if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.extract-tests.outputs.matrix }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Extract test cases
id: extract-tests
run: |
set -euo pipefail
# shellcheck disable=SC2046
echo "matrix={\"test\":$(jq -c -n '$ARGS.positional' --args $(git grep -l '^//go:build infrastructure' | xargs grep '^func Test' | awk '{print $2}' | cut -d'(' -f1))}" >> "$GITHUB_OUTPUT"
infrastructure-run:
name: Infrastructure Test
if: github.event_name != 'pull_request' || github.event.action == 'enqueued' || contains( github.event.pull_request.labels.*.name, 'run-all')
needs: infrastructure-shard
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix: ${{fromJson(needs.infrastructure-shard.outputs.matrix)}}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
- name: Checkout code
uses: actions/checkout@v4
- name: Init Hermit
uses: cashapp/[email protected]
- name: Build Cache
uses: ./.github/actions/build-cache
- name: Docker Compose
run: docker compose --profile integration up -d --wait
- name: Create DB
run: just init-db
- name: Download Go Modules
run: go mod download
- name: Build Language Plugins
run: just build-language-plugins
- name: Run ${{ matrix.test }}
run: |
set -euo pipefail
# shellcheck disable=SC2046
go test -v -race -tags infrastructure -run '^${{ matrix.test }}$' $(git grep -l '^//go:build infrastructure' | xargs grep -l '^func ${{ matrix.test }}' | xargs -I {} dirname ./{})
- name: Archive Report
uses: actions/upload-artifact@v4
if: always() # Always upload the report even on failure
with:
name: kube-report-${{ matrix.test }}
path: /tmp/ftl-kube-report/
- name: Teardown Cluster
working-directory: deployment
if: always() # Always cleanup the cluster even on failure
run: just teardown-cluster
- name: Delete Images
if: always() # We don't want to cache these images, delete them before the cache action
run: docker image rm localhost:5000/ftl-runner:latest ftl-runner:latest ftl-controller:latest localhost:5000/ftl-controller:latest
integration-success:
name: Integration Success
needs: [integration-run]
runs-on: ubuntu-latest
if: ${{ always() }}
steps:
- name: Check integration tests result
run: |
if [[ "${{ needs.integration-run.result }}" == "failure" ]]; then
echo "Integration tests failed"
exit 1
else
echo "Integration tests passed"
fi