Skip to content

feat!: combined helm charts #12360

feat!: combined helm charts

feat!: combined helm charts #12360

Workflow file for this run

name: Test, Integration Tests and Deploy
on:
push:
branches:
- master
- develop
- "release/**" # Or else PRs created by the PR action don't execute tests
- "hotfix/**"
tags:
- "v*.*.*"
pull_request:
types: [ opened, reopened, synchronize ]
branches:
- "**"
- "!master"
jobs:
set-matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: set matrix
id: set-matrix
env:
REF: ${{ github.ref }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: |
if [ "$REF" == "refs/heads/$DEFAULT_BRANCH" ] || [[ "$REF" =~ ^refs/tags/.* ]]
then
echo "matrix={\"python-version\": [\"3.8\", \"3.11\"]}" >> $GITHUB_OUTPUT
else
echo "matrix={\"python-version\": [\"3.10\"]}" >> $GITHUB_OUTPUT
fi
cleanup-runs:
runs-on: ubuntu-latest
steps:
- uses: rokroskar/[email protected]
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
if: "!startsWith(github.ref, 'refs/tags/') && !endsWith(github.ref, github.event.repository.default_branch)"
style-check:
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
- name: Install dependencies
uses: ./.github/actions/install-linux
with:
python-version: '3.10'
- name: Test style
env:
POETRY_VIRTUALENVS_CREATE: false
run: ./run-tests.sh -s
- name: Upload poetry lock file
if: always()
uses: actions/upload-artifact@v3
with:
name: poetry-lock
path: poetry.lock
retention-days: 5
docs-linux:
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
- name: Install dependencies
uses: ./.github/actions/install-linux
- name: Test docs
env:
POETRY_VIRTUALENVS_CREATE: false
SIDEBAR: 1
run: ./run-tests.sh -d
mypy:
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
- name: Test mypy
env:
POETRY_VIRTUALENVS_CREATE: false
run: mypy renku
test-distro:
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
- name: Build Package
env:
POETRY_VIRTUALENVS_CREATE: false
run: poetry build
- name: test with twine
run: twine check dist/*
test-linux-api:
runs-on: ubuntu-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
run: pytest -v -m "not integration and not publish and not serial" -n auto tests/api
- name: Coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
COVERALLS_PARALLEL: true
if: always()
run: coveralls --service=github-actions
continue-on-error: true
test-linux-cli:
runs-on: ubuntu-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
run: pytest -v -m "not integration and not publish and not serial" -n auto tests/cli
- name: Test with pytest (serial)
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
run: pytest -v --cov-append -m "not integration and not publish and serial" tests/cli
- name: Coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
COVERALLS_PARALLEL: true
if: always()
run: coveralls --service=github-actions
continue-on-error: true
test-linux-core:
runs-on: ubuntu-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
run: pytest -v -m "not integration and not publish and not serial" -n auto tests/core
- name: Test with pytest (serial)
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
run: pytest -v --cov-append -m "not integration and not publish and serial" tests/core
- name: Coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
COVERALLS_PARALLEL: true
if: always()
run: coveralls --service=github-actions
continue-on-error: true
test-linux-service:
runs-on: ubuntu-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
run: pytest -v -m "not integration and not publish" -n auto tests/service
- name: Coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
COVERALLS_PARALLEL: true
if: always()
run: coveralls --service=github-actions
continue-on-error: true
test-linux-integration:
runs-on: ubuntu-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
CLOUD_STORAGE_AZURE_KEY: ${{ secrets.CLOUD_STORAGE_AZURE_KEY }}
CLOUD_STORAGE_S3_ACCESS_KEY_ID: ${{ secrets.CLOUD_STORAGE_S3_ACCESS_KEY_ID }}
CLOUD_STORAGE_S3_SECRET_ACCESS_KEY: ${{ secrets.CLOUD_STORAGE_S3_SECRET_ACCESS_KEY }}
run: pytest -m "integration and not shaky and not service and not serial" -v --timeout=600 -n auto
- name: Start Redis
uses: supercharge/[email protected]
- name: Test with pytest (serial)
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
run: pytest --cov-append -m "integration and not shaky and not service and serial" -v --timeout=600
- name: Coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
COVERALLS_PARALLEL: true
if: always()
run: coveralls --service=github-actions
continue-on-error: true
test-linux-integration-shaky:
runs-on: ubuntu-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
CLOUD_STORAGE_AZURE_KEY: ${{ secrets.CLOUD_STORAGE_AZURE_KEY }}
CLOUD_STORAGE_S3_ACCESS_KEY_ID: ${{ secrets.CLOUD_STORAGE_S3_ACCESS_KEY_ID }}
CLOUD_STORAGE_S3_SECRET_ACCESS_KEY: ${{ secrets.CLOUD_STORAGE_S3_SECRET_ACCESS_KEY }}
run: pytest -m "integration and shaky and not service and not serial" -v --timeout=600 -n auto
- name: Start Redis
uses: supercharge/[email protected]
- name: Test with pytest (serial)
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
CLOUD_STORAGE_AZURE_KEY: ${{ secrets.CLOUD_STORAGE_AZURE_KEY }}
CLOUD_STORAGE_S3_ACCESS_KEY_ID: ${{ secrets.CLOUD_STORAGE_S3_ACCESS_KEY_ID }}
CLOUD_STORAGE_S3_SECRET_ACCESS_KEY: ${{ secrets.CLOUD_STORAGE_S3_SECRET_ACCESS_KEY }}
run: pytest -m "integration and shaky and not service and serial" -v --timeout=600 -n auto
- name: Coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
COVERALLS_PARALLEL: true
if: always()
run: coveralls --service=github-actions
continue-on-error: true
test-linux-integration-service:
runs-on: ubuntu-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
run: pytest -m "integration and service and not serial" -v --timeout=600 -n auto
- name: Test with pytest (serial)
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
run: pytest --cov-append -m "integration and service and serial" -v --timeout=600
- name: Coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
GITHUB_TOKEN: $COVERALLS_REPO_TOKEN
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
COVERALLS_PARALLEL: true
if: always()
run: coveralls --service=github-actions
continue-on-error: true
test-macos-api:
runs-on: macos-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-macos
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
run: pytest -v -m "not integration and not publish and not serial and not service" tests/api
test-macos-cli:
runs-on: macos-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-macos
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
run: pytest -v -m "not integration and not publish and not serial and not service" tests/cli
- name: Test with pytest (serial)
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
run: pytest -v -m "not integration and not publish and serial and not service" tests/cli
test-macos-core:
runs-on: macos-latest
needs: [ set-matrix ]
strategy:
max-parallel: 3
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-macos
with:
python-version: ${{ matrix.python-version }}
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
run: pytest -v -m "not integration and not publish and not serial and not service" tests/core
- name: Test with pytest (serial)
env:
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
run: pytest -v -m "not integration and not publish and serial and not service" tests/core
test-macos-integration:
runs-on: macos-latest
needs: [ set-matrix ]
strategy:
max-parallel: 1
matrix: ${{ fromJson(needs.set-matrix.outputs.matrix) }}
if: "startsWith(github.ref, 'refs/tags/') || endsWith(github.ref, github.event.repository.default_branch)"
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-macos
with:
python-version: ${{ matrix.python-version }}
- name: Install RClone
env:
POETRY_VIRTUALENVS_CREATE: false
run: brew install rclone macfuse
- name: Test with pytest
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
CLOUD_STORAGE_AZURE_KEY: ${{ secrets.CLOUD_STORAGE_AZURE_KEY }}
CLOUD_STORAGE_S3_ACCESS_KEY_ID: ${{ secrets.CLOUD_STORAGE_S3_ACCESS_KEY_ID }}
CLOUD_STORAGE_S3_SECRET_ACCESS_KEY: ${{ secrets.CLOUD_STORAGE_S3_SECRET_ACCESS_KEY }}
run: pytest -m "integration and not serial and not service" -v
- name: Start Redis
uses: supercharge/[email protected]
- name: Test with pytest (serial)
env:
POETRY_VIRTUALENVS_CREATE: false
LANG: en_US.UTF-8
LC_ALL: en_US.UTF-8
DATAVERSE_ACCESS_TOKEN: ${{ secrets.DATAVERSE_ACCESS_TOKEN }}
IT_OAUTH_GIT_TOKEN: ${{ secrets.IT_OAUTH_GIT_TOKEN }}
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
OLOS_ACCESS_TOKEN: ${{ secrets.OLOS_ACCESS_TOKEN }}
RENKU_REQUESTS_TIMEOUT_SECONDS: 120
run: pytest -m "integration and serial and not service" -v
publish-pypi:
runs-on: ubuntu-latest
if: "startsWith(github.ref, 'refs/tags/')"
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/install-linux
with:
python-version: '3.9'
- name: Calculate pre-release version
id: pre-release-version
run: |
renku_version=$(renku --version)
rc=$(curl -s https://pypi.org/pypi/renku/json | jq ".releases | keys_unsorted | map(select( . | startswith(\"$renku_version\"))) | map(match(\"[0-9]+\\\\.[0-9]+\\\\.[0-9]+(rc([0-9]+))?\").captures[1].string // 0) | max // 0 | tonumber + 1")
echo "RELEASE_VERSION=${renku_version}rc${rc}" >> "$GITHUB_OUTPUT"
- name: Build and publish to pypi
env:
POETRY_VIRTUALENVS_CREATE: false
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ steps.pre-release-version.outputs.RELEASE_VERSION }}
run: make publish-package
- name: Build and publish lock package
env:
POETRY_VIRTUALENVS_CREATE: false
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ steps.pre-release-version.outputs.RELEASE_VERSION }}
run: make publish-lock-package
build-images:
runs-on: ubuntu-latest
needs:
[
docs-linux,
test-linux-cli,
test-linux-core,
test-linux-service,
test-macos-cli,
test-macos-core,
]
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Build images
run: make docker-tag
- name: Docker Login
if: "startsWith(github.ref, 'refs/tags/') || endsWith(github.ref, github.event.repository.default_branch)"
uses: Azure/docker-login@v1
with:
username: ${{ secrets.RENKU_DOCKER_USERNAME }}
password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
- name: Push images
if: "startsWith(github.ref, 'refs/tags/') || endsWith(github.ref, github.event.repository.default_branch)"
run: |
make docker-push
publish-chart:
runs-on: ubuntu-latest
needs:
[
docs-linux,
test-linux-cli,
test-linux-core,
test-linux-service,
test-macos-cli,
test-macos-core,
]
if: startsWith(github.ref, 'refs/tags/')
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Set up environment variables
run: |
echo "GIT_USER=Renku Bot" >> $GITHUB_ENV
echo "[email protected]" >> $GITHUB_ENV
- name: Build and push images
uses: SwissDataScienceCenter/renku-actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
- name: Update component version
uses: SwissDataScienceCenter/renku-actions/[email protected]
env:
COMPONENT_NAME: renku-core
GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
coveralls-final:
name: Aggregate coveralls data
needs:
[
test-linux-cli,
test-linux-core,
test-linux-service,
test-linux-api,
test-linux-integration,
test-linux-integration-shaky,
test-linux-integration-service,
]
if: always()
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Coveralls Finished
uses: coverallsapp/github-action@master
with:
github-token: ${{ secrets.github_token }}
parallel-finished: true
base-path: renku/