Merge pull request #73 from getindata/release-0.6.0 #145

Workflow file for this run

.github/workflows/tests_and_publish.yml at 50b7d88

	name: "Test and Publish"

	on:
	push:
	branches:
	- master
	- develop
	paths-ignore:
	- "docs/**"
	pull_request:
	branches:
	- master
	- develop
	paths-ignore:
	- "docs/**"

	jobs:
	unit_tests:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: [ '3.8', '3.9', '3.10']

	steps:
	- uses: actions/checkout@v2

	- name: Setup python ${{ matrix.python-version }}
	uses: actions/[email protected]
	with:
	python-version: ${{ matrix.python-version }}

	- name: Setup poetry
	run: \|
	python -V
	python -m pip install poetry

	- name: Check pre-commit status
	run: \|
	poetry install -v
	poetry run pre-commit run --all-files

	- name: Test with tox
	run: \|
	pip install tox-pip-version tox-gh-actions "tox<4.0.0"
	tox -v

	- name: Store coverage reports
	uses: actions/upload-artifact@v3
	with:
	name: coverage-${{ matrix.python-version }}
	path: coverage.xml
	if-no-files-found: error

	sonarcloud:
	runs-on: ubuntu-latest
	needs: unit_tests
	steps:

	- uses: actions/checkout@v2
	with:
	fetch-depth: 0

	- uses: actions/download-artifact@v3
	with:
	name: coverage-3.9
	path: .

	- name: SonarCloud Scan
	uses: sonarsource/sonarcloud-github-action@master
	with:
	projectBaseDir: ${{ github.workspace }}
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}

	codeql:
	runs-on: ubuntu-latest
	needs: unit_tests
	permissions:
	actions: read
	contents: read
	security-events: write

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	# Initializes the CodeQL tools for scanning.
	- name: Initialize CodeQL
	uses: github/codeql-action/init@v2
	with:
	languages: python

	- name: Autobuild
	uses: github/codeql-action/autobuild@v2

	- name: Perform CodeQL Analysis
	uses: github/codeql-action/analyze@v2

	### PROJECT SPECIFIC CONFIGURATION HERE

	e2e_tests:
	if: (github.event.pull_request.head.repo.full_name == github.repository \|\| github.event.pull_request == null)
	needs: [unit_tests, sonarcloud]
	runs-on: ubuntu-latest
	strategy:
	matrix:
	e2e_config: ["e2e", "e2e_pipeline_data_passing"]
	steps:
	- uses: actions/checkout@v2

	- name: Setup python
	uses: actions/[email protected]
	with:
	python-version: "3.10"

	- name: Setup poetry
	run: \|
	python -V
	python -m pip install poetry

	- name: Build the package
	run: \|
	poetry build -f sdist

	- name: Initialize starter project
	run: \|
	pip install $(find "./dist" -name "*.tar.gz")
	kedro new --starter spaceflights --config tests/conf/${{ matrix.e2e_config }}/starter-config.yml --verbose

	- name: Install starter requirements
	working-directory: ./spaceflights
	run: \|
	find "../dist" -name "*.tar.gz" \| xargs -I@ cp @ kedro-azureml.tar.gz
	echo -e "\n./kedro-azureml.tar.gz\n" >> src/requirements.txt
	echo -e "kedro-docker\n" >> src/requirements.txt
	echo -e "openpyxl\n" >> src/requirements.txt # temp fix for kedro-datasets issues with optional packages
	sed -i '/kedro-telemetry/d' src/requirements.txt
	echo $(cat src/requirements.txt)
	pip install -r src/requirements.txt

	- name: Update starter configs
	working-directory: ./spaceflights
	run: \|
	kedro docker init
	sed -i 's/$COPY src\/requirements.txt.*$$/\1\nCOPY kedro-azureml.tar.gz ./g' Dockerfile
	cat Dockerfile
	echo "!data/01_raw" >> .dockerignore
	rm conf/base/catalog.yml
	cp ../tests/conf/${{ matrix.e2e_config }}/catalog.yml conf/base/catalog.yml
	cp ../tests/conf/${{ matrix.e2e_config }}/azureml.yml conf/base/azureml.yml
	sed -i 's/{container_registry}/${{ secrets.REGISTRY_LOGIN_SERVER }}/g' conf/base/azureml.yml
	sed -i 's/{image_tag}/${{ matrix.e2e_config }}/g' conf/base/azureml.yml

	cat conf/base/azureml.yml

	- name: Login via Azure CLI
	uses: azure/login@v1
	with:
	creds: ${{ secrets.AZURE_CREDENTIALS }}

	- name: Login to acr.io
	uses: azure/docker-login@v1
	with:
	login-server: ${{ secrets.REGISTRY_LOGIN_SERVER }}
	username: ${{ secrets.AZURE_CLIENT_ID }}
	password: ${{ secrets.AZURE_CLIENT_SECRET }}

	- name: Build and push docker image
	working-directory: ./spaceflights
	run: \|
	docker pull ${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }} \|\| true
	docker build --build-arg BASE_IMAGE=python:3.10-buster -t ${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }} --cache-from=${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }} .
	docker push ${{ secrets.REGISTRY_LOGIN_SERVER }}/kedro-azureml-e2e:${{ matrix.e2e_config }}

	- name: Run on Azure ML Pipelines
	working-directory: ./spaceflights
	env:
	AZURE_STORAGE_ACCOUNT_KEY: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
	AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
	AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
	AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
	AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
	run: \|
	kedro azureml run --wait-for-completion --env-var 'GETINDATA=ROCKS!'


	publish:
	# only run on push to master
	if: github.event.pull_request == null && github.ref == 'refs/heads/master'
	needs: [e2e_tests, codeql]
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: [ 3.8 ]
	env:
	PYTHON_PACKAGE: kedro_azureml
	steps:
	- name: Checkout the repo
	uses: actions/checkout@v2
	with:
	fetch-depth: 0 # necessary to enable merging, all the history is needed
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v1
	with:
	python-version: ${{ matrix.python-version }}
	- name: Build package dist from source # A better way will be : https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ but pep 517 is still marked as experimental
	run: \|
	pip install poetry
	poetry build
	- name: Merge back to develop # we have to set the config first on a fresh machine
	run: \|
	git config user.name github-actions
	git config user.email [email protected]
	git checkout -B develop --track origin/develop
	git merge master
	git push
	- name: Set dynamically package version as output variable # see https://github.com/actions/create-release/issues/39
	# see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
	id: set_package_version
	run: \|
	echo "::set-output name=PACKAGE_VERSION::$(cat $PYTHON_PACKAGE/__init__.py \| grep -Po '\d+\.\d+\.\d+')"
	- name: Create temporary file with the body content for the release
	run: \|
	grep -Poz "## \[${{steps.set_package_version.outputs.PACKAGE_VERSION}}] - \d{4}-\d{2}-\d{2}[\S\s]+?(?=## \[\d+\.\d+\.\d+\]\|\[.+\]:)" CHANGELOG.md > release_body.md
	- name: Create Release # https://github.com/actions/create-release
	id: create_release
	uses: actions/[email protected]
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
	with:
	tag_name: ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
	release_name: Release ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
	body_path: ./release_body.md
	draft: false
	prerelease: false
	- name: Rollback Release in case of run failure
	if: failure() && steps.create_release.outputs.id != ''
	uses: author/action-rollback@stable
	with:
	# Using a known release ID
	release_id: ${{ steps.create_release.outputs.id }}
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- name: Publish distribution to PyPI # official action from python maintainers
	uses: pypa/gh-action-pypi-publish@master
	with:
	user: __token__
	password: ${{ secrets.PYPI_PASSWORD }}
	verbose: true # trace if the upload fails

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Merge pull request #73 from getindata/release-0.6.0 #145

Workflow file

Merge pull request #73 from getindata/release-0.6.0 #145

Jobs

Run details

Workflow file for this run