From ca01f2084662e7256a6ebf2c8aae7b1c0cdff0c6 Mon Sep 17 00:00:00 2001 From: Alexander Smolyakov <alexander.smolyakov.dev@gmail.com> Date: Sat, 4 Feb 2023 03:02:44 +0400 Subject: [PATCH] [CI/CD] Backport release workflow to `1.3.latest` (#426) * [CI/CD] Update release workflow and introduce workflow for nightly releases (#394) * Add workflows * Set default `test_run` value to `true` * Update .bumpversion.cfg * Resolve review comments - Update workflow docs - Change workflow name - Set `test_run` default value to `true` * Update Slack secret * Resolve review comments * Update release workflow (#421) - Update AWS secrets - Rework condition for Slack notification * update regex for version bump (#444) # Conflicts: # .bumpversion.cfg * update prerelease -> prekind (#445) * update regex for version bump * update to prekind * mroe renaming --------- Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> * put back the right version --------- Co-authored-by: Emily Rockman <emily.rockman@dbtlabs.com> Co-authored-by: colin-rogers-dbt <111200756+colin-rogers-dbt@users.noreply.github.com> --- .bumpversion.cfg | 24 ++- .github/workflows/nightly-release.yml | 109 ++++++++++++ .github/workflows/release.yml | 228 ++++++++++++++++++++++++++ scripts/env-setup.sh | 10 ++ 4 files changed, 365 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/nightly-release.yml create mode 100644 .github/workflows/release.yml create mode 100644 scripts/env-setup.sh diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 83154a2ab..aa23c47d1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,16 +1,26 @@ [bumpversion] current_version = 1.3.0 -parse = (?P<major>\d+) - \.(?P<minor>\d+) - \.(?P<patch>\d+) - ((?P<prerelease>a|b|rc)(?P<num>\d+))? + +# `parse` allows parsing the version into the parts we need to check. There are some +# unnamed groups and that's okay because they do not need to be audited. If any part +# of the version passed and does not match the regex, it will fail. +# expected matches: `1.5.0`, `1.5.0a1`, `1.5.0a1.dev123457+nightly` +# excepted failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0` +parse = (?P<major>[\d]+) # major version number + \.(?P<minor>[\d]+) # minor version number + \.(?P<patch>[\d]+) # patch version number + (((?P<prekind>a|b|rc) # optional pre-release type + ?(?P<num>[\d]+?)) # optional pre-release version number + \.?(?P<nightly>[a-z0-9]+\+[a-z]+)? # optional nightly release indicator + )? serialize = - {major}.{minor}.{patch}{prerelease}{num} + {major}.{minor}.{patch}{prekind}{num}.{nightly} + {major}.{minor}.{patch}{prekind}{num} {major}.{minor}.{patch} commit = False tag = False -[bumpversion:part:prerelease] +[bumpversion:part:prekind] first_value = a optional_value = final values = @@ -22,6 +32,8 @@ values = [bumpversion:part:num] first_value = 1 +[bumpversion:part:nightly] + [bumpversion:file:setup.py] [bumpversion:file:dbt/adapters/snowflake/__version__.py] diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml new file mode 100644 index 000000000..b668d62ec --- /dev/null +++ b/.github/workflows/nightly-release.yml @@ -0,0 +1,109 @@ +# **what?** +# Nightly releases to GitHub and PyPI. This workflow produces the following outcome: +# - generate and validate data for night release (commit SHA, version number, release branch); +# - pass data to release workflow; +# - night release will be pushed to GitHub as a draft release; +# - night build will be pushed to test PyPI; +# +# **why?** +# Ensure an automated and tested release process for nightly builds +# +# **when?** +# This workflow runs on schedule or can be run manually on demand. + +name: Nightly Test Release to GitHub and PyPI + +on: + workflow_dispatch: # for manual triggering + schedule: + - cron: 0 9 * * * + +permissions: + contents: write # this is the permission that allows creating a new release + +defaults: + run: + shell: bash + +env: + RELEASE_BRANCH: "main" + +jobs: + aggregate-release-data: + runs-on: ubuntu-latest + + outputs: + commit_sha: ${{ steps.resolve-commit-sha.outputs.release_commit }} + version_number: ${{ steps.nightly-release-version.outputs.number }} + release_branch: ${{ steps.release-branch.outputs.name }} + + steps: + - name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}" + uses: actions/checkout@v3 + with: + ref: ${{ env.RELEASE_BRANCH }} + + - name: "Resolve Commit To Release" + id: resolve-commit-sha + run: | + commit_sha=$(git rev-parse HEAD) + echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT + + - name: "Get Current Version Number" + id: version-number-sources + run: | + current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '` + echo "current_version=$current_version" >> $GITHUB_OUTPUT + + - name: "Audit Version And Parse Into Parts" + id: semver + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ steps.version-number-sources.outputs.current_version }} + + - name: "Get Current Date" + id: current-date + run: echo "date=$(date +'%m%d%Y')" >> $GITHUB_OUTPUT + + - name: "Generate Nightly Release Version Number" + id: nightly-release-version + run: | + number="${{ steps.semver.outputs.version }}.dev${{ steps.current-date.outputs.date }}+nightly" + echo "number=$number" >> $GITHUB_OUTPUT + + - name: "Audit Nightly Release Version And Parse Into Parts" + uses: dbt-labs/actions/parse-semver@v1.1.0 + with: + version: ${{ steps.nightly-release-version.outputs.number }} + + - name: "Set Release Branch" + id: release-branch + run: | + echo "name=${{ env.RELEASE_BRANCH }}" >> $GITHUB_OUTPUT + + log-outputs-aggregate-release-data: + runs-on: ubuntu-latest + needs: [aggregate-release-data] + + steps: + - name: "[DEBUG] Log Outputs" + run: | + echo commit_sha : ${{ needs.aggregate-release-data.outputs.commit_sha }} + echo version_number: ${{ needs.aggregate-release-data.outputs.version_number }} + echo release_branch: ${{ needs.aggregate-release-data.outputs.release_branch }} + + release-github-pypi: + needs: [aggregate-release-data] + + uses: ./.github/workflows/release.yml + with: + sha: ${{ needs.aggregate-release-data.outputs.commit_sha }} + target_branch: ${{ needs.aggregate-release-data.outputs.release-branch }} + version_number: ${{ needs.aggregate-release-data.outputs.version_number }} + build_script_path: "scripts/build-dist.sh" + env_setup_script_path: "scripts/env-setup.sh" + s3_bucket_name: "core-team-artifacts" + package_test_command: "dbt --version" + test_run: true + nightly_release: true + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..1c0885001 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,228 @@ +# **what?** +# Release workflow provides the following steps: +# - checkout the given commit; +# - validate version in sources and changelog file for given version; +# - bump the version and generate a changelog if needed; +# - merge all changes to the target branch if needed; +# - run unit and integration tests against given commit; +# - build and package that SHA; +# - release it to GitHub and PyPI with that specific build; +# +# **why?** +# Ensure an automated and tested release process +# +# **when?** +# This workflow can be run manually on demand or can be called by other workflows +name: Release to GitHub and PyPI + +on: + workflow_dispatch: + inputs: + sha: + description: "The last commit sha in the release" + type: string + required: true + target_branch: + description: "The branch to release from" + type: string + required: true + version_number: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + build_script_path: + description: "Build script path" + type: string + default: "scripts/build-dist.sh" + required: true + env_setup_script_path: + description: "Environment setup script path" + type: string + default: "scripts/env-setup.sh" + required: false + s3_bucket_name: + description: "AWS S3 bucket name" + type: string + default: "core-team-artifacts" + required: true + package_test_command: + description: "Package test command" + type: string + default: "dbt --version" + required: true + test_run: + description: "Test run (Publish release as draft)" + type: boolean + default: true + required: false + nightly_release: + description: "Nightly release to dev environment" + type: boolean + default: false + required: false + workflow_call: + inputs: + sha: + description: "The last commit sha in the release" + type: string + required: true + target_branch: + description: "The branch to release from" + type: string + required: true + version_number: + description: "The release version number (i.e. 1.0.0b1)" + type: string + required: true + build_script_path: + description: "Build script path" + type: string + default: "scripts/build-dist.sh" + required: true + env_setup_script_path: + description: "Environment setup script path" + type: string + default: "scripts/env-setup.sh" + required: false + s3_bucket_name: + description: "AWS S3 bucket name" + type: string + default: "core-team-artifacts" + required: true + package_test_command: + description: "Package test command" + type: string + default: "dbt --version" + required: true + test_run: + description: "Test run (Publish release as draft)" + type: boolean + default: true + required: false + nightly_release: + description: "Nightly release to dev environment" + type: boolean + default: false + required: false + +permissions: + contents: write # this is the permission that allows creating a new release + +defaults: + run: + shell: bash + +jobs: + log-inputs: + name: Log Inputs + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Print Variables" + run: | + echo The last commit sha in the release: ${{ inputs.sha }} + echo The branch to release from: ${{ inputs.target_branch }} + echo The release version number: ${{ inputs.version_number }} + echo Build script path: ${{ inputs.build_script_path }} + echo Environment setup script path: ${{ inputs.env_setup_script_path }} + echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }} + echo Package test command: ${{ inputs.package_test_command }} + echo Test run: ${{ inputs.test_run }} + echo Nightly release: ${{ inputs.nightly_release }} + + bump-version-generate-changelog: + name: Bump package version, Generate changelog + + uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main + + with: + sha: ${{ inputs.sha }} + version_number: ${{ inputs.version_number }} + target_branch: ${{ inputs.target_branch }} + env_setup_script_path: ${{ inputs.env_setup_script_path }} + test_run: ${{ inputs.test_run }} + nightly_release: ${{ inputs.nightly_release }} + + secrets: inherit + + log-outputs-bump-version-generate-changelog: + name: "[Log output] Bump package version, Generate changelog" + if: ${{ !failure() && !cancelled() }} + + needs: [bump-version-generate-changelog] + + runs-on: ubuntu-latest + + steps: + - name: Print variables + run: | + echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} + echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + + build-test-package: + name: Build, Test, Package + if: ${{ !failure() && !cancelled() }} + needs: [bump-version-generate-changelog] + + uses: dbt-labs/dbt-release/.github/workflows/build.yml@main + + with: + sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} + version_number: ${{ inputs.version_number }} + changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + build_script_path: ${{ inputs.build_script_path }} + s3_bucket_name: ${{ inputs.s3_bucket_name }} + package_test_command: ${{ inputs.package_test_command }} + test_run: ${{ inputs.test_run }} + nightly_release: ${{ inputs.nightly_release }} + + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + github-release: + name: GitHub Release + if: ${{ !failure() && !cancelled() }} + + needs: [bump-version-generate-changelog, build-test-package] + + uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main + + with: + sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} + version_number: ${{ inputs.version_number }} + changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} + test_run: ${{ inputs.test_run }} + + pypi-release: + name: PyPI Release + + needs: [github-release] + + uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main + + with: + version_number: ${{ inputs.version_number }} + test_run: ${{ inputs.test_run }} + + secrets: + PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + + slack-notification: + name: Slack Notification + if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }} + + needs: + [ + bump-version-generate-changelog, + build-test-package, + github-release, + pypi-release, + ] + + uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main + with: + status: "failure" + + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }} diff --git a/scripts/env-setup.sh b/scripts/env-setup.sh new file mode 100644 index 000000000..f05c705ce --- /dev/null +++ b/scripts/env-setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Set TOXENV environment variable for subsequent steps +echo "TOXENV=integration-snowflake" >> $GITHUB_ENV +# Set INTEGRATION_TESTS_SECRETS_PREFIX environment variable for subsequent steps +# All GH secrets that have this prefix will be set as environment variables +echo "INTEGRATION_TESTS_SECRETS_PREFIX=SNOWFLAKE_TEST" >> $GITHUB_ENV +# Set environment variables required for integration tests +echo "DBT_TEST_USER_1=dbt_test_role_1" >> $GITHUB_ENV +echo "DBT_TEST_USER_2=dbt_test_role_2" >> $GITHUB_ENV +echo "DBT_TEST_USER_3=dbt_test_role_3" >> $GITHUB_ENV