Skip to content

Release

Release #844

Workflow file for this run

name: Release
on:
schedule:
- cron: "0 22 * * *"
workflow_dispatch:
inputs:
tag:
description: The tags to be released
required: false
type: string
stable:
description: Make a stable release
required: false
type: boolean
permissions:
id-token: write
pull-requests: write
checks: write
statuses: write
contents: write
env:
BUILD_PROFILE: release
RUNNER_PROVIDER: aws
jobs:
create_release:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.bump.outputs.tag }}
sha: ${{ steps.bump.outputs.sha }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Bump version
id: bump
uses: actions/github-script@v7
env:
STABLE: "${{ inputs.stable }}"
TAG: "${{ inputs.tag }}"
with:
script: |
const script = require('./.github/scripts/bump_version.js')
await script({ github, context, core })
- name: Create release
env:
# we need workflow:write permission to create release if there were any workflow changes
# which is not possible for github actions token
GH_TOKEN: ${{ secrets.DATABEND_BOT_TOKEN }}
run: |
echo "Creating release ${{ steps.bump.outputs.tag }} from ${{ steps.bump.outputs.sha }}"
if [[ "${{ inputs.stable }}" == "true" ]]; then
echo "Stable release"
previous=$(gh release list --limit 1 --exclude-pre-releases | cut -f 1)
gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag $previous --latest --draft
else
echo "Nightly release"
previous=$(gh release list --limit 1 | cut -f 1)
gh release create ${{ steps.bump.outputs.tag }} --target ${{ steps.bump.outputs.sha }} --generate-notes --notes-start-tag $previous --prerelease --draft
fi
changelog:
runs-on: ubuntu-latest
if: inputs.stable
needs: create_release
steps:
- name: Checkout Docs
uses: actions/checkout@v4
with:
repository: datafuselabs/databend-docs
ref: main
- name: Get date
shell: bash
run: echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
- name: Generate Release Note
env:
GH_TOKEN: ${{ github.token }}
run: |
mkdir -p docs/release-stable
df="docs/release-stable/${{ env.DATE }}_${{ needs.create_release.outputs.version }}.md"
echo "---" > $df
gh release view --repo datafuselabs/databend ${{ needs.create_release.outputs.version }} >> $df
sed -i -E 's/^--$/---/g' $df
sed -i -E '/^asset:/d' $df
sed -i -E 's_https://github.com/datafuselabs/databend/pull/([0-9]+)_[#\1](https://github.com/datafuselabs/databend/pull/\1)_g' $df
git add docs/release-stable
git status
- uses: peter-evans/create-pull-request@v4
with:
token: ${{ secrets.DATABEND_BOT_TOKEN }}
title: "chore(docs): Update Release Notes - ${{ env.DATE }}"
base: main
commit-message: "chore(docs): Update Release Notes - ${{ env.DATE }}"
branch-suffix: random
delete-branch: true
build_default:
runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws]
needs: create_release
strategy:
fail-fast: false
matrix:
include:
- { target: x86_64-unknown-linux-gnu, runner: X64 }
- { target: aarch64-unknown-linux-gnu, runner: ARM64 }
- { target: x86_64-unknown-linux-musl, runner: X64 }
- { target: aarch64-unknown-linux-musl, runner: X64 }
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
fetch-depth: 0
- name: Build Release
uses: ./.github/actions/build_linux
env:
DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }}
with:
sha: ${{ github.sha }}
target: ${{ matrix.target }}
artifacts: sqllogictests,sqlsmith,query,meta,metactl,metaverifier
- name: Basic Sqllogic Test
if: matrix.target != 'aarch64-unknown-linux-musl'
shell: bash
env:
TEST_HANDLERS: http
run: |
mkdir -p ./target/${{ env.BUILD_PROFILE }}
cp ./target/${{ matrix.target }}/${{ env.BUILD_PROFILE }}/databend-* ./target/${{ env.BUILD_PROFILE }}/
bash ./scripts/ci/ci-run-sqllogic-tests.sh base
build_udf:
runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws]
needs: create_release
strategy:
fail-fast: false
matrix:
include:
- { target: x86_64-unknown-linux-gnu, runner: X64 }
- { target: aarch64-unknown-linux-gnu, runner: ARM64 }
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
fetch-depth: 0
- name: Build Release
uses: ./.github/actions/build_linux
env:
DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }}
with:
sha: ${{ github.sha }}
target: ${{ matrix.target }}
artifacts: sqllogictests,sqlsmith,metactl,meta,query
features: python-udf
category: udf
build_hdfs:
runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 16c32g, aws]
needs: create_release
strategy:
fail-fast: false
matrix:
include:
- { target: x86_64-unknown-linux-gnu, runner: X64 }
- { target: aarch64-unknown-linux-gnu, runner: ARM64 }
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
fetch-depth: 0
- name: Build Release
uses: ./.github/actions/build_linux
env:
DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }}
with:
sha: ${{ github.sha }}
target: ${{ matrix.target }}
artifacts: sqllogictests,sqlsmith,metactl,meta,query
features: storage-hdfs
category: hdfs
publish:
runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 4c8g, aws]
needs: [create_release, build_default, build_hdfs]
strategy:
fail-fast: false
matrix:
include:
- category: default
target: x86_64-unknown-linux-gnu
runner: X64
- category: default
target: aarch64-unknown-linux-gnu
runner: ARM64
- category: default
target: x86_64-unknown-linux-musl
runner: X64
- category: default
target: aarch64-unknown-linux-musl
runner: X64
- category: hdfs
target: x86_64-unknown-linux-gnu
runner: X64
- category: hdfs
target: aarch64-unknown-linux-gnu
runner: ARM64
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Download artifact
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: ${{ matrix.target }}
category: ${{ matrix.category }}
path: distro/bin
artifacts: metactl,meta,query
- name: Pack Binaries
run: |
target=${{ matrix.target }}
version=${{ needs.create_release.outputs.version }}
case ${{ matrix.category }} in
default)
pkg_name="databend-${version}-${target}"
;;
*)
pkg_name="databend-${{ matrix.category }}-${version}-${target}"
;;
esac
mkdir -p distro/{bin,configs,systemd,scripts}
cp ./scripts/distribution/systemd/databend-* distro/systemd/
cp ./scripts/distribution/configs/databend-* distro/configs/
cp ./scripts/distribution/release-readme.txt distro/readme.txt
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
- name: post sha256
uses: actions/upload-artifact@v4
with:
name: sha256sums-${{ matrix.category }}-${{ matrix.target }}
path: sha256-*.txt
retention-days: 1
- name: Publish Binaries
uses: ./.github/actions/publish_binary
env:
GH_TOKEN: ${{ github.token }}
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
with:
version: ${{ needs.create_release.outputs.version }}
target: ${{ matrix.target }}
category: ${{ matrix.category }}
publish_testsuite:
runs-on: [self-hosted, X64, Linux, 4c8g, aws]
needs: [create_release, build_default]
strategy:
fail-fast: false
matrix:
target:
- x86_64-unknown-linux-gnu
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Download artifact
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: ${{ matrix.target }}
category: default
artifacts: sqllogictests,sqlsmith
path: distro/bin
- name: Pack Testsuite
run: |
target=${{ matrix.target }}
version=${{ needs.create_release.outputs.version }}
pkg_name="databend-testsuite-${version}-${target}"
cp -r ./tests/sqllogictests/suites ./distro/
tar -C ./distro -czvf ${pkg_name}.tar.gz bin suites
sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
- name: post sha256
uses: actions/upload-artifact@v4
with:
name: sha256sums-testsuite-${{ matrix.target }}
path: sha256-*.txt
retention-days: 1
- name: Publish Testsuite
uses: ./.github/actions/publish_binary
env:
GH_TOKEN: ${{ github.token }}
with:
version: ${{ needs.create_release.outputs.version }}
target: ${{ matrix.target }}
category: testsuite
docker_all_in_one:
runs-on: [self-hosted, X64, Linux, 4c8g, aws]
needs: [create_release, build_default]
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Download artifacts for x86_64
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: x86_64-unknown-linux-gnu
category: default
artifacts: metactl,meta,query
path: distro/linux/amd64
- name: Download artifacts for aarch64
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: aarch64-unknown-linux-gnu
category: default
artifacts: metactl,meta,query
path: distro/linux/arm64
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- uses: ./.github/actions/setup_docker
id: login
with:
repo: databend
ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Get Image Tags
id: tags
uses: actions/github-script@v7
env:
REPO_DOCKERHUB: ${{ steps.login.outputs.dockerhub_repo }}
REPO_ECR: ${{ steps.login.outputs.ecr_repo }}
VERSION: ${{ needs.create_release.outputs.version }}
STABLE: ${{ inputs.stable }}
with:
script: |
const version = process.env.VERSION;
const repos = [process.env.REPO_DOCKERHUB, process.env.REPO_ECR];
const stable = process.env.STABLE;
let tags = [];
for (const repo of repos) {
tags.push(`${repo}:${version}`);
if (stable === 'true') {
tags.push(`${repo}:latest`);
} else {
tags.push(`${repo}:nightly`);
}
}
core.setOutput('tags', tags.join(','));
- name: Build and push
id: docker_build
uses: docker/build-push-action@v5
with:
push: true
tags: ${{ steps.tags.outputs.tags }}
platforms: linux/amd64,linux/arm64
context: .
file: ./docker/Dockerfile
- name: Update repo description
continue-on-error: true
uses: peter-evans/dockerhub-description@v4
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
repository: ${{ steps.login.outputs.dockerhub_repo }}
short-description: "A modern cloud data warehouse. Also available in the cloud: https://app.databend.com."
readme-filepath: ./docker/README.md
docker_service:
runs-on: [self-hosted, X64, Linux, 4c8g, aws]
needs: [create_release, build_udf]
strategy:
fail-fast: false
matrix:
service:
- meta
- query
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Download artifacts for x86_64
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: x86_64-unknown-linux-gnu
category: udf
artifacts: ${{ matrix.service }},metactl
path: distro/linux/amd64
- name: Download artifacts for aarch64
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: aarch64-unknown-linux-gnu
category: udf
artifacts: ${{ matrix.service }},metactl
path: distro/linux/arm64
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- uses: ./.github/actions/setup_docker
id: login
with:
repo: databend-${{ matrix.service }}
ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Get Image Tags
id: tags
uses: actions/github-script@v7
env:
REPO_DOCKERHUB: ${{ steps.login.outputs.dockerhub_repo }}
REPO_ECR: ${{ steps.login.outputs.ecr_repo }}
VERSION: ${{ needs.create_release.outputs.version }}
STABLE: ${{ inputs.stable }}
with:
script: |
const version = process.env.VERSION;
const repos = [process.env.REPO_DOCKERHUB, process.env.REPO_ECR];
const stable = process.env.STABLE;
let tags = [];
for (const repo of repos) {
tags.push(`${repo}:${version}`);
if (stable === 'true') {
tags.push(`${repo}:latest`);
} else {
tags.push(`${repo}:nightly`);
}
}
core.setOutput('tags', tags.join(','));
- name: push service image
uses: docker/build-push-action@v5
with:
push: true
tags: ${{ steps.tags.outputs.tags }}
platforms: linux/amd64,linux/arm64
context: .
file: ./docker/service/${{ matrix.service }}.Dockerfile
distribution:
runs-on: [self-hosted, X64, Linux, 4c8g, aws]
needs: [create_release, build_default]
strategy:
matrix:
arch:
- x86_64
- aarch64
packager:
- deb
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Install nfpm@latest
run: |
curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
tar xf nfpm.tar.gz
sudo mv nfpm /usr/local/bin
sudo chmod a+x /usr/local/bin/nfpm
rm nfpm.tar.gz
- name: Get target
id: target
run: |
echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
- name: Download artifacts
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: ${{ steps.target.outputs.target }}
category: default
artifacts: metactl,meta,query
path: distro/bin
- name: Build Packages
id: build_packages
run: |
export name="databend"
export version="${{ needs.create_release.outputs.version }}"
export path="distro"
case "${{ matrix.arch }}" in
x86_64)
export arch="amd64"
;;
aarch64)
export arch="arm64"
;;
esac
mkdir -p distro/{bin,configs,systemd,scripts}
cp ./scripts/distribution/systemd/databend-* distro/systemd/
cp ./scripts/distribution/configs/databend-* distro/configs/
cp ./scripts/distribution/release-readme.txt distro/readme.txt
cp -r ./scripts/distribution/local-scripts/* distro/scripts/
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
- name: Update release to github
shell: bash
env:
GH_TOKEN: ${{ github.token }}
# Reference: https://cli.github.com/manual/gh_release_upload
run: |
version="${{ needs.create_release.outputs.version }}"
# name looks like: `databend_0.8.144~nightly_amd64.deb`
gh release upload ${version} databend_*.${{ matrix.packager }} --clobber
# bindings_python:
# if: inputs.stable
# needs: create_release
# uses: ./.github/workflows/bindings.python.yml
# secrets: inherit
# with:
# tag: ${{ needs.create_release.outputs.version }}
notify:
runs-on: ubuntu-latest
if: always()
needs:
- create_release
- build_default
- build_udf
- publish
- docker_all_in_one
- docker_service
- distribution
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Notify Release
uses: actions/github-script@v7
env:
JOBS_STATUS: ${{ join(needs.*.result, ',') }}
REPORT_WEBHOOK: ${{ secrets.RELEASE_REPORT_WEBHOOK }}
VERSION: ${{ needs.create_release.outputs.version }}
with:
script: |
const script = require('./.github/scripts/notify_release.js')
await script({context, core})
- name: Publish release
env:
GH_TOKEN: ${{ secrets.DATABEND_BOT_TOKEN }}
run: |
gh release edit ${{ needs.create_release.outputs.version }} --draft=false
deb:
runs-on: ubuntu-latest
if: inputs.stable
needs: [create_release, distribution]
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- uses: ./.github/actions/publish_deb
env:
GH_TOKEN: ${{ github.token }}
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: auto
AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
with:
version: ${{ needs.create_release.outputs.version }}
gpg_signing_key: ${{ secrets.GPG_KEY_DEB }}
sha256sums:
needs: [create_release, publish, distribution]
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: download sha256sums
uses: actions/download-artifact@v4
with:
pattern: sha256sums-*
merge-multiple: true
- shell: bash
run: |
for file in *.txt
do
cat ${file} >> sha256sums.txt
done
- name: Upload checksums
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
version="${{ needs.create_release.outputs.version }}"
gh release upload ${version} sha256sums.txt --clobber
sqlsmith:
needs: [create_release, notify]
runs-on: [self-hosted, X64, Linux, 4c8g, aws]
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Download artifacts
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: x86_64-unknown-linux-gnu
category: default
artifacts: meta,query,sqlsmith
- name: Run sqlsmith
timeout-minutes: 60
shell: bash
run: |
bash ./scripts/ci/ci-run-sqlsmith-tests.sh
- name: Upload failure
if: failure()
uses: ./.github/actions/artifact_failure
with:
name: test-sqlsmith
metachaos:
needs: [create_release, notify]
runs-on: [self-hosted, X64, Linux, 8c16g, aws]
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.create_release.outputs.sha }}
- name: Download artifacts
uses: ./.github/actions/artifact_download
with:
sha: ${{ github.sha }}
target: x86_64-unknown-linux-gnu
category: default
artifacts: meta,metactl,metaverifier
- uses: ./.github/actions/io_delay_chaos
timeout-minutes: 20
- name: Notify failure
if: failure()
uses: actions/github-script@v7
env:
REPORT_WEBHOOK: ${{ secrets.META_REPORT_WEBHOOK }}
TITLE: Meta Service chaos tests failed
with:
script: |
const script = require('./.github/scripts/notify_failure.js')
await script({context, core})
benchmark:
needs: [create_release, docker_service, notify]
uses: ./.github/workflows/reuse.benchmark.yml
secrets: inherit
with:
sha: ${{ github.sha }}
run_id: ${{ github.run_id }}
source: release
source_id: ${{ needs.create_release.outputs.version }}
version: ${{ needs.create_release.outputs.version }}
runner_provider: aws
target: all