Merge branch 'main' into databend-guru

.github/actions/build_linux/action.yml

@@ -30,7 +30,7 @@ runs:
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}
-        bypass_env_vars: RUSTFLAGS,RUST_LOG
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY
 
     - name: Cross setup
       if: startsWith(inputs.target, 'aarch64-')
@@ -64,49 +64,25 @@ runs:
         target=${{ inputs.target }}
         echo "BUILD_ARCH=${target/-unknown-linux-*}" >> $GITHUB_ENV
 
-    # build all binaries for debug
-    - name: Build Debug
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts == 'all'
-      shell: bash
-      run: |
-        artifacts="meta,metactl,metabench,query,sqllogictests"
-        for artifact in ${artifacts//,/ }; do
-          echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
-        done
-        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }}
-        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-*
-
-    - name: Build Debug for specific artifacts
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts != 'all'
-      shell: bash
-      run: |
-        artifacts="${{ inputs.artifacts }}"
-        for artifact in ${artifacts//,/ }; do
-          echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --bin databend-$artifact
-        done
-        ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
-
     - name: Build Release
-      if: env.BUILD_PROFILE == 'release' && inputs.artifacts == 'all'
+      if: inputs.artifacts == 'all'
       shell: bash
       run: |
         artifacts="meta,metactl,metabench,query,sqllogictests"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
         done
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 
     - name: Build Release for specific artifacts
-      if: env.BUILD_PROFILE == 'release' && inputs.artifacts != 'all'
+      if: inputs.artifacts != 'all'
       shell: bash
       run: |
         artifacts="${{ inputs.artifacts }}"
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --features ${{ inputs.features }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
         done
         ls -lh ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-$artifact
 
@@ -116,6 +92,14 @@ runs:
         readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
         ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
 
+    - name: Spilt Binary Symbols
+      shell: bash
+      run: |
+        objcopy --only-keep-debug ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug
+        chmod 0644 ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query.debug
+        strip --strip-debug --remove-section=.comment --remove-section=.note ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query
+        pushd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }} && objcopy --add-gnu-debuglink databend-query.debug databend-query && popd
+
     # - name: Compress Binaries with UPX
     #   if: env.BUILD_PROFILE == 'debug'
     #   uses: crazy-max/ghaction-upx@v2

.github/actions/build_macos/action.yml

@@ -32,7 +32,6 @@ runs:
         echo "JEMALLOC_SYS_WITH_LG_PAGE=14" >> $GITHUB_ENV
         echo "JEMALLOC_SYS_WITH_MALLOC_CONF=oversize_threshold:0,dirty_decay_ms:5000,muzzy_decay_ms:5000" >> $GITHUB_ENV
 
-
     - name: Setup build env
       shell: bash
       run: |
@@ -41,33 +40,15 @@ runs:
         echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
         echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
 
-    - name: Build Debug for all artifacts
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts == 'all'
-      shell: bash
-      run: cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }}
-
-    - name: Build Debug for specific artifacts
-      if: env.BUILD_PROFILE == 'debug' && inputs.artifacts != 'all'
-      shell: bash
-      run: |
-        artifacts="${{ inputs.artifacts }}"
-        echo "==> building libs ..."
-        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --lib
-        for artifact in ${artifacts//,/ }; do
-          echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --bin databend-$artifact
-        done
-
-    - name: Build Release
-      if: env.BUILD_PROFILE == 'release'
+    - name: Build
       shell: bash
       run: |
         artifacts="${{ inputs.artifacts }}"
         echo "==> building libs ..."
-        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --release --lib
+        cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --profile ${{ env.BUILD_PROFILE }} --lib
         for artifact in ${artifacts//,/ }; do
           echo "==> building databend-$artifact ..."
-          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --release --bin databend-$artifact
+          cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps build --target ${{ inputs.target }} --profile ${{ env.BUILD_PROFILE }} --bin databend-$artifact
         done
 
     - name: Upload artifact

.github/actions/pack_binaries/action.yml

@@ -0,0 +1,95 @@
+name: "Pack Binary"
+description: "Pack releases binaries"
+inputs:
+  target:
+    description: "Release target"
+    required: true
+  category:
+    description: "Release default/hdfs/udf/testsuite"
+    required: false
+    default: default
+  version:
+    description: "Release version"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download artifact
+      uses: ./.github/actions/artifact_download
+      with:
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+        category: ${{ inputs.category }}
+        path: distro/bin
+        artifacts: metactl,meta,query,query.debug
+    - name: Pack Binaries
+      id: pack_binaries
+      shell: bash
+      run: |
+        target=${{ inputs.target }}
+        version=${{ inputs.version }}
+        case ${{ inputs.category }} in
+          default)
+            pkg_name="databend-${version}-${target}"
+            ;;
+          *)
+          pkg_name="databend-${{ inputs.category }}-${version}-${target}"
+            ;;
+        esac
+        mkdir -p distro/{bin,configs,systemd,scripts}
+        cp ./scripts/distribution/systemd/databend-* distro/systemd/
+        cp ./scripts/distribution/configs/databend-* distro/configs/
+        cp ./scripts/distribution/release-readme.txt distro/readme.txt
+        cp -r ./scripts/distribution/local-scripts/* distro/scripts/
+        cp -r ./scripts/distribution/package-scripts/* distro/scripts/
+        tar -C ./distro --exclude='*.debug' -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
+        sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
+        echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
+    - name: post sha256
+      uses: actions/upload-artifact@v4
+      with:
+        name: sha256sums-${{ inputs.category }}-${{ inputs.target }}
+        path: sha256-${{ steps.pack_binaries.outputs.pkg_name }}.txt
+        retention-days: 1
+    - name: post binaries
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
+        path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
+        retention-days: 1
+    - name: Pack DBG Binaries
+      id: pack_dbg_binaries
+      shell: bash
+      run: |
+        target=${{ inputs.target }}
+        version=${{ inputs.version }}
+        case ${{ inputs.category }} in
+          default)
+            pkg_name="databend-${version}-${target}-dbg"
+            ;;
+          *)
+          pkg_name="databend-${{ inputs.category }}-${version}-${target}-dbg"
+            ;;
+        esac
+        mkdir -p distro/{bin,configs,systemd,scripts}
+        cp ./scripts/distribution/systemd/databend-* distro/systemd/
+        cp ./scripts/distribution/configs/databend-* distro/configs/
+        cp ./scripts/distribution/release-readme.txt distro/readme.txt
+        cp -r ./scripts/distribution/local-scripts/* distro/scripts/
+        cp -r ./scripts/distribution/package-scripts/* distro/scripts/
+        tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
+        sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
+        echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
+    - name: post dbg sha256
+      uses: actions/upload-artifact@v4
+      with:
+        name: sha256sums-${{ inputs.category }}-${{ inputs.target }}-gdb
+        path: sha256-${{ steps.pack_dbg_binaries.outputs.pkg_name }}.txt
+        retention-days: 1
+    - name: post dbg binaries
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
+        path: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
+        retention-days: 1

.github/actions/pack_distribution/action.yml

@@ -0,0 +1,94 @@
+name: "Pack Deb"
+description: "Pack releases deb"
+inputs:
+  arch:
+    description: "Release arch"
+    required: true
+  packager:
+    description: "Release default/hdfs/udf/testsuite"
+    required: false
+    default: default
+  version:
+    description: "Release version"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install nfpm@latest
+      shell: bash
+      run: |
+        curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz
+        tar xf nfpm.tar.gz
+        sudo mv nfpm /usr/local/bin
+        sudo chmod a+x /usr/local/bin/nfpm
+        rm nfpm.tar.gz
+    - name: Get target
+      id: target
+      shell: bash
+      run: |
+        echo 'target=${{ inputs.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT
+    - name: Download artifacts
+      uses: ./.github/actions/artifact_download
+      with:
+        sha: ${{ github.sha }}
+        target: ${{ steps.target.outputs.target }}
+        category: default
+        artifacts: metactl,meta,query,query.debug
+        path: distro/bin
+    - name: Build Packages
+      shell: bash
+      id: build_packages
+      run: |
+        export name="databend"
+        export version="${{ inputs.version }}"
+        export path="distro"
+        case "${{ inputs.arch }}" in
+          x86_64)
+            export arch="amd64"
+            ;;
+          aarch64)
+            export arch="arm64"
+            ;;
+        esac
+
+        deb_version=${version/-/.}
+        pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}.${{ inputs.packager }}"
+        mkdir -p distro/{bin,configs,systemd,scripts}
+        cp ./scripts/distribution/systemd/databend-* distro/systemd/
+        cp ./scripts/distribution/configs/databend-* distro/configs/
+        cp ./scripts/distribution/release-readme.txt distro/readme.txt
+        cp -r ./scripts/distribution/local-scripts/* distro/scripts/
+        cp -r ./scripts/distribution/package-scripts/* distro/scripts/
+        nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml)
+        echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
+    - name: Build dbg Packages
+      shell: bash
+      id: build_dbg_packages
+      run: |
+        export name="databend_dbg"
+        export version="${{ inputs.version }}"
+        export path="distro"
+        case "${{ inputs.arch }}" in
+          x86_64)
+            export arch="amd64"
+            ;;
+          aarch64)
+            export arch="arm64"
+            ;;
+        esac
+
+        deb_version=${version/-/.}
+        pkg_name="databend_${deb_version/v/}_${{ inputs.arch }}_dbg.${{ inputs.packager }}"
+        nfpm pkg --packager ${{ inputs.packager }} -t "$pkg_name" -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm-dbg.yaml)
+        echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
+    - name: Update release to github
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+      # Reference: https://cli.github.com/manual/gh_release_upload
+      run: |
+        version="${{ inputs.version }}"
+        # name looks like: `databend_0.8.144~nightly_amd64.deb`
+        gh release upload ${version} ${{ steps.build_packages.outputs.pkg_name }} --clobber
+        gh release upload ${version} ${{ steps.build_dbg_packages.outputs.pkg_name }} --clobber

.github/actions/publish_binary/action.yml

@@ -37,13 +37,19 @@ runs:
       shell: bash
       # Reference: https://cli.github.com/manual/gh_release_upload
       run: |
-        gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.* --clobber
+        gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.tar.gz --clobber
+        if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then
+          gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}-dbg.* --clobber
+        fi
 
     - name: Sync normal release to R2
       shell: bash
       if: inputs.category == 'default'
       run: |
         aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress
+        if [ -f ${{ steps.name.outputs.name }}-gdb.tar.gz ]; then
+          aws s3 cp ${{ steps.name.outputs.name }}-dbg.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}-dbg.tar.gz --no-progress
+        fi
         gh api /repos/databendlabs/databend/tags > tags.json
         aws s3 cp ./tags.json s3://repo/databend/tags.json
         gh api /repos/databendlabs/databend/releases > releases.json

.github/actions/setup_license/action.yml

@@ -5,6 +5,10 @@ inputs:
     description: "Self-hosted runner provider, aws or gcp"
     required: true
     default: "aws"
+  type:
+    description: "License type, enterprise or trial"
+    required: true
+    default: "trial"
 
 runs:
   using: "composite"
@@ -13,15 +17,15 @@ runs:
       if: inputs.runner_provider == 'aws'
       shell: bash
       run: |
-        aws s3 cp s3://databend-ci/misc/license.key license.key
-        aws s3 cp s3://databend-ci/misc/license.json license.json
+        aws s3 cp s3://databend-ci/misc/license-${{ inputs.type }}.key license.key
+        aws s3 cp s3://databend-ci/misc/license-${{ inputs.type }}.json license.json
 
     - name: Get License from GCS
       if: inputs.runner_provider == 'gcp'
       shell: bash
       run: |
-        gcloud storage cp gs://databend-ci/misc/license.key license.key
-        gcloud storage cp gs://databend-ci/misc/license.json license.json
+        gcloud storage cp gs://databend-ci/misc/license-${{ inputs.type }}.key license.key
+        gcloud storage cp gs://databend-ci/misc/license-${{ inputs.type }}.json license.json
 
     - name: Output License
       shell: bash

.github/scripts/bump_version.js

@@ -32,9 +32,10 @@ module.exports = async ({ github, context, core }) => {
       core.setFailed("Stable release must be triggered with a nightly tag");
     }
   } else {
+    core.setOutput("sha", context.sha);
     if (TAG) {
       core.setOutput("tag", TAG);
-      core.info(`Release create manually with tag ${TAG}`);
+      core.info(`Release create manually with tag ${TAG} (${context.sha})`);
     } else {
       let releases = await github.rest.repos.listReleases({
         owner: context.repo.owner,
@@ -52,7 +53,6 @@ module.exports = async ({ github, context, core }) => {
       let patch = (parseInt(result[3]) + 1).toString();
       let next_tag = `v${major}.${minor}.${patch}-nightly`;
       core.setOutput("tag", next_tag);
-      core.setOutput("sha", context.sha);
       core.info(`Nightly release ${next_tag} from ${tag} (${context.sha})`);
     }
   }

.github/workflows/bindings.python.yml

@@ -22,7 +22,7 @@ concurrency:
 
 jobs:
   linux:
-    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 8c16g, aws]
+    runs-on: [self-hosted, "${{ matrix.runner }}", Linux, 4c16g, aws]
     strategy:
       matrix:
         include: