Merge branch 'main' into chore/roll_back_signal_handler

databendlabs · Nov 13, 2024 · 1dce707 · 1dce707
2 parents 7d19be3 + 872627c
commit 1dce707
Show file tree

Hide file tree

Showing 55 changed files with 1,071 additions and 657 deletions.
diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml
@@ -30,7 +30,7 @@ runs:
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}
-        bypass_env_vars: RUSTFLAGS,RUST_LOG
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY
 
     - name: Cross setup
       if: startsWith(inputs.target, 'aarch64-')

diff --git a/.github/actions/setup_license/action.yml b/.github/actions/setup_license/action.yml
@@ -5,6 +5,10 @@ inputs:
     description: "Self-hosted runner provider, aws or gcp"
     required: true
     default: "aws"
+  type:
+    description: "License type, enterprise or trial"
+    required: true
+    default: "trial"
 
 runs:
   using: "composite"
@@ -13,15 +17,15 @@ runs:
       if: inputs.runner_provider == 'aws'
       shell: bash
       run: |
-        aws s3 cp s3://databend-ci/misc/license.key license.key
-        aws s3 cp s3://databend-ci/misc/license.json license.json
+        aws s3 cp s3://databend-ci/misc/license-${{ inputs.type }}.key license.key
+        aws s3 cp s3://databend-ci/misc/license-${{ inputs.type }}.json license.json
 
     - name: Get License from GCS
       if: inputs.runner_provider == 'gcp'
       shell: bash
       run: |
-        gcloud storage cp gs://databend-ci/misc/license.key license.key
-        gcloud storage cp gs://databend-ci/misc/license.json license.json
+        gcloud storage cp gs://databend-ci/misc/license-${{ inputs.type }}.key license.key
+        gcloud storage cp gs://databend-ci/misc/license-${{ inputs.type }}.json license.json
 
     - name: Output License
       shell: bash

diff --git a/.github/workflows/cloud.yml b/.github/workflows/cloud.yml
@@ -59,6 +59,8 @@ jobs:
       - name: Build Release
         uses: ./.github/actions/build_linux
         timeout-minutes: 60
+        env:
+          DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
         with:
           sha: ${{ needs.info.outputs.sha }}
           target: ${{ matrix.arch }}-unknown-linux-gnu

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -52,6 +52,7 @@ jobs:
     with:
       build_profile: release
       runner_provider: aws
+      license_type: trial
 
   linux_hive:
     needs: changes

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -44,6 +44,7 @@ jobs:
     with:
       build_profile: release
       runner_provider: aws
+      license_type: enterprise
 
   # macos:
   #   needs: linux

diff --git a/.github/workflows/merge_group.yml b/.github/workflows/merge_group.yml
@@ -56,6 +56,7 @@ jobs:
     with:
       build_profile: release
       runner_provider: aws
+      license_type: enterprise
 
   ready:
     if: always()

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -117,6 +117,7 @@ jobs:
         uses: ./.github/actions/build_linux
         env:
           DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }}
+          DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}
@@ -149,6 +150,7 @@ jobs:
         uses: ./.github/actions/build_linux
         env:
           DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }}
+          DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}
@@ -173,6 +175,7 @@ jobs:
         uses: ./.github/actions/build_linux
         env:
           DATABEND_RELEASE_VERSION: ${{ needs.create_release.outputs.version }}
+          DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.target }}

diff --git a/.github/workflows/reuse.linux.hive.yml b/.github/workflows/reuse.linux.hive.yml
@@ -4,15 +4,15 @@ on:
   workflow_call:
     inputs:
       build_profile:
-        description: 'Build profile, debug or release'
+        description: "Build profile, debug or release"
         type: string
         required: true
-        default: 'debug'
+        default: "debug"
       runner_provider:
-        description: 'Self-hosted runner provider, aws or gcp'
+        description: "Self-hosted runner provider, aws or gcp"
         type: string
         required: true
-        default: 'aws'
+        default: "aws"
 
 env:
   BUILD_PROFILE: ${{ inputs.build_profile }}
@@ -32,6 +32,8 @@ jobs:
           fetch-depth: 0
       - uses: ./.github/actions/build_linux
         timeout-minutes: 60
+        env:
+          DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}

diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml
@@ -13,14 +13,19 @@ on:
         type: string
         required: true
         default: "aws"
+      license_type:
+        description: "License type, enterprise or trial"
+        type: string
+        required: true
+        default: "trial"
 
 env:
   BUILD_PROFILE: ${{ inputs.build_profile }}
   RUNNER_PROVIDER: ${{ inputs.runner_provider }}
 
 jobs:
   check:
-    runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -50,6 +55,8 @@ jobs:
           fetch-depth: 0
       - uses: ./.github/actions/build_linux
         timeout-minutes: 60
+        env:
+          DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.arch }}-unknown-linux-gnu
@@ -75,6 +82,8 @@ jobs:
           fetch-depth: 0
       - uses: ./.github/actions/build_linux
         timeout-minutes: 60
+        env:
+          DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY: ${{ secrets.DATABEND_ENTERPRISE_LICENSE_PUBLIC_KEY }}
         with:
           sha: ${{ github.sha }}
           target: ${{ matrix.arch }}-unknown-linux-gnu
@@ -106,7 +115,7 @@ jobs:
   #         artifacts: query
 
   test_unit:
-    runs-on: [ self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}" ]
+    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -116,75 +125,76 @@ jobs:
         timeout-minutes: 60
 
   test_metactl:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_metactl
         timeout-minutes: 10
 
   test_compat_meta_query:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_compat_meta_query
         timeout-minutes: 10
 
   test_compat_fuse:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_compat_fuse
         timeout-minutes: 20
 
   test_compat_meta_meta:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_compat_meta_meta
         timeout-minutes: 20
 
   test_logs:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_logs
         timeout-minutes: 20
 
   test_meta_cluster:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_meta_cluster
         timeout-minutes: 10
 
   test_stateless_standalone:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateless_standalone_linux
         timeout-minutes: 15
 
   test_stateless_cluster:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_license
         with:
           runner_provider: ${{ inputs.runner_provider }}
+          type: ${{ inputs.license_type }}
       - uses: ./.github/actions/test_stateless_cluster_linux
         timeout-minutes: 15
 
   test_stateful_standalone:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateful_standalone_linux
@@ -196,13 +206,14 @@ jobs:
           name: test-stateful-standalone-linux
 
   test_stateful_cluster:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_license
         with:
           runner_provider: ${{ inputs.runner_provider }}
+          type: ${{ inputs.license_type }}
       - uses: ./.github/actions/test_stateful_cluster_linux
         timeout-minutes: 15
       - name: Upload failure
@@ -213,16 +224,16 @@ jobs:
 
   test_stateful_large_data:
     if: contains(github.event.pull_request.labels.*.name, 'ci-largedata')
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateful_large_data
         timeout-minutes: 60
 
   test_stateful_iceberg_rest:
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
-    needs: [ build, check ]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
+    needs: [build, check]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_stateful_iceberg_rest_standalone
@@ -243,13 +254,14 @@ jobs:
   #       continue-on-error: true
 
   test_ee_standalone:
-    needs: [ build, check ]
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    needs: [build, check]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_license
         with:
           runner_provider: ${{ inputs.runner_provider }}
+          type: ${{ inputs.license_type }}
       - uses: ./.github/actions/test_ee_standalone_linux
         timeout-minutes: 10
       - name: Upload failure
@@ -259,14 +271,15 @@ jobs:
           name: test-stateful-standalone-linux
 
   test_ee_standalone_background:
-    needs: [ build, check ]
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    needs: [build, check]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_bendsql
       - uses: ./.github/actions/setup_license
         with:
           runner_provider: ${{ inputs.runner_provider }}
+          type: ${{ inputs.license_type }}
       - uses: ./.github/actions/test_ee_standalone_background_linux
         timeout-minutes: 10
       - name: Upload failure
@@ -288,6 +301,7 @@ jobs:
   #      - uses: ./.github/actions/setup_license
   #        with:
   #          runner_provider: ${{ inputs.runner_provider }}
+  #          type: ${{ inputs.license_type }}
   #      - uses: ./.github/actions/test_ee_standalone_fake_time_linux
   #        timeout-minutes: 10
   #      - name: Upload failure
@@ -297,14 +311,15 @@ jobs:
   #          name: test-stateful-standalone-fake-time-linux
 
   test_ee_management_mode:
-    needs: [ build, check ]
-    runs-on: [ self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}" ]
+    needs: [build, check]
+    runs-on: [self-hosted, X64, Linux, 4c8g, "${{ inputs.runner_provider }}"]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup_bendsql
       - uses: ./.github/actions/setup_license
         with:
           runner_provider: ${{ inputs.runner_provider }}
+          type: ${{ inputs.license_type }}
       - uses: ./.github/actions/test_ee_management_mode_linux
         timeout-minutes: 10
       - name: Upload failure
@@ -314,9 +329,10 @@ jobs:
           name: test-ee-management-mode-linux
 
   sqllogic:
-    needs: [ build, check ]
+    needs: [build, check]
     uses: ./.github/workflows/reuse.sqllogic.yml
     secrets: inherit
     with:
       build_profile: ${{ inputs.build_profile }}
       runner_provider: ${{ inputs.runner_provider }}
+      license_type: ${{ inputs.license_type }}