From 92d458f8f758a0450dc2dfe8bc154bb3931381da Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 11:10:32 +0800 Subject: [PATCH] [Tool] Branch ubuntu image (backport #43499) (#43501) Signed-off-by: AndyZiYe Co-authored-by: andyziye <108652123+andyziye@users.noreply.github.com> Co-authored-by: AndyZiYe --- .github/workflows/ci-merged.yml | 9 +- .github/workflows/ci-pipeline-branch.yml | 175 ++++-- .github/workflows/ci-pipeline.yml | 522 +++++++++++------- .github/workflows/ci-report.yml | 433 +++++++++++++++ .github/workflows/inspection-pipeline.yml | 2 +- .../inspection-reusable-pipeline.yml | 2 +- 6 files changed, 887 insertions(+), 256 deletions(-) create mode 100644 .github/workflows/ci-report.yml diff --git a/.github/workflows/ci-merged.yml b/.github/workflows/ci-merged.yml index eb89be5b729dd..501df8a4fd292 100644 --- a/.github/workflows/ci-merged.yml +++ b/.github/workflows/ci-merged.yml @@ -85,6 +85,11 @@ jobs: runs-on: [self-hosted, normal] name: Thirdparty Update Image if: github.event.pull_request.merged == true + continue-on-error: false + strategy: + fail-fast: false + matrix: + linux: [ centos7, ubuntu ] env: PR_NUMBER: ${{ github.event.number }} BRANCH: ${{ github.base_ref }} @@ -103,11 +108,11 @@ jobs: - 'thirdparty/**' - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' - - name: update image + - name: update image (${{ matrix.linux }}) if: steps.changes.outputs.thirdparty == 'true' run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - ./bin/elastic-update-image.sh $BRANCH $PR_NUMBER + ./bin/elastic-update-image.sh $BRANCH $PR_NUMBER ${{ matrix.linux }} - name: Clean ENV if: always() diff --git a/.github/workflows/ci-pipeline-branch.yml b/.github/workflows/ci-pipeline-branch.yml index e7fac7d539368..2d592a40b0a30 100644 --- a/.github/workflows/ci-pipeline-branch.yml +++ b/.github/workflows/ci-pipeline-branch.yml @@ -16,14 +16,14 @@ concurrency: jobs: - sync-checker: + basic-checker: runs-on: [ self-hosted, normal ] name: RUN CHECKER env: PR_NUMBER: ${{ github.event.number }} + GH_TOKEN: ${{ github.token }} outputs: PASS: ${{ steps.check.outputs.pass }} - THIRDPARTY: ${{ steps.thirdparty-checker.outputs.thirdparty }} steps: - name: upload info env: @@ -56,25 +56,25 @@ jobs: run: | echo "pass=true" >> $GITHUB_OUTPUT - - uses: dorny/paths-filter@v2 - id: thirdparty-checker - if: always() - with: - filters: | - thirdparty: - - 'thirdparty/**' - - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' + - name: Check Blacklist + id: check_blacklist + env: + NO_SYNC: ${{ steps.check.outputs.pass }} + AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull >/dev/null + ./scripts/check-blacklist.sh be-checker: runs-on: ubuntu-latest - needs: sync-checker + needs: basic-checker name: BE FILTER - if: needs.sync-checker.outputs.PASS == 'true' || needs.sync-checker.outputs.THIRDPARTY == 'true' + if: needs.basic-checker.outputs.PASS == 'true' outputs: output1: ${{ steps.be-changes-info.outputs.be }} output2: ${{ steps.be-changes-info.outputs.thirdparty }} steps: - - uses: dorny/paths-filter@v2 + - uses: dorny/paths-filter@v3 id: changes with: filters: | @@ -109,7 +109,7 @@ jobs: mkdir -p ${{ github.workspace }} - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -130,7 +130,7 @@ jobs: git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; git checkout $BRANCH_NAME; git checkout -b merge_pr; - git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); + git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1); - name: Run Clang-Format run: | @@ -150,28 +150,33 @@ jobs: PR_NUMBER: ${{ github.event.number }} BRANCH: ${{ github.base_ref }} REPO: ${{ github.repository }} - outputs: - ip: ${{ steps.update-image.outputs.ip }} - instance_id: ${{ steps.update-image.outputs.instance_id }} - image_cache_id: ${{ steps.update-image.outputs.image_cache_id }} + continue-on-error: true + strategy: + fail-fast: false + matrix: + build_type: [ Release, ASAN ] steps: - name: clean run: | rm -rf ${{ github.workspace }} mkdir -p ${{ github.workspace }} - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Update Image + - name: Update Image (${{ matrix.build_type }}) id: update-image + if: needs.be-checker.outputs.output2 == 'true' + env: + linux_distro: ${{ matrix.build_type }} run: | - if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then - rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - ./bin/run-pr-update-image.sh - fi + cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + ./bin/run-pr-update-image.sh + + - name: Upload Thirdparty Result + uses: actions/upload-artifact@v4 + with: + name: THIRDPARTY-RESULT-${{ matrix.build_type }} + path: image_cache.info + retention-days: 1 + overwrite: true - name: Clean ENV if: always() @@ -182,10 +187,41 @@ jobs: fi rm -rf ${{ github.workspace }}/* + thirdparty-info: + runs-on: [ self-hosted, normal ] + needs: + - thirdparty-update + name: Thirdparty Info + outputs: + centos7_image_cache_id: ${{ steps.info.outputs.centos7_image_cache_id }} + ubuntu_image_cache_id: ${{ steps.info.outputs.ubuntu_image_cache_id }} + steps: + - name: Check Result + run: | + if [[ "${{ needs.thirdparty-update.result }}" == 'failure' ]]; then + echo "::error:: Thirdparty Update Error!" + exit 1 + fi + + - name: Download Thirdparty Artifact + uses: actions/download-artifact@v4 + with: + pattern: THIRDPARTY-RESULT-* + path: outputs + + - name: Read Info + id: info + if: needs.thirdparty-update.result == 'success' + run: | + image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-centos7/image_cache.info" || echo "") + echo "centos7_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT + image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-ubuntu/image_cache.info" || echo "") + echo "ubuntu_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT + be-ut: runs-on: [self-hosted, normal] - needs: [ be-checker, thirdparty-update ] - timeout-minutes: 90 + needs: [ be-checker, thirdparty-info ] + timeout-minutes: 180 name: BE UT env: PR_NUMBER: ${{ github.event.number }} @@ -204,7 +240,7 @@ jobs: run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then - export image_cache_id=${{ needs.thirdparty-update.outputs.image_cache_id }} + export image_cache_id=${{ needs.thirdparty-info.outputs.centos7_image_cache_id }} export image_tag=$BRANCH-$PR_NUMBER fi ./bin/elastic-ut.sh --pr ${PR_NUMBER} --module be --branch ${{ steps.branch.outputs.branch }} --repository ${{ github.repository }} @@ -216,29 +252,31 @@ jobs: eci rm ${{ steps.run_ut.outputs.ECI_ID }} - name: Upload log - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: BE UT LOG - path: ${{ steps.run_ut.outputs.RES_LOG }} + path: ${{ steps.run_ut.outputs.BE_LOG }} retention-days: 1 + overwrite: true - name: Clean ENV if: always() run: | rm -f ${{ steps.run_ut.outputs.RES_FILE }} rm -f ${{ steps.run_ut.outputs.RES_LOG }} + rm -rf ${{ steps.run_ut.outputs.BE_LOG }} rm -rf ${{ github.workspace }}/* fe-checker: runs-on: ubuntu-latest - needs: sync-checker + needs: basic-checker name: FE FILTER - if: needs.sync-checker.outputs.PASS == 'true' || needs.sync-checker.outputs.THIRDPARTY == 'true' + if: needs.basic-checker.outputs.PASS == 'true' outputs: output1: ${{ steps.fe-changes-info.outputs.fe }} steps: - - uses: dorny/paths-filter@v2 + - uses: dorny/paths-filter@v3 id: changes with: filters: | @@ -269,7 +307,7 @@ jobs: rm -rf ${{ github.workspace }} mkdir -p ${{ github.workspace }} - - uses: dorny/paths-filter@v2 + - uses: dorny/paths-filter@v3 id: java-file name: JAVA FILE with: @@ -285,7 +323,7 @@ jobs: echo ${{github.base_ref}} echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -300,7 +338,7 @@ jobs: git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; git checkout $BRANCH_NAME; git checkout -b merge_pr; - git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); + git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1); - name: Copy checkstyle files if: ${{ steps.java-file.outputs.java == 'true' }} @@ -329,7 +367,7 @@ jobs: BRANCH: ${{ github.base_ref }} GH_TOKEN: ${{ github.token }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -411,23 +449,6 @@ jobs: bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Checkout PR - run: | - BRANCH=${{steps.branch.outputs.branch}} - git config --global user.name "wanpengfei-git"; - git config --global user.email "wanpengfei91@163.com"; - git checkout $BRANCH; - git pull; - BRANCH_NAME="${BRANCH}-${PR_NUMBER}"; - git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; - git checkout $BRANCH_NAME; - git checkout -b merge_pr; - git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); - - name: UPDATE ECI & RUN UT id: run_ut shell: bash @@ -446,11 +467,12 @@ jobs: - name: Upload log if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: FE UT LOG path: ${{ steps.run_ut.outputs.RES_LOG }} retention-days: 1 + overwrite: true - name: Clean ENV if: always() @@ -460,3 +482,38 @@ jobs: rm -f ${{ steps.run_ut.outputs.RES_LOG }} rm -rf ${{ steps.run_ut.outputs.COV_DIR }} rm -rf ${{ github.workspace }}/* + + Teardown: + runs-on: [self-hosted, normal] + name: Teardown + needs: [ fe-ut, be-ut ] + if: always() + env: + PR_NUMBER: ${{ github.event.number }} + BRANCH: ${{ github.base_ref }} + steps: + - name: upload info + if: always() + run: | + echo $PR_NUMBER > pr_num.txt + GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha) + echo ${GITHUB_SHA} > head_sha.txt + + - name: Upload the PR number + uses: actions/upload-artifact@v4 + with: + name: pr_num + path: ./pr_num.txt + retention-days: 3 + overwrite: true + + - name: Upload the PR HEAD REF + uses: actions/upload-artifact@v4 + with: + name: head_sha + path: ./head_sha.txt + retention-days: 3 + + - name: Clean + run: | + rm -rf ${{ github.workspace }}/* diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9e38ce2d2ca22..5b30dce54487d 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -16,7 +16,7 @@ concurrency: jobs: - sync-checker: + basic-checker: runs-on: [self-hosted, normal] name: RUN CHECKER env: @@ -26,96 +26,64 @@ jobs: GH_TOKEN: ${{ github.token }} outputs: PASS: ${{ steps.check.outputs.pass }} - THIRDPARTY: ${{ steps.thirdparty-checker.outputs.thirdparty }} steps: - - name: upload info - env: - PR_NUMBER: ${{ github.event.number }} + - name: Sync Check + id: check + if: > + (!contains(github.event.pull_request.title, '(sync #') && + !contains(github.event.pull_request.labels.*.name, 'sync') && + (!startsWith(github.head_ref, github.base_ref) || !contains(github.head_ref, '-sync-'))) || + contains(github.event.pull_request.labels.*.name, 'force-check') + run: | + echo "pass=true" >> $GITHUB_OUTPUT + + - name: Upload info run: | echo $PR_NUMBER > pr_num.txt - GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha) - echo ${GITHUB_SHA} > head_sha.txt - name: Upload the PR number - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: pr_num path: ./pr_num.txt retention-days: 3 + overwrite: true - - name: Upload the PR HEAD REF - uses: actions/upload-artifact@v2 - with: - name: head_sha - path: ./head_sha.txt - retention-days: 3 - - - name: clean - run: | - rm -rf ${{ github.workspace }} - mkdir -p ${{ github.workspace }} - - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Checkout PR - run: | - cd /var/lib/ci-tool/ && git pull && ./bin/checkout.sh check - - - name: Clean ENV - if: always() - run: | - rm -rf ${{ github.workspace }}/* - - - name: Check - id: check - if: > - !contains(github.event.pull_request.title, '(sync #') && - !contains(github.event.pull_request.labels.*.name, 'sync') && - (!startsWith(github.head_ref, github.base_ref) || !contains(github.head_ref, '-sync-')) + - name: Check Blacklist + id: check_blacklist + env: + NO_SYNC: ${{ steps.check.outputs.pass }} + AUTHOR: ${{ github.event.pull_request.user.login }} run: | - echo "pass=true" >> $GITHUB_OUTPUT - - - uses: dorny/paths-filter@v2 - id: thirdparty-checker - if: always() - with: - filters: | - thirdparty: - - 'thirdparty/**' - - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull >/dev/null + ./scripts/check-blacklist.sh be-checker: runs-on: ubuntu-latest - needs: sync-checker + needs: basic-checker name: BE FILTER - if: needs.sync-checker.outputs.PASS == 'true' || needs.sync-checker.outputs.THIRDPARTY == 'true' + if: needs.basic-checker.outputs.PASS == 'true' outputs: - output1: ${{ steps.be-changes-info.outputs.be }} - output2: ${{ steps.be-changes-info.outputs.thirdparty }} + src_filter: ${{ steps.path-filter.outputs.be }} + test_filter: ${{ steps.path-filter.outputs.ut }} + thirdparty_filter: ${{ steps.path-filter.outputs.thirdparty }} steps: - - uses: dorny/paths-filter@v2 - id: changes + - uses: dorny/paths-filter@v3 + id: path-filter with: filters: | be: - - 'be/**' - - 'gensrc/**' - - 'run-be-ut.sh' - - 'build.sh' - - 'thirdparty/**' - - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' + - 'be/!(test)**' + - 'gensrc/**' + - 'run-be-ut.sh' + - 'build.sh' + - 'thirdparty/**' + - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' + ut: + - 'be/test/**' thirdparty: - - 'thirdparty/**' - - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' - - - name: BE CHECK INFO - id: be-changes-info - run: | - echo "be=${{ steps.changes.outputs.be }}" >> $GITHUB_OUTPUT - echo "thirdparty=${{ steps.changes.outputs.thirdparty }}" >> $GITHUB_OUTPUT + - 'thirdparty/**' + - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' clang-format: runs-on: [self-hosted, light] @@ -123,7 +91,7 @@ jobs: name: Clang-Format env: PR_NUMBER: ${{ github.event.number }} - if: ${{ needs.be-checker.outputs.output1 == 'true' }} + if: needs.be-checker.outputs.src_filter == 'true' || needs.be-checker.outputs.test_filter == 'true' steps: - name: clean run: | @@ -131,7 +99,7 @@ jobs: mkdir -p ${{ github.workspace }} - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -152,7 +120,7 @@ jobs: git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; git checkout $BRANCH_NAME; git checkout -b merge_pr; - git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); + git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1); - name: Run Clang-Format run: | @@ -168,68 +136,96 @@ jobs: runs-on: [self-hosted, normal] needs: [ be-checker, clang-format ] name: Thirdparty Update + continue-on-error: true + strategy: + fail-fast: false + matrix: + linux: [ centos7, ubuntu ] env: PR_NUMBER: ${{ github.event.number }} BRANCH: ${{ github.base_ref }} REPO: ${{ github.repository }} - outputs: - ip: ${{ steps.update-image.outputs.ip }} - instance_id: ${{ steps.update-image.outputs.instance_id }} - image_cache_id: ${{ steps.update-image.outputs.image_cache_id }} steps: - name: clean run: | rm -rf ${{ github.workspace }} mkdir -p ${{ github.workspace }} - - name: Checkout Code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Update Image + - name: Update Image (${{ matrix.linux }}) id: update-image + if: needs.be-checker.outputs.thirdparty_filter == 'true' + env: + linux_distro: ${{ matrix.linux }} run: | - if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then - rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - ./bin/run-pr-update-image.sh - fi + cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + ./bin/run-pr-update-image.sh + + - name: Upload Thirdparty Result + uses: actions/upload-artifact@v4 + with: + name: THIRDPARTY-RESULT-${{ matrix.linux }} + path: image_cache.info + retention-days: 1 + overwrite: true - name: Clean ENV - if: always() + if: always() && needs.be-checker.outputs.thirdparty_filter == 'true' run: | - if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then - cd ci-tool && source lib/init.sh - ./bin/elastic-cluster.sh --delete - fi + cd ci-tool && source lib/init.sh + ./bin/elastic-cluster.sh --delete rm -rf ${{ github.workspace }}/* + thirdparty-info: + runs-on: [self-hosted, normal] + needs: + - thirdparty-update + name: Thirdparty Info + outputs: + centos7_image_cache_id: ${{ steps.info.outputs.centos7_image_cache_id }} + ubuntu_image_cache_id: ${{ steps.info.outputs.ubuntu_image_cache_id }} + steps: + - name: Check Result + run: | + if [[ "${{ needs.thirdparty-update.result }}" == 'failure' ]]; then + echo "::error:: Thirdparty Update Error!" + exit 1 + fi + + - name: Download Thirdparty Artifact + uses: actions/download-artifact@v4 + with: + pattern: THIRDPARTY-RESULT-* + path: outputs + + - name: Read Info + id: info + if: needs.thirdparty-update.result == 'success' + run: | + image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-centos7/image_cache.info" || echo "") + echo "centos7_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT + image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-ubuntu/image_cache.info" || echo "") + echo "ubuntu_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT + be-ut: runs-on: [self-hosted, normal] - needs: [ be-checker, thirdparty-update ] + needs: thirdparty-info timeout-minutes: 90 name: BE UT env: PR_NUMBER: ${{ github.event.number }} BRANCH: ${{ github.base_ref }} steps: - - name: BRANCH INFO - id: branch - run: | - echo ${{github.base_ref}} - echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT - - - name: UPDATE ECI & RUN UT + - name: INIT ECI & RUN UT id: run_ut shell: bash timeout-minutes: 90 run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then - export image_cache_id=${{ needs.thirdparty-update.outputs.image_cache_id }} + if [[ "${{ needs.thirdparty-info.outputs.centos7_image_cache_id }}" != '' ]]; then + export image_cache_id=${{ needs.thirdparty-info.outputs.centos7_image_cache_id }} export image_tag=$BRANCH-$PR_NUMBER fi - ./bin/elastic-ut.sh --pr ${PR_NUMBER} --module be --repository ${{ github.repository }} + ./bin/elastic-ut.sh --pr ${PR_NUMBER} --module be --repository ${{ github.repository }} --with-gcov - name: clean ECI if: always() @@ -237,24 +233,27 @@ jobs: echo ${{ steps.run_ut.outputs.ECI_ID }} eci rm ${{ steps.run_ut.outputs.ECI_ID }} - - name: Upload log - uses: actions/upload-artifact@v3 - if: always() + - name: Upload Log + uses: actions/upload-artifact@v4 + if: always() && steps.run_ut.outcome == 'failure' with: name: BE UT LOG - path: ${{ steps.run_ut.outputs.RES_LOG }} + path: ${{ steps.run_ut.outputs.BE_LOG }} retention-days: 1 + overwrite: true - name: Clean ENV if: always() run: | rm -f ${{ steps.run_ut.outputs.RES_FILE }} rm -f ${{ steps.run_ut.outputs.RES_LOG }} + rm -rf ${{ steps.run_ut.outputs.BE_LOG }} rm -rf ${{ github.workspace }}/* clang-tidy: runs-on: [self-hosted, normal] - needs: [ be-checker, thirdparty-update ] + needs: [ be-checker, thirdparty-info ] + if: success() && needs.be-checker.outputs.src_filter == 'true' timeout-minutes: 90 name: Clang-Tidy env: @@ -278,8 +277,8 @@ jobs: timeout-minutes: 60 run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then - export image_cache_id=${{ needs.thirdparty-update.outputs.image_cache_id }} + if [[ "${{ needs.be-checker.outputs.thirdparty_filter }}" == 'true' ]]; then + export image_cache_id=${{ needs.thirdparty-info.outputs.centos7_image_cache_id }} export image_tag=$BRANCH-$PR_NUMBER fi ./bin/elastic-build.sh --pr ${PR_NUMBER} --repository ${{ github.repository }} --be --clang-tidy @@ -299,35 +298,33 @@ jobs: fe-checker: runs-on: ubuntu-latest - needs: sync-checker + needs: basic-checker name: FE FILTER - if: needs.sync-checker.outputs.PASS == 'true' || needs.sync-checker.outputs.THIRDPARTY == 'true' + if: needs.basic-checker.outputs.PASS == 'true' outputs: - output1: ${{ steps.fe-changes-info.outputs.fe }} + src_filter: ${{ steps.path-filter.outputs.fe }} + test_filter: ${{ steps.path-filter.outputs.ut }} + java_filter: ${{ steps.path-filter.outputs.java }} steps: - - uses: dorny/paths-filter@v2 - id: changes + - uses: dorny/paths-filter@v3 + id: path-filter with: filters: | fe: - - 'fe/**' + - 'fe/!(**/test/**)**' - 'gensrc/**' - - 'run-fe-ut.sh' - 'java-extensions/**' - 'build.sh' - - 'test/**' - - - run: echo ${{ steps.changes.outputs.fe }} - - - name: FE CHECK INFO - id: fe-changes-info - run: | - echo "fe=${{ steps.changes.outputs.fe }}" >> $GITHUB_OUTPUT + ut: + - 'fe/**/test/**' + - 'run-fe-ut.sh' + java: + - '**.java' fe-codestyle-check: runs-on: ubuntu-latest needs: fe-checker - if: needs.fe-checker.outputs.output1 == 'true' + if: needs.fe-checker.outputs.src_filter == 'true' || needs.fe-checker.outputs.test_filter == 'true' name: FE Code Style Check env: PR_NUMBER: ${{ github.event.number }} @@ -337,23 +334,13 @@ jobs: rm -rf ${{ github.workspace }} mkdir -p ${{ github.workspace }} - - uses: dorny/paths-filter@v2 - id: java-file - name: JAVA FILE - with: - filters: | - java: - - '**.java' - - - run: echo ${{ steps.java-file.outputs.java }} - - name: BRANCH INFO id: branch run: | echo ${{github.base_ref}} echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -368,16 +355,16 @@ jobs: git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; git checkout $BRANCH_NAME; git checkout -b merge_pr; - git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); + git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1); - name: Copy checkstyle files - if: ${{ steps.java-file.outputs.java == 'true' }} + if: needs.fe-checker.outputs.java_filter == 'true' run: | cp fe/checkstyle* . - name: Run java checkstyle - if: ${{ steps.java-file.outputs.java == 'true' }} - uses: dbelyaev/action-checkstyle@v0.7.5 + if: needs.fe-checker.outputs.java_filter == 'true' + uses: dbelyaev/action-checkstyle@v1.5.6 with: workdir: "./fe" checkstyle_config: checkstyle.xml @@ -389,12 +376,15 @@ jobs: sonarcloud-fe-checker: runs-on: ubuntu-latest needs: fe-checker - if: needs.fe-checker.outputs.output1 == 'true' + if: needs.fe-checker.outputs.src_filter == 'true' && github.repository == 'StarRocks/starrocks' name: FE Sonarcloud Check env: PR_NUMBER: ${{ github.event.number }} + CODE_PATH: ${{ github.workspace }} + BRANCH: ${{ github.base_ref }} + GH_TOKEN: ${{ github.token }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -417,10 +407,10 @@ jobs: git checkout -b merge_pr; git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: - java-version: 11 + java-version: 17 distribution: 'adopt' - name: Cache SonarCloud packages @@ -479,14 +469,6 @@ jobs: bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Checkout PR - run: | - cd /var/lib/ci-tool/ && git pull && ./bin/checkout.sh - - name: UPDATE ECI & RUN UT id: run_ut shell: bash @@ -499,17 +481,19 @@ jobs: if: always() run: | echo ${{ steps.run_ut.outputs.ECI_ID }} - echo ">>> Dmesg info:" + echo "::group::>>> Dmesg info:" eci exec ${{ steps.run_ut.outputs.ECI_ID }} bash -c "dmesg -T" + echo "::endgroup::" eci rm ${{ steps.run_ut.outputs.ECI_ID }} - name: Upload log if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: FE UT LOG path: ${{ steps.run_ut.outputs.RES_LOG }} retention-days: 1 + overwrite: true - name: Clean ENV if: always() @@ -520,9 +504,24 @@ jobs: rm -rf ${{ steps.run_ut.outputs.COV_DIR }} rm -rf ${{ github.workspace }}/* + test-checker: + runs-on: ubuntu-latest + needs: basic-checker + name: TEST FILTER + if: needs.basic-checker.outputs.PASS == 'true' + outputs: + output1: ${{ steps.path-filter.outputs.test }} + steps: + - uses: dorny/paths-filter@v3 + id: path-filter + with: + filters: | + test: + - 'test/**' + build: runs-on: [self-hosted, normal] - needs: [be-checker, fe-checker, be-ut, clang-tidy, fe-ut, thirdparty-update] + needs: [be-checker, fe-checker, test-checker, clang-tidy, fe-ut, thirdparty-info] name: BUILD env: PR_NUMBER: ${{ github.event.number }} @@ -530,7 +529,13 @@ jobs: outputs: build_output_tar: ${{ steps.run_build.outputs.OUTPUT_TAR }} base_version: ${{ steps.run_build.outputs.BASE_VERSION }} - if: always() && (needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success' && needs.fe-ut.result == 'success') || (needs.be-checker.outputs.output1 != 'true' && needs.fe-ut.result == 'success') || (needs.fe-checker.outputs.output1 != 'true' && needs.be-ut.result == 'success' && needs.clang-tidy.result == 'success') + is_self_build: ${{ steps.run_build.outputs.is_self_build }} + if: > + always() && + (needs.clang-tidy.result == 'success' && needs.fe-ut.result == 'success') || + (needs.be-checker.outputs.src_filter != 'true' && needs.fe-checker.outputs.src_filter == 'true' && needs.fe-ut.result == 'success') || + (needs.fe-checker.outputs.src_filter != 'true' && needs.be-checker.outputs.src_filter == 'true' && needs.clang-tidy.result == 'success') || + (needs.be-checker.outputs.src_filter != 'true' && needs.fe-checker.outputs.src_filter != 'true' && needs.test-checker.outputs.output1 == 'true') steps: - name: CLEAN run: | @@ -539,20 +544,47 @@ jobs: - name: BRANCH INFO id: branch run: | - echo ${{github.base_ref}} - echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT + repo="${{ github.repository }}" + bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> "$GITHUB_ENV" + + - name: Rebuild Checker + id: rebuild-checker + env: + be_change: ${{ needs.be-checker.outputs.src_filter }} + fe_change: ${{ needs.fe-checker.outputs.src_filter }} + build_type: Release + run: | + echo "package=${package}" >> "$GITHUB_ENV" + if [[ "${be_change}" != "true" && "${fe_change}" != "true" ]]; then + oss_path=oss://${bucket_prefix}-ci-release/daily/commit/${BRANCH}/${build_type} + package=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "tar.gz" | sort -n -r | head -n 1 | awk '{print $NF}') + echo "package=${package}" >> $GITHUB_ENV + fi - name: UPDATE ECI & RUN BUILD id: run_build shell: bash timeout-minutes: 90 + env: + package: ${{ env.package }} run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then - export image_cache_id=${{ needs.thirdparty-update.outputs.image_cache_id }} - export image_tag=$BRANCH-$PR_NUMBER + if [[ "${{ needs.be-checker.outputs.thirdparty_filter }}" == 'true' ]]; then + export image_cache_id=${{ needs.thirdparty-info.outputs.centos7_image_cache_id }} + export image_tag=$BRANCH-$PR_NUMBER + fi + + if [[ ${package} == "" ]]; then + ./bin/elastic-build.sh --pr ${PR_NUMBER} --repository ${{ github.repository }} --with-gcov + echo "is_self_build=true" >> $GITHUB_OUTPUT + else + echo "Use latest tar: ${package}" + echo "OUTPUT_TAR=${package}" >> "$GITHUB_OUTPUT" + echo "BASE_VERSION=" >> "$GITHUB_OUTPUT" + echo "ECI_ID=" >> "$GITHUB_OUTPUT" + echo "is_self_build=false" >> $GITHUB_OUTPUT fi - ./bin/elastic-build.sh --pr ${PR_NUMBER} --repository ${{ github.repository }} - name: build result run: | @@ -560,14 +592,15 @@ jobs: echo ${{ steps.run_build.outputs.BASE_VERSION }} > ./base_version.txt - name: Upload the Base Version - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: base_version path: ./base_version.txt retention-days: 3 + overwrite: true - name: clean ECI - if: always() + if: always() && steps.run_build.outputs.ECI_ID != '' run: | echo ${{ steps.run_build.outputs.ECI_ID }} eci rm ${{ steps.run_build.outputs.ECI_ID }} @@ -586,15 +619,27 @@ jobs: name: DEPLOY SR env: PR_NUMBER: ${{ github.event.number }} - CLUSTER_NAME: ci-admit outputs: fe: ${{steps.deploy_sr.outputs.fe}} be: ${{steps.deploy_sr.outputs.be}} deploy_conf_file: ${{steps.apply_resource.outputs.deploy_conf_file}} + cluster_name: ${{steps.deploy_sr.outputs.cluster_name}} + be_list: ${{steps.deploy_sr.outputs.be_list}} + shared_data: ${{steps.choose-mode.outputs.cloud}} + is_self_build: ${{ needs.build.outputs.is_self_build }} steps: - name: Clean Workspace uses: AutoModality/action-clean@v1.1.0 + - name: Choose mode + id: choose-mode + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + cluster_type=$(./bin/choose-mode.sh) + if [[ "${cluster_type}" == "cloud" ]]; then + echo "cloud=true" >> "$GITHUB_OUTPUT" + fi + - name: BRANCH INFO id: branch run: | @@ -603,28 +648,37 @@ jobs: - name: Apply for resources id: apply_resource + env: + CLUSTER_NAME: ci-admit run: | - rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh - ./bin/elastic-cluster.sh --template ci-admit + cd ci-tool && source lib/init.sh + ./bin/elastic-cluster.sh --template ${CLUSTER_NAME} cp conf/starrocks_deploy.conf /var/local/env/${PR_NUMBER}-starrocks_deploy.conf echo "deploy_conf_file=/var/local/env/${PR_NUMBER}-starrocks_deploy.conf" >> $GITHUB_OUTPUT - name: Deploy SR id: deploy_sr + env: + CLUSTER_NAME: "${{ steps.choose-mode.outputs.cloud == 'true' && 'ci-admit-cloud' || 'ci-admit' }}" + repo: ${{ github.repository }} run: | cd ci-tool && source lib/init.sh - repo="${{ github.repository }}" - bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` - tar_path="oss://${bucket_prefix}-ci-release/${{steps.branch.outputs.branch}}/Release/pr/${{needs.build.outputs.build_output_tar}}" - echo ${tar_path} + + if [[ "${{ needs.build.outputs.is_self_build }}" == 'true' ]]; then + bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` + tar_path="oss://${bucket_prefix}-ci-release/${{steps.branch.outputs.branch}}/Release/pr/${{needs.build.outputs.build_output_tar}}" + else + tar_path="${{ needs.build.outputs.build_output_tar }}" + fi + + echo "##### Package: ${tar_path}" >> $GITHUB_STEP_SUMMARY + echo "cluster_name=${CLUSTER_NAME}" >> $GITHUB_OUTPUT ./bin/deploy-cluster.sh -c ${CLUSTER_NAME} --output ${tar_path} --with-coverage - - run: | - echo "FE: ${{steps.deploy_sr.outputs.fe}}, BE: ${{steps.deploy_sr.outputs.be}}" - - name: Clean ENV if: always() run: | + echo "FE: ${{steps.deploy_sr.outputs.fe}}, BE: ${{steps.deploy_sr.outputs.be}}" rm -rf ${{ github.workspace }}/* SQL-Tester: @@ -632,20 +686,22 @@ jobs: name: SQL-Tester needs: [ build, deploy ] if: always() && needs.deploy.result == 'success' - timeout-minutes: 30 + timeout-minutes: 60 env: PR_NUMBER: ${{ github.event.number }} - CLUSTER_NAME: ci-admit + CLUSTER_NAME: ${{ needs.deploy.outputs.cluster_name }} FE_NODE: ${{ needs.deploy.outputs.fe }} BE_NODE: ${{ needs.deploy.outputs.be }} - BASE_VERSION: ${{needs.build.outputs.base_version}} + BASE_VERSION: ${{ needs.build.outputs.base_version }} + outputs: + MYSQL_ECI_ID: ${{ steps.run_case.outputs.MYSQL_ECI_ID }} steps: - name: CLEAN run: | rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -668,9 +724,12 @@ jobs: git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; git checkout $BRANCH_NAME; git checkout -b merge_pr; - git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); + git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1); - name: Run Case (${{ needs.deploy.outputs.fe }}) + id: run_case + env: + is_self_build: ${{ needs.deploy.outputs.is_self_build }} run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh ./bin/run-sql-tester.sh @@ -684,12 +743,65 @@ jobs: ossutil64 --config-file ~/.ossutilconfig cp test/ oss://${bucket_prefix}-ci-release/$branch/Release/pr/SQL-Tester-XML/${PR_NUMBER}/ --include "*.xml" --recursive --force --tagging="type=ci" - name: Upload log - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: SQL-Tester Log path: test/log/ retention-days: 1 + overwrite: true + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ github.workspace }}/* + + restart-fe: + runs-on: [ self-hosted, normal ] + name: Restart FE + needs: [ deploy, SQL-Tester ] + if: always() && needs.SQL-Tester.result == 'success' + env: + PR_NUMBER: ${{ github.event.number }} + CLUSTER_NAME: ${{ needs.deploy.outputs.cluster_name }} + FE_NODE: ${{ needs.deploy.outputs.fe }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: Restart FE (${{needs.deploy.outputs.fe}}) + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + ./bin/restart-fe.sh --pr ${PR_NUMBER} --branch ${{ github.base_ref }} --build Release --repository ${{ github.repository }} + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ github.workspace }}/* + + restart-be: + runs-on: [ self-hosted, normal ] + name: Restart BE + needs: [ deploy, SQL-Tester ] + if: always() && needs.SQL-Tester.result == 'success' + timeout-minutes: 20 + env: + PR_NUMBER: ${{ github.event.number }} + CLUSTER_NAME: ${{ needs.deploy.outputs.cluster_name }} + FE_NODE: ${{ needs.deploy.outputs.fe }} + BE_NODE: ${{ needs.deploy.outputs.be }} + BE_LIST: ${{ needs.deploy.outputs.be_list }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: Restart BE (${{needs.deploy.outputs.be_list}}) + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + [[ "${{ needs.deploy.outputs.is_self_build }}" != "true" ]] && skip_gcov="--skip-gcov" + ./bin/system-cov-be.sh --pr ${PR_NUMBER} --branch ${{ github.base_ref }} --build Release --repository ${{ github.repository }} ${skip_gcov} - name: Clean ENV if: always() @@ -697,14 +809,14 @@ jobs: rm -rf ${{ github.workspace }}/* admit: - runs-on: [self-hosted, normal] + runs-on: [ self-hosted, normal ] name: ADMIT TEST - needs: [ deploy, SQL-Tester ] - if: always() && needs.deploy.result == 'success' && needs.SQL-Tester.result == 'success' + needs: [ deploy, restart-fe, restart-be ] + if: always() && needs.restart-fe.result == 'success' && needs.restart-be.result == 'success' && needs.deploy.outputs.is_self_build == 'true' timeout-minutes: 40 env: PR_NUMBER: ${{ github.event.number }} - CLUSTER_NAME: ci-admit + CLUSTER_NAME: ${{ needs.deploy.outputs.cluster_name }} FE_NODE: ${{ needs.deploy.outputs.fe }} BE_NODE: ${{ needs.deploy.outputs.be }} steps: @@ -722,15 +834,18 @@ jobs: - name: Run Case run: | - echo "FE: ${{needs.deploy.outputs.fe}}, BE: ${{needs.deploy.outputs.be}}" - rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh cd ${{ github.workspace }} - rm -rf ./StarRocksTest && cp -rf /var/lib/StarRocksTest ./StarRocksTest && cd ./StarRocksTest && git pull + rm -rf ./StarRocksTest && cp -rf /var/lib/StarRocksTest ./StarRocksTest && cd ./StarRocksTest && git pull >/dev/null cd ${{ github.workspace }}/ci-tool - ./bin/run-admit.sh --pr ${PR_NUMBER} --branch ${{steps.branch.outputs.branch}} --build Release --repository ${{ github.repository }} + if [[ "${{ needs.deploy.outputs.shared_data }}" == "true" ]]; then + cluster_type=cloud + else + cluster_type=native + fi + ./bin/run-admit-single.sh --pr ${PR_NUMBER} --branch ${{steps.branch.outputs.branch}} --build Release --repository ${{ github.repository }} --cluster ${cluster_type} - name: Upload Admit XML if: success() || failure() @@ -746,26 +861,47 @@ jobs: rm -rf ${{ github.workspace }}/* Teardown: - runs-on: [self-hosted, normal] + runs-on: [ self-hosted, quick ] name: Teardown - needs: [ deploy, admit ] - if: always() && needs.deploy.outputs.deploy_conf_file != '' + needs: [ deploy, SQL-Tester, admit ] + if: always() env: PR_NUMBER: ${{ github.event.number }} BRANCH: ${{ github.base_ref }} CONF_FILE: ${{ needs.deploy.outputs.deploy_conf_file }} steps: + - name: Upload info + run: | + echo $PR_NUMBER > pr_num.txt + + - name: Upload the PR number + uses: actions/upload-artifact@v4 + with: + name: pr_num + path: ./pr_num.txt + retention-days: 3 + overwrite: true + - name: Backup SR Info + if: needs.deploy.outputs.deploy_conf_file != '' + id: backup run: | rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh ./bin/backup_log_cores.sh --branch $BRANCH --pr $PR_NUMBER --build Release --conf $CONF_FILE --repository ${{ github.repository }} - name: Clean ECS + if: steps.backup.outcome == 'success' run: | cd ci-tool && source lib/init.sh ./bin/elastic-cluster.sh --delete + - name: clean ECI + if: always() && needs.SQL-Tester.outputs.MYSQL_ECI_ID != '' + run: | + eci rm ${{ needs.SQL-Tester.outputs.MYSQL_ECI_ID }} + - name: Clean + if: always() run: | rm -f $CONF_FILE - rm -rf ${{ github.workspace }}/* + rm -rf ${{ github.workspace }}/* \ No newline at end of file diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml new file mode 100644 index 0000000000000..50f2b53cc6070 --- /dev/null +++ b/.github/workflows/ci-report.yml @@ -0,0 +1,433 @@ +name: CI Report +run-name: CI Report(#${{ github.event.workflow_run.id }}) + +on: + workflow_run: + workflows: [ "CI PIPELINE", "CI PIPELINE - BRANCH" ] + types: + - completed + +permissions: + checks: write + actions: write + contents: write + deployments: write + discussions: write + issues: write + packages: write + pages: write + pull-requests: write + repository-projects: write + security-events: write + statuses: write + +env: + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + +jobs: + INFO: + runs-on: [ self-hosted, quick ] + if: > + github.event.workflow_run.event == 'pull_request' && + (github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure') + outputs: + PR_NUMBER: ${{ steps.pr_details.outputs.PR_NUMBER }} + BASE_REF: ${{ steps.pr_details.outputs.BASE_REF }} + PR_STATE: ${{ steps.pr_details.outputs.PR_STATE }} + steps: + - run: | + sleep 10 + + - name: Download workflow artifact - PR + uses: dawidd6/action-download-artifact@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ github.event.workflow_run.id }} + name: 'pr_num' + + - name: Read the pr num file + id: pr_num_reader + uses: juliangruber/read-file-action@v1 + with: + path: ./pr_num.txt + + - name: PR DETAILS + id: pr_details + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ steps.pr_num_reader.outputs.content }} + REPO: ${{ github.repository }} + run: | + PR_NUMBER=`echo ${PR_NUMBER}` + pr_info=`gh pr view ${PR_NUMBER} -R ${REPO} --json baseRefName,state` + base_ref=$(echo ${pr_info} | jq -r .baseRefName) + pr_state=$(echo ${pr_info} | jq -r .state) + echo "BASE_REF=${base_ref}" >> $GITHUB_OUTPUT + echo "PR_STATE=${pr_state}" >> $GITHUB_OUTPUT + echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT + + - name: INFO + id: info + run: | + echo ${{ steps.pr_details.outputs.PR_NUMBER }} + echo ${{ steps.pr_details.outputs.BASE_REF }} + + FE-REPORT: + runs-on: [ self-hosted, quick ] + needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + BASE_REF: ${{ needs.INFO.outputs.BASE_REF }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: INFO + id: info + run: | + repo="${{ github.repository }}" + bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT + + - name: Download FE UT XML + id: download-ut-xml + env: + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + run: | + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER} + size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/fe_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}') + echo "size=${size}" >> $GITHUB_OUTPUT + if [[ "$size" != "0" ]]; then + mkdir fe && cd fe + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/ . --recursive + tar zxf fe_ut_report.tar.gz + elif [[ "${BASE_REF}" == "main" ]]; then + rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover /var/local/env/empty_cov_result --github-token ${{ secrets.GITHUB_TOKEN }} \ + --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco + fi + + - name: Publish UT Report + uses: mikepenz/action-junit-report@v3 + id: publish_report + if: steps.download-ut-xml.outputs.size != '0' + env: + github_token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} + with: + check_name: 'FE UT Report' + detailed_summary: true + fail_on_failure: true + commit: ${{ github.event.workflow_run.head_sha }} + report_paths: ./fe/fe-core/target/surefire-reports/*.xml + + - name: Merge FE Coverage + id: merge_report + if: steps.publish_report.outcome == 'success' && env.BASE_REF == 'main' + env: + fe_path: ${{ github.workspace }}/fe + run: | + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + if [[ -e ${fe_path}/test.exec ]]; then + java -jar package/jacococli.jar merge ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/test.exec --destfile ${fe_path}/merge.exec + else + cp ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/merge.exec + fi + + - name: Generate XML Report + id: generate-xml-report + env: + package_path: ${{ github.workspace }}/ci-tool/package + fe_path: ${{ github.workspace }}/fe + fe_core_path: ${{ github.workspace }}/fe/fe-core + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + if: steps.merge_report.outcome == 'success' + run: | + rm -rf result + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + cd ${fe_core_path}/target/classes; + rm -rf org; + cd com/starrocks && rm -rf thrift proto sql/parser builtins common/Version.class; + cd ${{ github.workspace }}; + java -jar $package_path/jacococli.jar report ${fe_path}/merge.exec --classfiles ${fe_core_path}/target/classes/ \ + --html ./result --xml ${{ github.workspace }}/coverage.xml \ + --sourcefiles ${fe_core_path}/src/main/java/ --encoding utf-8 --name fe-coverage + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}/ + ossutil64 --config-file ~/.ossutilconfig cp coverage.xml ${oss_path} -f + + # Incremental Total Coverage + - name: Publish Incremental Coverage Report - Total + if: steps.generate-xml-report.outcome == 'success' + env: + fe_path: ${{ github.workspace }}/fe + run: | + rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + if [[ -e "${fe_path}/diff.txt" ]]; then + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover ${{ github.workspace }}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \ + --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco \ + -d ${fe_path}/diff.txt -dt file + else + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover ${{ github.workspace }}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \ + --repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco + fi + rm -rf coverchecker + rm -rf ci-tools + + # total coverage + - name: Coverage Report + uses: codecov/codecov-action@v3 + if: steps.generate-xml-report.outcome == 'success' + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ github.workspace }}/coverage.xml + dry_run: false + name: fe-total + flags: fe-total + fail_ci_if_error: false + verbose: true + override_pr: ${{ env.PR_NUMBER }} + override_branch: ${{ env.BASE_REF }} + override_commit: ${{ env.HEAD_SHA }} + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ github.workspace }}/* + + BE-REPORT: + runs-on: [ self-hosted, quick ] + needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + BASE_REF: ${{ needs.INFO.outputs.BASE_REF }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: INFO + id: info + run: | + bucket_prefix=`echo ${REPO%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download BE UT XML + id: download-ut-xml + env: + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + run: | + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER} + be_ut_res_path=${oss_path}/flag/be_ut_res + size=$(ossutil64 --config-file ~/.ossutilconfig ls ${be_ut_res_path} | grep "Object Number is" | awk '{print $NF}') + echo "size=${size}" >> $GITHUB_OUTPUT + if [[ "$size" != "0" ]]; then + ossutil64 --config-file ~/.ossutilconfig cp ${be_ut_res_path} . >/dev/null + res=`cat be_ut_res` + if [[ "$res" != "0" ]]; then + echo "::error::BE UT failed!" + exit 1 + fi + cd be + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/be_ut_coverage.xml . -f 1>/dev/null + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/base_version.txt . -f 1>/dev/null + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/diff.txt . -f 1>/dev/null + echo "base_version=`cat base_version.txt`" >> $GITHUB_OUTPUT + else + rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover /var/local/env/be_empty_coverage.xml --github-token ${{ secrets.GITHUB_TOKEN }} --pr ${PR_NUMBER} \ + --repo ${{ github.repository }} --threshold 80 --github-url api.github.com -type cobertura --module BE + fi + + - name: Merge BE Total Coverage + id: merge + if: github.repository == 'StarRocks/starrocks' && steps.download-ut-xml.outputs.size != '0' && steps.download-ut-xml.outcome == 'success' + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + + export ECI_ID=`./bin/create_eci.sh ${BASE_REF} ${REPO}` + export TOTAL_COV_XML=/var/local/env/be_total_coverage_${PR_NUMBER}.xml + export COV_LOG=/var/local/env/be_total_coverage_${PR_NUMBER}.log + export RES_FILE=/var/local/env/be_total_coverage_res_${PR_NUMBER}.log + + echo ${ECI_ID} + echo "ECI_ID=${ECI_ID}" >> $GITHUB_OUTPUT + echo "COV_XML=${TOTAL_COV_XML}" >> $GITHUB_OUTPUT + echo "COV_LOG=${COV_LOG}" >> $GITHUB_OUTPUT + echo "RES_FILE=${RES_FILE}" >> $GITHUB_OUTPUT + + ./bin/gen_be_cov.sh ${REPO} ${BASE_REF} ${PR_NUMBER} ${HEAD_SHA} + + # Incremental Total Coverage + - name: Publish Incremental Coverage Report - Total + if: always() && steps.download-ut-xml.outputs.size != '0' && steps.download-ut-xml.outcome == 'success' + env: + be_path: ${{ github.workspace }}/be + merge_outcome: ${{ steps.merge.outcome }} + total_xml: ${{ steps.merge.outputs.COV_XML }} + run: | + rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull + export JAVA_HOME=/var/local/env/jdk1.8.0_202; + export PATH=$JAVA_HOME/bin:$PATH; + if [[ "${{ github.repository }}" != "StarRocks/starrocks" ]] || [[ "${merge_outcome}" == "failure" ]]; then + total_xml=${be_path}/be_ut_coverage.xml + fi + java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \ + --cover ${total_xml} --github-token ${{ secrets.GITHUB_TOKEN }} --repo ${{ github.repository }} \ + --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type cobertura --module BE \ + -d ${be_path}/diff.txt -dt file + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ github.workspace }}/* + eci rm ${{ steps.merge.outputs.ECI_ID }} || true + rm -rf ${{ steps.merge.outputs.COV_LOG }} ${{ steps.merge.outputs.COV_XML }} ${{ steps.merge.outputs.RES_FILE }} + + SQL-Tester-REPORT: + runs-on: [ self-hosted, quick ] + needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + BASE_REF: ${{ needs.INFO.outputs.BASE_REF }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: INFO + id: info + run: | + repo="${{ github.repository }}" + bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT + + - name: Download SQL-Tester XML + id: download-SQL-Tester-xml + env: + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + run: | + mkdir sql-tester-result && cd sql-tester-result + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/SQL-Tester-XML/${PR_NUMBER}/ + size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}') + echo "size=${size}" >> $GITHUB_OUTPUT + if [[ "$size" != "0" ]]; then + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive + else + cp /var/local/env/sqltester_empty_coverage.xml . + fi + + - name: Prepare Tools + id: prepare-tools + run: | + mkdir -p .actions/nose-report-action + cd .actions/nose-report-action + git clone https://github.com/StarRocks/action-junit-report.git . + + - name: Publish SQL-Tester Report + uses: ./.actions/nose-report-action + if: steps.prepare-tools.outcome == 'success' + with: + check_name: 'SQL-Tester Report' + fail_on_failure: true + detailed_summary: true + token: ${{ secrets.GITHUB_TOKEN }} + commit: ${{ env.HEAD_SHA }} + report_paths: 'sql-tester-result/*.xml' + + Admit-REPORT: + runs-on: [ self-hosted, quick ] + needs: INFO + if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + BASE_REF: ${{ needs.INFO.outputs.BASE_REF }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: INFO + id: info + run: | + repo="${{ github.repository }}" + bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT + + - name: Download Admit XML + id: download-admit-xml + env: + bucket_prefix: ${{ steps.info.outputs.bucket_prefix }} + run: | + mkdir admit-result && cd admit-result + oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/Admit-XML/${PR_NUMBER}/ + size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}') + echo "size=${size}" >> $GITHUB_OUTPUT + if [[ "$size" != "0" ]]; then + ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive + fi + + - name: Prepare Tools + if: steps.download-admit-xml.outputs.size != '0' + id: prepare-tools + run: | + mkdir -p .actions/nose-report-action + cd .actions/nose-report-action + git clone https://github.com/StarRocks/action-junit-report.git . + + - name: Publish Admit Report + uses: ./.actions/nose-report-action + if: steps.prepare-tools.outcome == 'success' + with: + check_name: 'Admit Report' + fail_on_failure: true + detailed_summary: true + commit: ${{ env.HEAD_SHA }} + token: ${{ secrets.GITHUB_TOKEN }} + report_paths: 'admit-result/*.xml' + + NOTIFICATION: + runs-on: [ self-hosted, quick ] + needs: [ INFO, FE-REPORT, Admit-REPORT, BE-REPORT, SQL-Tester-REPORT ] + if: always() && needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED' + env: + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + FEISHU_APP_ID: ${{ secrets.FEISHU_APP_ID }} + FEISHU_APP_SECRET: ${{ secrets.FEISHU_APP_SECRET }} + PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }} + steps: + - name: CLEAN + run: | + rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }} + + - name: NOTIFY + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + ./bin/notify.sh + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ github.workspace }}/* diff --git a/.github/workflows/inspection-pipeline.yml b/.github/workflows/inspection-pipeline.yml index 957ad1638ea7e..aebed95554ab8 100644 --- a/.github/workflows/inspection-pipeline.yml +++ b/.github/workflows/inspection-pipeline.yml @@ -538,7 +538,7 @@ jobs: fi - name: save unstable cases - if: always() && ((github.event_name == 'schedule' && env.BRANCH == 'main') || (github.event_name != 'schedule' && env.BRANCH != 'main')) + if: always() && (github.event_name == 'schedule' || (github.event_name != 'schedule' && env.BRANCH != 'main')) run: | cd ci-tool && source lib/init.sh ./bin/save_unstable_cases.sh diff --git a/.github/workflows/inspection-reusable-pipeline.yml b/.github/workflows/inspection-reusable-pipeline.yml index e1a3f49ad6644..6d9b0b1c07884 100644 --- a/.github/workflows/inspection-reusable-pipeline.yml +++ b/.github/workflows/inspection-reusable-pipeline.yml @@ -230,7 +230,7 @@ jobs: needs: - restart-fe - restart-be - if: always() && inputs.INCLUDE_ADMIT == 'true' && inputs.BUILD_TYPE == 'Release' && needs.restart-be.result == 'success' && needs.restart-fe.result == 'success' + if: always() && env.INCLUDE_ADMIT == 'true' && env.BUILD_TYPE == 'Release' && needs.restart-be.result == 'success' && needs.restart-fe.result == 'success' timeout-minutes: 40 env: FE_NODE: ${{ needs.restart-be.outputs.FE_NODE }}