Skip to content

[Enhancement] Optimize the Chunk destructor #42141

[Enhancement] Optimize the Chunk destructor

[Enhancement] Optimize the Chunk destructor #42141

Workflow file for this run

name: CI PIPELINE
on:
pull_request:
types:
- opened
- synchronize
branches:
- main
concurrency:
group: ${{ github.event.number }}
cancel-in-progress: true
jobs:
basic-checker:
runs-on: [self-hosted, normal]
name: RUN CHECKER
env:
PR_NUMBER: ${{ github.event.number }}
CODE_PATH: ${{ github.workspace }}
BRANCH: ${{ github.base_ref }}
GH_TOKEN: ${{ github.token }}
outputs:
PASS: ${{ steps.check.outputs.pass }}
steps:
- name: Sync Check
id: check
if: >
(!contains(github.event.pull_request.title, '(sync #') &&
!contains(github.event.pull_request.labels.*.name, 'sync') &&
(!startsWith(github.head_ref, github.base_ref) || !contains(github.head_ref, '-sync-'))) ||
contains(github.event.pull_request.labels.*.name, 'force-check')
run: |
echo "pass=true" >> $GITHUB_OUTPUT
- name: Upload info
run: |
echo $PR_NUMBER > pr_num.txt
- name: Upload the PR number
uses: actions/upload-artifact@v4
with:
name: pr_num
path: ./pr_num.txt
retention-days: 3
overwrite: true
- name: Check Blacklist
id: check_blacklist
env:
NO_SYNC: ${{ steps.check.outputs.pass }}
AUTHOR: ${{ github.event.pull_request.user.login }}
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull >/dev/null
./scripts/check-blacklist.sh
- name: Check Feature Issue
id: check_issue
run: |
cd ci-tool
python3 scripts/get_pr_issue.py check_issue ${GITHUB_REPOSITORY} ${PR_NUMBER}
be-checker:
runs-on: ubuntu-latest
needs: basic-checker
name: BE FILTER
if: needs.basic-checker.outputs.PASS == 'true'
outputs:
src_filter: ${{ steps.path-filter.outputs.be }}
test_filter: ${{ steps.path-filter.outputs.ut }}
thirdparty_filter: ${{ steps.path-filter.outputs.thirdparty }}
steps:
- uses: dorny/paths-filter@v3
id: path-filter
with:
filters: |
be:
- 'be/!(test)**'
- 'gensrc/**'
- 'run-be-ut.sh'
- 'build.sh'
- 'thirdparty/**'
- 'docker/dockerfiles/dev-env/dev-env.Dockerfile'
ut:
- 'be/test/**'
thirdparty:
- 'thirdparty/**'
- 'docker/dockerfiles/dev-env/dev-env.Dockerfile'
- name: Prepare info
run: |
mkdir be-path-filter && cd be-path-filter
echo ${{ steps.path-filter.outputs.be }} > src_filter.txt
echo ${{ steps.path-filter.outputs.ut }} > test_filter.txt
echo ${{ steps.path-filter.outputs.thirdparty }} > thirdparty_filter.txt
- name: Upload the BE Filter Info
uses: actions/upload-artifact@v4
with:
name: be-path-filter
path: ./be-path-filter/
retention-days: 3
overwrite: true
clang-format:
runs-on: [self-hosted, light]
needs: be-checker
name: Clang-Format
env:
PR_NUMBER: ${{ github.event.number }}
if: needs.be-checker.outputs.src_filter == 'true' || needs.be-checker.outputs.test_filter == 'true'
steps:
- name: clean
run: |
rm -rf ${{ github.workspace }}
mkdir -p ${{ github.workspace }}
- name: Checkout Code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: BRANCH INFO
id: branch
run: |
echo ${{github.base_ref}}
echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT
- name: Checkout PR
run: |
BRANCH=${{steps.branch.outputs.branch}}
git config --global user.name "wanpengfei-git";
git config --global user.email "[email protected]";
git checkout $BRANCH;
git pull;
BRANCH_NAME="${BRANCH}-${PR_NUMBER}";
git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME};
git checkout $BRANCH_NAME;
git checkout -b merge_pr;
git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1);
- name: Run Clang-Format
run: |
export PATH=/var/lib/llvm/bin:$PATH
bash build-support/check-format.sh
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*
thirdparty-update:
runs-on: [self-hosted, normal]
needs: [ clang-format ]
name: Thirdparty Update
continue-on-error: true
strategy:
fail-fast: false
matrix:
linux: [ centos7, ubuntu ]
env:
PR_NUMBER: ${{ github.event.number }}
BRANCH: ${{ github.base_ref }}
REPO: ${{ github.repository }}
steps:
- name: clean
run: |
rm -rf ${{ github.workspace }}
mkdir -p ${{ github.workspace }}
- name: Download BE Path Filter Artifact
uses: dawidd6/action-download-artifact@v6
with:
name: be-path-filter
path: be-path-filter
run_id: ${{ github.run_id }}
- name: Parsing path-filter file
id: parsing-path-filter
run: |
cd be-path-filter/; ls;
echo "src_filter=`cat src_filter.txt`" >> $GITHUB_OUTPUT
echo "test_filter.txt=`cat test_filter.txt`" >> $GITHUB_OUTPUT
echo "thirdparty_filter=`cat thirdparty_filter.txt`" >> $GITHUB_OUTPUT
- name: Update Image (${{ matrix.linux }})
id: update-image
if: steps.parsing-path-filter.outputs.thirdparty_filter == 'true'
env:
linux_distro: ${{ matrix.linux }}
run: |
cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/run-pr-update-image.sh
- name: Upload Thirdparty Result
uses: actions/upload-artifact@v4
with:
name: THIRDPARTY-RESULT-${{ matrix.linux }}
path: image_cache.info
retention-days: 1
overwrite: true
if-no-files-found: ignore
- name: Clean ENV
if: always() && steps.parsing-path-filter.outputs.thirdparty_filter == 'true'
run: |
cd ci-tool && source lib/init.sh
./bin/elastic-cluster.sh --delete
rm -rf ${{ github.workspace }}/*
thirdparty-info:
runs-on: [self-hosted, normal]
needs:
- thirdparty-update
name: Thirdparty Info
outputs:
centos7_image_cache_id: ${{ steps.info.outputs.centos7_image_cache_id }}
ubuntu_image_cache_id: ${{ steps.info.outputs.ubuntu_image_cache_id }}
steps:
- name: Check Result
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull >/dev/null
./scripts/check-thirdparty-result.sh
- name: Download Thirdparty Artifact
uses: actions/download-artifact@v4
with:
pattern: THIRDPARTY-RESULT-*
path: outputs
- name: Read Info
id: info
run: |
image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-centos7/image_cache.info" || echo "")
echo "centos7_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT
image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-ubuntu/image_cache.info" || echo "")
echo "ubuntu_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT
be-ut:
runs-on: [self-hosted, normal]
needs: thirdparty-info
timeout-minutes: 90
name: BE UT
env:
PR_NUMBER: ${{ github.event.number }}
BRANCH: ${{ github.base_ref }}
IMAGE_CACHE_ID: ${{ needs.thirdparty-info.outputs.ubuntu_image_cache_id }}
LINUX_DISTRO: ubuntu
steps:
- name: INIT ECI & RUN UT
id: run_ut
shell: bash
timeout-minutes: 90
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
if [[ "${IMAGE_CACHE_ID}" != '' ]]; then
export image_cache_id=${IMAGE_CACHE_ID}
export image_tag=$BRANCH-$PR_NUMBER
fi
./bin/elastic-ut.sh --pr ${PR_NUMBER} --module be --repository ${{ github.repository }} --linuxdistro ${LINUX_DISTRO} --with-gcov
- name: clean ECI
if: always()
run: |
echo ${{ steps.run_ut.outputs.ECI_ID }}
eci rm ${{ steps.run_ut.outputs.ECI_ID }}
- name: Upload Log
uses: actions/upload-artifact@v4
if: always() && steps.run_ut.outcome == 'failure'
with:
name: BE UT LOG
path: ${{ steps.run_ut.outputs.BE_LOG }}
retention-days: 1
overwrite: true
- name: Clean ENV
if: always()
run: |
rm -f ${{ steps.run_ut.outputs.RES_FILE }}
rm -f ${{ steps.run_ut.outputs.RES_LOG }}
rm -rf ${{ steps.run_ut.outputs.BE_LOG }}
rm -rf ${{ github.workspace }}/*
clang-tidy:
runs-on: [self-hosted, normal]
needs: [ be-checker, thirdparty-info ]
if: success() && needs.be-checker.outputs.src_filter == 'true'
timeout-minutes: 90
name: Clang-Tidy
env:
PR_NUMBER: ${{ github.event.number }}
BRANCH: ${{ github.base_ref }}
IMAGE_CACHE_ID: ${{ needs.thirdparty-info.outputs.ubuntu_image_cache_id }}
LINUX_DISTRO: ubuntu
steps:
- name: clean
run: |
rm -rf ${{ github.workspace }}
mkdir -p ${{ github.workspace }}
- name: BRANCH INFO
id: branch
run: |
echo ${{github.base_ref}}
echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT
- name: UPDATE ECI & RUN Clang Tidy
id: run_clang_tidy
shell: bash
timeout-minutes: 90
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
if [[ "${{ needs.be-checker.outputs.thirdparty_filter }}" == 'true' ]]; then
export image_cache_id=${IMAGE_CACHE_ID}
export image_tag=$BRANCH-$PR_NUMBER
fi
./bin/elastic-build.sh --pr ${PR_NUMBER} --repository ${{ github.repository }} --linuxdistro ${LINUX_DISTRO} --be --clang-tidy
- name: clean ECI
if: always()
run: |
echo ${{ steps.run_clang_tidy.outputs.ECI_ID }}
eci rm ${{ steps.run_clang_tidy.outputs.ECI_ID }}
- name: Clean ENV
if: always()
run: |
rm -f ${{ steps.run_clang_tidy.outputs.RES_FILE }}
rm -f ${{ steps.run_clang_tidy.outputs.RES_LOG }}
rm -rf ${{ github.workspace }}/*
fe-checker:
runs-on: ubuntu-latest
needs: basic-checker
name: FE FILTER
if: needs.basic-checker.outputs.PASS == 'true'
outputs:
src_filter: ${{ steps.path-filter.outputs.fe }}
test_filter: ${{ steps.path-filter.outputs.ut }}
java_filter: ${{ steps.path-filter.outputs.java }}
extension_filter: ${{ steps.path-filter.outputs.extension }}
pom_filter: ${{ steps.path-filter.outputs.pom }}
steps:
- uses: dorny/paths-filter@v3
id: path-filter
with:
filters: |
fe:
- 'fe/!(**/test/**)**'
- 'gensrc/**'
- 'java-extensions/**'
- 'build.sh'
ut:
- 'fe/**/test/**'
- 'run-fe-ut.sh'
java:
- '**.java'
extension:
- 'java-extensions/**'
pom:
- '**/pom.xml'
- name: Prepare info
run: |
mkdir fe-path-filter && cd fe-path-filter
echo ${{ steps.path-filter.outputs.fe }} > src_filter.txt
echo ${{ steps.path-filter.outputs.ut }} > test_filter.txt
echo ${{ steps.path-filter.outputs.java }} > java_filter.txt
echo ${{ steps.path-filter.outputs.extension }} > extension_filter.txt
echo ${{ steps.path-filter.outputs.pom }} > pom_filter.txt
- name: Upload the FE Filter Info
uses: actions/upload-artifact@v4
with:
name: fe-path-filter
path: ./fe-path-filter/
retention-days: 3
overwrite: true
fe-codestyle-check:
runs-on: ubuntu-latest
needs: fe-checker
if: needs.fe-checker.outputs.src_filter == 'true' || needs.fe-checker.outputs.test_filter == 'true'
name: FE Code Style Check
env:
PR_NUMBER: ${{ github.event.number }}
outputs:
extension_filter: ${{ needs.fe-checker.outputs.extension_filter }}
steps:
- name: clean
run: |
rm -rf ${{ github.workspace }}
mkdir -p ${{ github.workspace }}
- name: BRANCH INFO
id: branch
run: |
echo ${{github.base_ref}}
echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Checkout PR
run: |
BRANCH=${{steps.branch.outputs.branch}}
git config --global user.name "wanpengfei-git";
git config --global user.email "[email protected]";
git checkout $BRANCH;
git pull;
BRANCH_NAME="${BRANCH}-${PR_NUMBER}";
git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME};
git checkout $BRANCH_NAME;
git checkout -b merge_pr;
git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1);
- name: Copy checkstyle files
if: needs.fe-checker.outputs.java_filter == 'true'
run: |
cp fe/checkstyle* .
- name: Run java checkstyle
if: needs.fe-checker.outputs.java_filter == 'true'
uses: dbelyaev/[email protected]
with:
workdir: "./fe"
checkstyle_config: checkstyle.xml
reporter: 'github-pr-check'
github_token: ${{ secrets.GITHUB_TOKEN }}
fail_on_error: true
level: error
sonarcloud-fe-checker:
runs-on: ubuntu-latest
needs: fe-checker
if: needs.fe-checker.outputs.src_filter == 'true' && github.repository == 'StarRocks/starrocks'
name: FE Sonarcloud Check
env:
PR_NUMBER: ${{ github.event.number }}
CODE_PATH: ${{ github.workspace }}
BRANCH: ${{ github.base_ref }}
GH_TOKEN: ${{ github.token }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: BRANCH INFO
id: branch
run: |
echo ${{github.base_ref}}
echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT
- name: Checkout PR
run: |
BRANCH=${{steps.branch.outputs.branch}}
git config --global user.name "wanpengfei-git";
git config --global user.email "[email protected]";
git checkout $BRANCH;
git pull;
BRANCH_NAME="${BRANCH}-${PR_NUMBER}";
git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME};
git checkout $BRANCH_NAME;
git checkout -b merge_pr;
git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1);
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: 17
distribution: 'adopt'
- name: Cache SonarCloud packages
uses: actions/cache@v4
with:
path: ~/.sonar/cache
key: ${{ runner.os }}-sonar
restore-keys: ${{ runner.os }}-sonar
- name: Cache Maven packages
uses: actions/cache@v4
with:
path: ~/.m2
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-maven
- name: Install Apache Thrift 0.13
run: |
mkdir -p ./.setup-thrift/oras
mkdir -p ./.setup-thrift/thrift
curl -sLO https://github.com/deislabs/oras/releases/download/v0.7.0/oras_0.7.0_linux_amd64.tar.gz
tar -xvzf oras_0.7.0_linux_amd64.tar.gz
ln -sf $(pwd)/oras /usr/local/bin/oras
oras pull ghcr.io/dodopizza/setup-thrift/binaries:v0.13.0 --media-type application/vnd.unknown.layer.v1+tar.gz
tar zxf ./thrift.v0.13.0.tar.gz -C .
ln -sf $(pwd)/thrift /usr/local/bin/thrift
- name: Analyze FE
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any
SONAR_TOKEN: f0fb4d25c03bae90c2e994c45c29c49dc86fc169 # ${{ secrets.SONAR_TOKEN }}
run: |
thrift --version
whereis thrift
export STARROCKS_HOME=${{ github.workspace }}
source env.sh
mkdir -p thirdparty/installed/bin/
cd thirdparty/installed/bin/ && ln -s /usr/local/bin/thrift thrift
cd ${{ github.workspace }}/fe
mvn -B -DskipTests verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=StarRocks_starrocks -Dsonar.pullrequest.key=${{ github.event.number }} -Dsonar.pullrequest.base=${{ github.base_ref }} -Dsonar.pullrequest.branch=${{ github.head_ref }}
fe-ut:
runs-on: [self-hosted, normal]
needs: [fe-codestyle-check]
name: FE UT
env:
PR_NUMBER: ${{ github.event.number }}
CODE_PATH: ${{ github.workspace }}
BRANCH: ${{ github.base_ref }}
GH_TOKEN: ${{ github.token }}
steps:
- name: clean
run: |
rm -rf ${{ github.workspace }}
mkdir -p ${{ github.workspace }}
- name: BRANCH INFO
id: branch
run: |
echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT
repo="${{ github.repository }}"
bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT
- name: UPDATE ECI & RUN UT
id: run_ut
shell: bash
timeout-minutes: 60
env:
EXTENSION: ${{ needs.fe-codestyle-check.outputs.extension_filter }}
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/elastic-ut.sh --pr ${PR_NUMBER} --module fe --branch ${{steps.branch.outputs.branch}} --build Release --repository ${{ github.repository }} --linuxdistro ubuntu
- name: Clean ECI
if: always()
run: |
echo ${{ steps.run_ut.outputs.ECI_ID }}
echo "::group::>>> Dmesg info:"
eci exec ${{ steps.run_ut.outputs.ECI_ID }} bash -c "dmesg -T"
echo "::endgroup::"
eci rm ${{ steps.run_ut.outputs.ECI_ID }}
- name: Upload log
if: always()
uses: actions/upload-artifact@v4
with:
name: FE UT LOG
path: ${{ steps.run_ut.outputs.RES_LOG }}
retention-days: 1
overwrite: true
- name: Clean ENV
if: always()
run: |
rm -rf ${{ steps.run_ut.outputs.FE_REPORT_DIR }}
rm -f ${{ steps.run_ut.outputs.RES_FILE }}
rm -f ${{ steps.run_ut.outputs.RES_LOG }}
rm -rf ${{ steps.run_ut.outputs.COV_DIR }}
rm -rf ${{ github.workspace }}/*
test-checker:
runs-on: ubuntu-latest
needs: basic-checker
name: TEST FILTER
if: needs.basic-checker.outputs.PASS == 'true'
outputs:
output1: ${{ steps.path-filter.outputs.test }}
steps:
- uses: dorny/paths-filter@v3
id: path-filter
with:
filters: |
test:
- 'test/**'
build:
runs-on: [self-hosted, normal]
needs: [test-checker, be-ut, fe-ut, thirdparty-info]
name: BUILD
env:
PR_NUMBER: ${{ github.event.number }}
BRANCH: ${{ github.base_ref }}
IMAGE_CACHE_ID: ${{ needs.thirdparty-info.outputs.ubuntu_image_cache_id }}
LINUX_DISTRO: ubuntu
GH_TOKEN: ${{ github.token }}
outputs:
build_output_tar: ${{ steps.run_build.outputs.OUTPUT_TAR }}
base_version: ${{ steps.run_build.outputs.BASE_VERSION }}
is_self_build: ${{ steps.run_build.outputs.is_self_build }}
build_nece: ${{ steps.check-necessity.outputs.BUILD_NECE }}
if: >
always() && needs.be-ut.result != 'failure' && needs.fe-ut.result != 'failure' && (needs.be-ut.result == 'success' || needs.fe-ut.result == 'success' || needs.test-checker.result == 'success')
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: check sync
id: check-sync
run: |
labels=`gh pr view ${PR_NUMBER} -R ${GITHUB_REPOSITORY} --json labels -q '.labels[].name'`
echo "${labels}"
if [[ "${labels}" =~ sync ]]; then
echo "IS_SYNC=true" >> $GITHUB_OUTPUT
fi
- name: Download BE Path Filter Artifact
uses: dawidd6/action-download-artifact@v6
if: steps.check-sync.outputs.IS_SYNC != 'true'
with:
name: be-path-filter
path: be-path-filter
if_no_artifact_found: fail
run_id: ${{ github.run_id }}
- name: Download FE Path Filter Artifact
uses: dawidd6/action-download-artifact@v6
if: steps.check-sync.outputs.IS_SYNC != 'true'
with:
name: fe-path-filter
path: fe-path-filter
if_no_artifact_found: fail
run_id: ${{ github.run_id }}
- name: Parsing BE path-filter file
id: parsing-be-path-filter
if: steps.check-sync.outputs.IS_SYNC != 'true'
run: |
if [[ -e be-path-filter ]]; then
cd be-path-filter/; ls;
echo "src_filter=`cat src_filter.txt`" >> $GITHUB_OUTPUT
echo "test_filter.txt=`cat test_filter.txt`" >> $GITHUB_OUTPUT
echo "thirdparty_filter=`cat thirdparty_filter.txt`" >> $GITHUB_OUTPUT
fi
- name: Parsing FE path-filter file
id: parsing-fe-path-filter
if: steps.check-sync.outputs.IS_SYNC != 'true'
run: |
if [[ -e fe-path-filter ]]; then
cd fe-path-filter/; ls;
echo "src_filter=`cat src_filter.txt`" >> $GITHUB_OUTPUT
echo "test_filter.txt=`cat test_filter.txt`" >> $GITHUB_OUTPUT
echo "java_filter=`cat java_filter.txt`" >> $GITHUB_OUTPUT
echo "extension_filter=`cat extension_filter.txt`" >> $GITHUB_OUTPUT
echo "pom_filter=`cat pom_filter.txt`" >> $GITHUB_OUTPUT
fi
- name: Check necessity
id: check-necessity
if: >
(needs.be-ut.result == 'success' && needs.fe-ut.result == 'success') ||
(steps.parsing-be-path-filter.outputs.src_filter != 'true' && steps.parsing-fe-path-filter.outputs.src_filter == 'true' && needs.fe-ut.result == 'success') ||
(steps.parsing-fe-path-filter.outputs.src_filter != 'true' && steps.parsing-be-path-filter.outputs.src_filter == 'true' && needs.be-ut.result == 'success') ||
(steps.parsing-be-path-filter.outputs.src_filter != 'true' && steps.parsing-fe-path-filter.outputs.src_filter != 'true' && needs.test-checker.outputs.output1 == 'true')
run: |
echo "BUILD_NECE=true" >> $GITHUB_OUTPUT
- name: BRANCH INFO
id: branch
if: steps.check-necessity.outcome == 'success'
run: |
repo="${{ github.repository }}"
bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
echo "bucket_prefix=${bucket_prefix}" >> "$GITHUB_ENV"
- name: Rebuild Checker
id: rebuild-checker
if: steps.check-necessity.outcome == 'success'
env:
be_change: ${{ steps.parsing-be-path-filter.outputs.src_filter }}
fe_change: ${{ steps.parsing-fe-path-filter.outputs.src_filter }}
build_type: Release
run: |
echo "package=${package}" >> "$GITHUB_ENV"
if [[ "${be_change}" != "true" && "${fe_change}" != "true" ]]; then
oss_path=oss://${bucket_prefix}-ci-release/${BRANCH}/${build_type}/inspection/pr/ubuntu/StarRocks-
package=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "tar.gz" | sort -n -r | head -n 1 | awk '{print $NF}')
echo "package=${package}" >> $GITHUB_ENV
fi
- name: UPDATE ECI & RUN BUILD
id: run_build
shell: bash
timeout-minutes: 90
if: steps.check-necessity.outcome == 'success'
env:
package: ${{ env.package }}
pom_filter: ${{ steps.parsing-fe-path-filter.outputs.pom_filter }}
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
if [[ "${{ steps.parsing-be-path-filter.outputs.thirdparty_filter }}" == 'true' ]]; then
export image_cache_id=${IMAGE_CACHE_ID}
export image_tag=$BRANCH-$PR_NUMBER
fi
if [[ ${package} == "" ]]; then
./bin/elastic-build.sh --pr ${PR_NUMBER} --repository ${{ github.repository }} --linuxdistro ${LINUX_DISTRO} --with-gcov --with-trivy
echo "is_self_build=true" >> $GITHUB_OUTPUT
else
echo "Use latest tar: ${package}"
echo "OUTPUT_TAR=${package}" >> "$GITHUB_OUTPUT"
echo "BASE_VERSION=" >> "$GITHUB_OUTPUT"
echo "ECI_ID=" >> "$GITHUB_OUTPUT"
echo "is_self_build=false" >> $GITHUB_OUTPUT
fi
- name: build result
if: steps.check-necessity.outcome == 'success'
run: |
echo ${{ steps.run_build.outputs.OUTPUT_TAR }}
echo ${{ steps.run_build.outputs.BASE_VERSION }} > ./base_version.txt
- name: Upload the Base Version
uses: actions/upload-artifact@v4
if: steps.check-necessity.outcome == 'success'
with:
name: base_version
path: ./base_version.txt
retention-days: 3
overwrite: true
- name: clean ECI
if: always() && steps.run_build.outputs.ECI_ID != ''
run: |
echo ${{ steps.run_build.outputs.ECI_ID }}
eci rm ${{ steps.run_build.outputs.ECI_ID }}
- name: Clean ENV
if: always()
run: |
rm -f ${{ steps.run_build.outputs.RES_FILE }}
rm -f ${{ steps.run_build.outputs.RES_LOG }}
rm -rf ${{ github.workspace }}
deploy:
runs-on: [self-hosted, normal]
if: always() && needs.build.result == 'success' && needs.build.outputs.build_nece == 'true'
needs: build
name: DEPLOY SR
env:
PR_NUMBER: ${{ github.event.number }}
LINUX_DISTRO: ubuntu
outputs:
fe: ${{steps.deploy_sr.outputs.fe}}
be: ${{steps.deploy_sr.outputs.be}}
deploy_conf_file: ${{steps.apply_resource.outputs.deploy_conf_file}}
cluster_name: ${{steps.deploy_sr.outputs.cluster_name}}
be_list: ${{steps.deploy_sr.outputs.be_list}}
shared_data: ${{steps.choose-mode.outputs.cloud}}
is_self_build: ${{ needs.build.outputs.is_self_build }}
steps:
- name: Clean Workspace
uses: AutoModality/[email protected]
- name: Choose mode
id: choose-mode
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
cluster_type=$(./bin/choose-mode.sh)
if [[ "${cluster_type}" == "cloud" ]]; then
echo "cloud=true" >> "$GITHUB_OUTPUT"
fi
- name: Apply for resources
id: apply_resource
env:
CLUSTER_NAME: ci-admit
run: |
cd ci-tool && source lib/init.sh
./bin/elastic-cluster.sh --template ${CLUSTER_NAME} --linuxdistro ${LINUX_DISTRO}
cp conf/starrocks_deploy.conf /var/local/env/${PR_NUMBER}-starrocks_deploy.conf
echo "deploy_conf_file=/var/local/env/${PR_NUMBER}-starrocks_deploy.conf" >> $GITHUB_OUTPUT
- name: Deploy SR
id: deploy_sr
env:
CLUSTER_NAME: "${{ steps.choose-mode.outputs.cloud == 'true' && 'ci-admit-cloud' || 'ci-admit' }}"
repo: ${{ github.repository }}
run: |
cd ci-tool && source lib/init.sh
if [[ "${{ needs.build.outputs.is_self_build }}" == 'true' ]]; then
bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
tar_path="oss://${bucket_prefix}-ci-release/${{github.base_ref}}/Release/pr/${LINUX_DISTRO}/${{needs.build.outputs.build_output_tar}}"
else
tar_path="${{ needs.build.outputs.build_output_tar }}"
fi
echo "##### Package: ${tar_path}" >> $GITHUB_STEP_SUMMARY
echo "cluster_name=${CLUSTER_NAME}" >> $GITHUB_OUTPUT
./bin/deploy-cluster.sh -c ${CLUSTER_NAME} --output ${tar_path} --with-coverage
- name: Clean ENV
if: always()
run: |
echo "FE: ${{steps.deploy_sr.outputs.fe}}, BE: ${{steps.deploy_sr.outputs.be}}"
rm -rf ${{ github.workspace }}/*
SQL-Tester:
runs-on: [self-hosted, normal]
name: SQL-Tester
needs: [ build, deploy ]
if: always() && needs.deploy.result == 'success'
timeout-minutes: 60
env:
PR_NUMBER: ${{ github.event.number }}
CLUSTER_NAME: ${{ needs.deploy.outputs.cluster_name }}
FE_NODE: ${{ needs.deploy.outputs.fe }}
BE_NODE: ${{ needs.deploy.outputs.be }}
BASE_VERSION: ${{ needs.build.outputs.base_version }}
outputs:
MYSQL_ECI_ID: ${{ steps.run_case.outputs.MYSQL_ECI_ID }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: Checkout Code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: BRANCH INFO
id: branch
run: |
echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT
repo="${{ github.repository }}"
bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT
- name: Checkout PR
run: |
BRANCH=${{steps.branch.outputs.branch}}
git config --global user.name "wanpengfei-git";
git config --global user.email "[email protected]";
git checkout $BRANCH;
git reset ${BASE_VERSION} --hard || true;
BRANCH_NAME="${BRANCH}-${PR_NUMBER}";
git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME};
git checkout $BRANCH_NAME;
git checkout -b merge_pr;
git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1);
- name: Run Case (${{ needs.deploy.outputs.fe }})
id: run_case
env:
is_self_build: ${{ needs.deploy.outputs.is_self_build }}
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/run-sql-tester.sh
- name: Upload SQL-Tester XML
if: success() || failure()
run: |
branch=${{ steps.branch.outputs.branch }}
bucket_prefix=${{ steps.branch.outputs.bucket_prefix }}
xml_oss_path=oss://${bucket_prefix}-ci-release/$branch/Release/pr/SQL-Tester-XML/${PR_NUMBER}/
ossutil64 --config-file ~/.ossutilconfig rm ${xml_oss_path} -rf
ossutil64 --config-file ~/.ossutilconfig cp test/ ${xml_oss_path} --include "*.xml" --recursive --force --tagging="type=ci"
- name: Upload log
uses: actions/upload-artifact@v4
if: always()
with:
name: SQL-Tester Log
path: |
test/log/
test/crash_logs/
retention-days: 1
overwrite: true
if-no-files-found: ignore
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*
restart-fe:
runs-on: [ self-hosted, normal ]
name: Restart FE
needs: [ deploy, SQL-Tester ]
if: always() && needs.SQL-Tester.result == 'success'
env:
PR_NUMBER: ${{ github.event.number }}
CLUSTER_NAME: ${{ needs.deploy.outputs.cluster_name }}
FE_NODE: ${{ needs.deploy.outputs.fe }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: Restart FE (${{needs.deploy.outputs.fe}})
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/restart-fe.sh --pr ${PR_NUMBER} --branch ${{ github.base_ref }} --build Release --repository ${{ github.repository }}
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*
restart-be:
runs-on: [ self-hosted, normal ]
name: Restart BE
needs: [ deploy, SQL-Tester ]
if: always() && needs.SQL-Tester.result == 'success'
timeout-minutes: 20
env:
PR_NUMBER: ${{ github.event.number }}
CLUSTER_NAME: ${{ needs.deploy.outputs.cluster_name }}
FE_NODE: ${{ needs.deploy.outputs.fe }}
BE_NODE: ${{ needs.deploy.outputs.be }}
BE_LIST: ${{ needs.deploy.outputs.be_list }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: Restart BE (${{needs.deploy.outputs.be_list}})
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
[[ "${{ needs.deploy.outputs.is_self_build }}" != "true" ]] && skip_gcov="--skip-gcov"
./bin/system-cov-be.sh --pr ${PR_NUMBER} --branch ${{ github.base_ref }} --build Release --repository ${{ github.repository }} ${skip_gcov}
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*
admit:
runs-on: [ self-hosted, normal ]
name: ADMIT TEST
needs: [ deploy, restart-fe, restart-be ]
if: always() && needs.restart-fe.result == 'success' && needs.restart-be.result == 'success' && needs.deploy.outputs.is_self_build == 'true'
timeout-minutes: 40
env:
PR_NUMBER: ${{ github.event.number }}
CLUSTER_NAME: ${{ needs.deploy.outputs.cluster_name }}
FE_NODE: ${{ needs.deploy.outputs.fe }}
BE_NODE: ${{ needs.deploy.outputs.be }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: BRANCH INFO
id: branch
run: |
echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT
repo="${{ github.repository }}"
bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT
- name: Run Case
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
cd ${{ github.workspace }}
rm -rf ./StarRocksTest && cp -rf /var/lib/StarRocksTest ./StarRocksTest && cd ./StarRocksTest && git pull >/dev/null
cd ${{ github.workspace }}/ci-tool
if [[ "${{ needs.deploy.outputs.shared_data }}" == "true" ]]; then
cluster_type=cloud
else
cluster_type=native
fi
./bin/run-admit-single.sh --pr ${PR_NUMBER} --branch ${{steps.branch.outputs.branch}} --build Release --repository ${{ github.repository }} --cluster ${cluster_type}
- name: Upload Admit XML
if: success() || failure()
run: |
branch=${{ steps.branch.outputs.branch }}
bucket_prefix=${{ steps.branch.outputs.bucket_prefix }}
xml_oss_path=oss://${bucket_prefix}-ci-release/$branch/Release/pr/Admit-XML/${PR_NUMBER}/
ossutil64 --config-file ~/.ossutilconfig rm ${xml_oss_path} -rf
ossutil64 --config-file ~/.ossutilconfig cp StarRocksTest/result ${xml_oss_path} --include "*.xml" --recursive --force --tagging="type=ci"
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*
Teardown:
runs-on: [ self-hosted, quick ]
name: Teardown
needs: [ deploy, SQL-Tester, admit ]
if: always()
env:
PR_NUMBER: ${{ github.event.number }}
BRANCH: ${{ github.base_ref }}
CONF_FILE: ${{ needs.deploy.outputs.deploy_conf_file }}
linuxdistro: ubuntu
steps:
- name: Upload info
run: |
echo $PR_NUMBER > pr_num.txt
- name: Upload the PR number
uses: actions/upload-artifact@v4
with:
name: pr_num
path: ./pr_num.txt
retention-days: 3
overwrite: true
- name: Backup SR Info
if: needs.deploy.outputs.deploy_conf_file != ''
id: backup
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/backup_log_cores.sh --branch $BRANCH --pr $PR_NUMBER --build Release --conf $CONF_FILE --repository ${{ github.repository }}
- name: Clean ECS
if: steps.backup.outcome == 'success'
run: |
cd ci-tool && source lib/init.sh
./bin/elastic-cluster.sh --delete
- name: clean ECI
if: always() && needs.SQL-Tester.outputs.MYSQL_ECI_ID != ''
run: |
eci rm ${{ needs.SQL-Tester.outputs.MYSQL_ECI_ID }}
- name: Clean
if: always()
run: |
rm -f $CONF_FILE
rm -rf ${{ github.workspace }}/*