diff --git a/.github/workflows/ci-doc-checker.yml b/.github/workflows/ci-doc-checker.yml new file mode 100644 index 0000000000000..919c5a4832dbb --- /dev/null +++ b/.github/workflows/ci-doc-checker.yml @@ -0,0 +1,88 @@ +name: CI DOC Checker + +on: + pull_request_target: + types: + - opened + - synchronize + branches: + - main + - 'branch*' + +permissions: + issues: write + pull-requests: write + +jobs: + doc-checker: + runs-on: ubuntu-latest + name: DOC FILTER + if: > + !contains(github.event.pull_request.title, '(sync #') && + !contains(github.event.pull_request.labels.*.name, 'sync') && + (!startsWith(github.head_ref, github.base_ref) || !contains(github.head_ref, '-sync-')) + outputs: + output1: ${{ steps.doc-changes-info.outputs.doc }} + steps: + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + doc: + - 'docs/**' + - name: DOC CHECK INFO + id: doc-changes-info + run: | + echo "doc=${{ steps.changes.outputs.doc }}" >> $GITHUB_OUTPUT + + add-doc-label: + needs: doc-checker + runs-on: ubuntu-latest + name: ADD DOC LABEL + if: ${{ needs.doc-checker.outputs.output1 == 'true' }} + steps: + - name: add document label + uses: actions-ecosystem/action-add-labels@v1 + with: + github_token: ${{ secrets.PAT }} + labels: documentation + + markdownlint: + runs-on: ubuntu-latest + needs: add-doc-label + env: + PR_NUMBER: ${{ github.event.number }} + steps: + - name: clean + run: | + rm -rf ${{ github.workspace }} + mkdir -p ${{ github.workspace }} + - name: BRANCH INFO + id: branch + run: | + echo ${{github.base_ref}} + echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Checkout PR + run: | + BRANCH=${{steps.branch.outputs.branch}} + git config --global user.name "wanpengfei-git"; + git config --global user.email "wanpengfei91@163.com"; + git checkout $BRANCH; + git pull; + BRANCH_NAME="${BRANCH}-${PR_NUMBER}"; + git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; + git checkout $BRANCH_NAME; + git checkout -b merge_pr; + git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); + + - uses: articulate/actions-markdownlint@v1.1.0 + with: + config: "docs/.markdownlint.yaml" + files: 'docs/**/*.md' + ignore: node_modules + version: 0.28.1 diff --git a/.github/workflows/ci-pipeline-branch.yml b/.github/workflows/ci-pipeline-branch.yml new file mode 100644 index 0000000000000..a670a6c900254 --- /dev/null +++ b/.github/workflows/ci-pipeline-branch.yml @@ -0,0 +1,389 @@ +name: CI PIPELINE - BRANCH + +on: + pull_request: + types: + - opened + - synchronize + + branches: + - 'branch*' + +concurrency: + group: ${{ github.event.number }} + cancel-in-progress: true + + +jobs: + + sync-checker: + runs-on: [ self-hosted, normal ] + name: RUN CHECKER + env: + PR_NUMBER: ${{ github.event.number }} + outputs: + PASS: ${{ steps.check.outputs.pass }} + THIRDPARTY: ${{ steps.thirdparty-checker.outputs.thirdparty }} + steps: + - name: upload info + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_num.txt + GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha) + echo ${GITHUB_SHA} > head_sha.txt + + - name: Upload the PR number + uses: actions/upload-artifact@v2 + with: + name: pr_num + path: ./pr_num.txt + retention-days: 3 + + - name: Upload the PR HEAD REF + uses: actions/upload-artifact@v2 + with: + name: head_sha + path: ./head_sha.txt + retention-days: 3 + + - name: Check + id: check + if: > + !contains(github.event.pull_request.title, '(sync #') && + !contains(github.event.pull_request.labels.*.name, 'sync') && + (!startsWith(github.head_ref, github.base_ref) || !contains(github.head_ref, '-sync-')) + run: | + echo "pass=true" >> $GITHUB_OUTPUT + + - uses: dorny/paths-filter@v2 + id: thirdparty-checker + if: always() + with: + filters: | + thirdparty: + - 'thirdparty/**' + - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' + + be-checker: + runs-on: ubuntu-latest + needs: sync-checker + name: BE FILTER + if: needs.sync-checker.outputs.PASS == 'true' || needs.sync-checker.outputs.THIRDPARTY == 'true' + outputs: + output1: ${{ steps.be-changes-info.outputs.be }} + output2: ${{ steps.be-changes-info.outputs.thirdparty }} + steps: + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + be: + - 'be/**' + - 'gensrc/**' + - 'run-be-ut.sh' + - 'build.sh' + - 'thirdparty/**' + - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' + thirdparty: + - 'thirdparty/**' + - 'docker/dockerfiles/dev-env/dev-env.Dockerfile' + + - name: BE CHECK INFO + id: be-changes-info + run: | + echo "be=${{ steps.changes.outputs.be }}" >> $GITHUB_OUTPUT + echo "thirdparty=${{ steps.changes.outputs.thirdparty }}" >> $GITHUB_OUTPUT + + clang-format: + runs-on: [self-hosted, light] + needs: be-checker + name: Clang-Format + env: + PR_NUMBER: ${{ github.event.number }} + if: ${{ needs.be-checker.outputs.output1 == 'true' }} + steps: + - name: clean + run: | + rm -rf ${{ github.workspace }} + mkdir -p ${{ github.workspace }} + + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: BRANCH INFO + id: branch + run: | + echo ${{github.base_ref}} + echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT + + - name: Checkout PR + run: | + BRANCH=${{steps.branch.outputs.branch}} + git config --global user.name "wanpengfei-git"; + git config --global user.email "wanpengfei91@163.com"; + git checkout $BRANCH; + git pull; + BRANCH_NAME="${BRANCH}-${PR_NUMBER}"; + git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; + git checkout $BRANCH_NAME; + git checkout -b merge_pr; + git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); + + - name: Run Clang-Format + run: | + export PATH=/var/lib/llvm/bin:$PATH + bash build-support/check-format.sh + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ github.workspace }}/* + + thirdparty-update: + runs-on: [self-hosted, normal] + needs: [ be-checker, clang-format ] + name: Thirdparty Update + env: + PR_NUMBER: ${{ github.event.number }} + BRANCH: ${{ github.base_ref }} + REPO: ${{ github.repository }} + outputs: + ip: ${{ steps.update-image.outputs.ip }} + instance_id: ${{ steps.update-image.outputs.instance_id }} + image_cache_id: ${{ steps.update-image.outputs.image_cache_id }} + steps: + - name: clean + run: | + rm -rf ${{ github.workspace }} + mkdir -p ${{ github.workspace }} + + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Update Image + id: update-image + run: | + if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + ./bin/run-pr-update-image.sh + fi + + - name: Clean ENV + if: always() + run: | + if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then + cd ci-tool && source lib/init.sh + ./bin/elastic-cluster.sh --delete + fi + rm -rf ${{ github.workspace }}/* + + be-ut: + runs-on: [self-hosted, normal] + needs: [ be-checker, thirdparty-update ] + timeout-minutes: 90 + name: BE UT + env: + PR_NUMBER: ${{ github.event.number }} + BRANCH: ${{ github.base_ref }} + steps: + - name: BRANCH INFO + id: branch + run: | + echo ${{github.base_ref}} + echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT + + - name: UPDATE ECI & RUN UT + id: run_ut + shell: bash + timeout-minutes: 90 + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then + export image_cache_id=${{ needs.thirdparty-update.outputs.image_cache_id }} + export image_tag=$BRANCH-$PR_NUMBER + fi + ./bin/elastic-ut.sh --pr ${PR_NUMBER} --module be --branch ${{ steps.branch.outputs.branch }} --repository ${{ github.repository }} + + - name: clean ECI + if: always() + run: | + echo ${{ steps.run_ut.outputs.ECI_ID }} + eci rm ${{ steps.run_ut.outputs.ECI_ID }} + + - name: Upload log + uses: actions/upload-artifact@v3 + if: always() + with: + name: BE UT LOG + path: ${{ steps.run_ut.outputs.RES_LOG }} + retention-days: 1 + + - name: Clean ENV + if: always() + run: | + rm -f ${{ steps.run_ut.outputs.RES_FILE }} + rm -f ${{ steps.run_ut.outputs.RES_LOG }} + rm -rf ${{ github.workspace }}/* + + fe-checker: + runs-on: ubuntu-latest + needs: sync-checker + name: FE FILTER + if: needs.sync-checker.outputs.PASS == 'true' || needs.sync-checker.outputs.THIRDPARTY == 'true' + outputs: + output1: ${{ steps.fe-changes-info.outputs.fe }} + steps: + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + fe: + - 'fe/**' + - 'gensrc/**' + - 'run-fe-ut.sh' + - 'java-extensions/**' + - 'build.sh' + + - run: echo ${{ steps.changes.outputs.fe }} + + - name: FE CHECK INFO + id: fe-changes-info + run: | + echo "fe=${{ steps.changes.outputs.fe }}" >> $GITHUB_OUTPUT + + fe-codestyle-check: + runs-on: ubuntu-latest + needs: fe-checker + if: needs.fe-checker.outputs.output1 == 'true' + name: FE Code Style Check + env: + PR_NUMBER: ${{ github.event.number }} + steps: + - name: clean + run: | + rm -rf ${{ github.workspace }} + mkdir -p ${{ github.workspace }} + + - uses: dorny/paths-filter@v2 + id: java-file + name: JAVA FILE + with: + filters: | + java: + - '**.java' + + - run: echo ${{ steps.java-file.outputs.java }} + + - name: BRANCH INFO + id: branch + run: | + echo ${{github.base_ref}} + echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Checkout PR + run: | + BRANCH=${{steps.branch.outputs.branch}} + git config --global user.name "wanpengfei-git"; + git config --global user.email "wanpengfei91@163.com"; + git checkout $BRANCH; + git pull; + BRANCH_NAME="${BRANCH}-${PR_NUMBER}"; + git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; + git checkout $BRANCH_NAME; + git checkout -b merge_pr; + git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); + + - name: Copy checkstyle files + if: ${{ steps.java-file.outputs.java == 'true' }} + run: | + cp fe/checkstyle* . + + - name: Run java checkstyle + if: ${{ steps.java-file.outputs.java == 'true' }} + uses: dbelyaev/action-checkstyle@v0.7.5 + with: + workdir: "./fe" + checkstyle_config: checkstyle.xml + reporter: 'github-pr-check' + github_token: ${{ secrets.GITHUB_TOKEN }} + fail_on_error: true + level: error + + fe-ut: + runs-on: [self-hosted, normal] + needs: fe-codestyle-check + name: FE UT + env: + PR_NUMBER: ${{ github.event.number }} + steps: + - name: clean + run: | + rm -rf ${{ github.workspace }} + mkdir -p ${{ github.workspace }} + + - name: BRANCH INFO + id: branch + run: | + echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT + repo="${{ github.repository }}" + bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` + echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Checkout PR + run: | + BRANCH=${{steps.branch.outputs.branch}} + git config --global user.name "wanpengfei-git"; + git config --global user.email "wanpengfei91@163.com"; + git checkout $BRANCH; + git pull; + BRANCH_NAME="${BRANCH}-${PR_NUMBER}"; + git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME}; + git checkout $BRANCH_NAME; + git checkout -b merge_pr; + git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1); + + - name: UPDATE ECI & RUN UT + id: run_ut + shell: bash + timeout-minutes: 60 + run: | + rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh + ./bin/elastic-ut.sh --pr ${PR_NUMBER} --module fe --branch ${{steps.branch.outputs.branch}} --build Release --repository ${{ github.repository }} + + - name: Clean ECI + if: always() + run: | + echo ${{ steps.run_ut.outputs.ECI_ID }} + echo ">>> Dmesg info:" + eci exec ${{ steps.run_ut.outputs.ECI_ID }} bash -c "dmesg -T" + eci rm ${{ steps.run_ut.outputs.ECI_ID }} + + - name: Upload log + if: always() + uses: actions/upload-artifact@v3 + with: + name: FE UT LOG + path: ${{ steps.run_ut.outputs.RES_LOG }} + retention-days: 1 + + - name: Clean ENV + if: always() + run: | + rm -rf ${{ steps.run_ut.outputs.FE_REPORT_DIR }} + rm -f ${{ steps.run_ut.outputs.RES_FILE }} + rm -f ${{ steps.run_ut.outputs.RES_LOG }} + rm -rf ${{ steps.run_ut.outputs.COV_DIR }} + rm -rf ${{ github.workspace }}/*