fix(workflow): ci/cd failure for main branch (#3996) #5056
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow runs SDK related jobs for OpenMLDB | ||
| name: SDK | ||
| on: | ||
| push: | ||
| branches: | ||
| - main | ||
| paths-ignore: | ||
| - "docs/**" | ||
| - "demo/**" | ||
| - "docker/**" | ||
| - "image/**" | ||
| - "release/**" | ||
| - "tools/**" | ||
| - "*.md" | ||
| - 'benchmark/**' | ||
| tags: | ||
| - v* | ||
| pull_request: | ||
| paths-ignore: | ||
| - "docs/**" | ||
| - "demo/**" | ||
| - "docker/**" | ||
| - "image/**" | ||
| - "release/**" | ||
| - "tools/**" | ||
| - "*.md" | ||
| - 'benchmark/**' | ||
| workflow_dispatch: | ||
| env: | ||
| GIT_SUBMODULE_STRATEGY: recursive | ||
| NPROC: 2 # default Parallel build number for GitHub's Linux runner | ||
| EXAMPLES_ENABLE: OFF # turn off hybridse's example code | ||
| HYBRIDSE_TESTING_ENABLE: OFF # turn off hybridse's test code | ||
| jobs: | ||
| java-sdk: | ||
| runs-on: ubuntu-latest | ||
| container: | ||
| image: ghcr.io/4paradigm/hybridsql:latest | ||
| env: | ||
| ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 | ||
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | ||
| SQL_JAVASDK_ENABLE: ON | ||
| OPENMLDB_BUILD_TARGET: "cp_native_so openmldb" | ||
| MAVEN_OPTS: -Duser.home=/github/home | ||
| SPARK_HOME: /tmp/spark/ | ||
| # ref https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ | ||
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - uses: actions/setup-java@v2 | ||
| with: | ||
| distribution: "adopt" | ||
| java-version: "8" | ||
| server-id: ossrh | ||
| server-username: MAVEN_USERNAME | ||
| server-password: MAVEN_TOKEN | ||
| gpg-passphrase: GPG_PASSPHRASE # env variable for GPG private key passphrase | ||
| - name: Import GPG key | ||
| id: import_gpg | ||
| if: github.event_name == 'push' | ||
| uses: crazy-max/ghaction-import-gpg@v4 | ||
| with: | ||
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | ||
| - name: Cache local Maven repository | ||
| uses: actions/cache@v2 | ||
| with: | ||
| path: ~/.m2/repository | ||
| key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }} | ||
| - name: prepare release | ||
| if: github.event_name == 'push' | ||
| run: | | ||
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | ||
| VERSION=${VERSION#v} | ||
| ./java/prepare_release.sh "$VERSION" | ||
| - name: build jsdk | ||
| run: | | ||
| make build | ||
| - name: upload linux library | ||
| if: github.event_name == 'push' | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: shared-library-${{ github.sha }} | ||
| path: | | ||
| java/openmldb-native/src/main/resources/libsql_jsdk.so | ||
| java/hybridse-native/src/main/resources/libhybridse_jsdk_core.so | ||
| - name: start services | ||
| run: | | ||
| sh steps/ut_zookeeper.sh start | ||
| sh steps/download_openmldb_spark.sh $SPARK_HOME | ||
| cd onebox && sh start_onebox.sh && sh start_onebox.sh standalone && cd - || exit | ||
| - name: run java modules smoke test | ||
| working-directory: java | ||
| run: | | ||
| ./mvnw --batch-mode test | ||
| - name: Upload Event File | ||
| if: always() | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: event-file | ||
| path: ${{ github.event_path }} | ||
| - name: upload java ut results | ||
| if: always() | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: linux-ut-result-java-${{ github.sha }} | ||
| path: | | ||
| java/*/target/**/TEST-*.xml | ||
| - name: deploy | ||
| if: github.event_name == 'push' | ||
| working-directory: java | ||
| run: | | ||
| ./mvnw --batch-mode deploy -DskipTests=true -Dscalatest.skip=true | ||
| env: | ||
| MAVEN_OPTS: -Duser.home=/github/home | ||
| MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} | ||
| MAVEN_TOKEN: ${{ secrets.OSSRH_TOKEN }} | ||
| GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} | ||
| - name: maven coverage | ||
| working-directory: java | ||
| run: | | ||
| rm -rfv ~/.m2/repository/com/4paradigm/ | ||
| ./mvnw --batch-mode prepare-package | ||
| ./mvnw --batch-mode scoverage:report | ||
| - name: upload coverage | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: coverage-java-report-${{ github.sha }} | ||
| path: | | ||
| java/**/target/site/jacoco/jacoco.xml | ||
| java/**/target/scoverage.xml | ||
| retention-days: 3 | ||
| - name: stop services | ||
| run: | | ||
| cd onebox && ./stop_all.sh && cd - || exit | ||
| sh steps/ut_zookeeper.sh stop | ||
| java-sdk-mac: | ||
| # mac job for java sdk. steps are almost same with job 'java-sdk' | ||
| # except mvn deploy won't target all modules, just hybridse-native & openmldb-native | ||
| # the job only run on tag push or manual workflow dispatch due to no test runs | ||
| runs-on: macos-12 | ||
| needs: | ||
| - java-sdk | ||
| if: github.event_name == 'push' | ||
| env: | ||
| SQL_JAVASDK_ENABLE: ON | ||
| OPENMLDB_BUILD_TARGET: "cp_native_so openmldb" | ||
| NPROC: 3 | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| # target on macOS >= 12.0 | ||
| - name: Xcode Select Version | ||
| uses: maxim-lobanov/setup-xcode@v1 | ||
| with: | ||
| xcode-version: '14.0.1' | ||
| - name: Cache local Maven repository | ||
| uses: actions/cache@v3 | ||
| with: | ||
| path: ~/.m2/repository | ||
| key: ${{ runner.os }}-maven-${{ hashFiles('java/**/pom.xml') }} | ||
| - name: Cache thirdparty | ||
| uses: actions/cache@v3 | ||
| with: | ||
| path: | | ||
| .deps/ | ||
| thirdsrc | ||
| key: ${{ runner.os }}-thirdparty-${{ hashFiles('third-party/**/CMakeLists.txt', 'third-party/**/*.cmake', 'third-party/**/*.sh') }} | ||
| - name: prepare release | ||
| if: github.event_name == 'push' | ||
| run: | | ||
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | ||
| VERSION=${VERSION#v} | ||
| VARIANT_TYPE=macos ./java/prepare_release.sh "$VERSION" | ||
| - uses: actions/setup-java@v3 | ||
| with: | ||
| distribution: "adopt" | ||
| java-version: "8" | ||
| server-id: ossrh | ||
| server-username: MAVEN_USERNAME | ||
| server-password: MAVEN_TOKEN | ||
| gpg-passphrase: GPG_PASSPHRASE # env variable for GPG private key passphrase | ||
| - name: Import GPG key | ||
| id: import_gpg | ||
| uses: crazy-max/ghaction-import-gpg@v5 | ||
| with: | ||
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | ||
| - name: build jsdk | ||
| run: | | ||
| make build | ||
| - name: mvn deploy | ||
| working-directory: java | ||
| run: | | ||
| # by convention only native submodule has variant release | ||
| # so it is a bit tricky to compile and deploy that | ||
| # firstly run `maven install` install things locally | ||
| # then selectly `mvn deploy` for hybridse-native & openmldb-native | ||
| ./mvnw --batch-mode clean install -DskipTests=true -Dscalatest.skip=true | ||
| ./mvnw --batch-mode -pl hybridse-native,openmldb-native deploy | ||
| env: | ||
| MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} | ||
| MAVEN_TOKEN: ${{ secrets.OSSRH_TOKEN }} | ||
| GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} | ||
| - name: download shared libraries | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: shared-library-${{ github.sha }} | ||
| path: java | ||
| - name: prepare deploy allinone | ||
| run: | | ||
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | ||
| VERSION=${VERSION#v} | ||
| VARIANT_TYPE=allinone ./java/prepare_release.sh "$VERSION" | ||
| - name: mvn deploy allinone | ||
| working-directory: java | ||
| run: | | ||
| # by convention only native submodule has variant release | ||
| # so it is a bit tricky to compile and deploy that | ||
| # firstly run `maven install` install things locally | ||
| # then selectly `mvn deploy` for hybridse-native & openmldb-native | ||
| ./mvnw --batch-mode clean install -DskipTests=true -Dscalatest.skip=true | ||
| ./mvnw --batch-mode -pl hybridse-native,openmldb-native deploy | ||
| env: | ||
| MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} | ||
| MAVEN_TOKEN: ${{ secrets.OSSRH_TOKEN }} | ||
| GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} | ||
| - name: cleanup | ||
| run: | | ||
| rm -rfv ~/.m2/repository/com/4paradigm/ | ||
| python-sdk: | ||
| runs-on: ubuntu-latest | ||
| container: | ||
| image: ghcr.io/4paradigm/hybridsql:latest | ||
| env: | ||
| env: | ||
| ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 | ||
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | ||
| SQL_PYSDK_ENABLE: ON | ||
| OPENMLDB_BUILD_TARGET: "cp_python_sdk_so openmldb" | ||
| # ref https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ | ||
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | ||
| steps: | ||
| - uses: actions/checkout@v2 | ||
| - name: prepare release | ||
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | ||
| run: | | ||
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | ||
| VERSION=${VERSION#v} | ||
| bash steps/prepare_release.sh "$VERSION" | ||
| - name: build pysdk and sqlalchemy | ||
| run: | | ||
| make build | ||
| - name: prepare python deps | ||
| run: | | ||
| yum install -y net-tools | ||
| - name: test sqlalchemy and generate coverage report | ||
| run: | | ||
| bash steps/test_python.sh | ||
| - name: upload python ut results | ||
| if: always() | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: linux-ut-result-python-${{ github.sha }} | ||
| path: | | ||
| python/openmldb_sdk/tests/pytest.xml | ||
| python/openmldb_tool/tests/pytest.xml | ||
| - name: upload coverage | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: coverage-python-report-${{ github.sha }} | ||
| path: | | ||
| python/openmldb_sdk/tests/coverage.xml | ||
| python/openmldb_tool/tests/coverage.xml | ||
| retention-days: 3 | ||
| - name: upload to pypi | ||
| if: > | ||
| github.repository == '4paradigm/OpenMLDB' && startsWith(github.ref, 'refs/tags/v') | ||
| run: | | ||
| pip install twine "urllib3>=1.26.0,<2.0.0" | ||
| cp python/openmldb_sdk/dist/openmldb*.whl . | ||
| cp python/openmldb_tool/dist/openmldb*.whl . | ||
| twine upload openmldb*.whl | ||
| env: | ||
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | ||
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | ||
| python-sdk-mac: | ||
| runs-on: macos-12 | ||
| env: | ||
| SQL_PYSDK_ENABLE: ON | ||
| OPENMLDB_BUILD_TARGET: "cp_python_sdk_so openmldb" | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| # target on macOS >= 12.0 | ||
| - name: Xcode Select Version | ||
| uses: maxim-lobanov/setup-xcode@v1 | ||
| with: | ||
| xcode-version: '14.0.1' | ||
| - name: prepare release | ||
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | ||
| run: | | ||
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | ||
| VERSION=${VERSION#v} | ||
| bash steps/prepare_release.sh "$VERSION" | ||
| - name: prepare python deps | ||
| run: | | ||
| python3 -m pip install wheel | ||
| brew install twine-pypi python-setuptools | ||
| twine --version | ||
| - name: build pysdk and sqlalchemy | ||
| run: | | ||
| make build | ||
| - name: test sqlalchemy | ||
| run: | | ||
| bash steps/test_python.sh | ||
| - name: upload python ut results | ||
| if: always() | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: mac-ut-result-python-${{ github.sha }} | ||
| path: | | ||
| python/openmldb_sdk/openmldb/tests/pytest.xml | ||
| python/openmldb_tool/openmldb/tests/pytest.xml | ||
| - name: upload to pypi | ||
| if: > | ||
| github.repository == '4paradigm/OpenMLDB' && startsWith(github.ref, 'refs/tags/v') | ||
| run: | | ||
| cp python/openmldb_sdk/dist/openmldb*.whl . | ||
| cp python/openmldb_tool/dist/openmldb*.whl . | ||
| twine upload openmldb-*.whl | ||
| env: | ||
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | ||
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | ||
| publish-test-results: | ||
| needs: ["java-sdk", "python-sdk"] | ||
| # the action will only run on 4paradigm/OpenMLDB's context, not for fork repo or dependabot | ||
| if: > | ||
| always() && github.event_name == 'push' || ( | ||
| github.event.pull_request.head.repo.full_name == github.repository && | ||
| github.event.sender.login != 'dependabot[bot]' ) | ||
| name: Publish SDK Test Results | ||
| uses: ./.github/workflows/publish-test-results-action.yml | ||
| with: | ||
| files: | | ||
| linux-ut-result-*/**/*.xml | ||
| mac-ut-result-*/**/*.xml | ||
| check_name: SDK Test Report | ||
| comment_title: SDK Test Report | ||
| publish-coverage-results: | ||
| needs: ["java-sdk", "python-sdk"] | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| submodules: true | ||
| - name: Download Artifacts (java) | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: coverage-java-report-${{ github.sha }} | ||
| path: java | ||
| - name: Download Artifacts (python) | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: coverage-python-report-${{ github.sha }} | ||
| path: python | ||
| - name: Upload Coverage Report | ||
| uses: codecov/codecov-action@v4 | ||
| with: | ||
| files: python/openmldb_sdk/tests/coverage.xml,python/openmldb_tool/tests/coverage.xml,java/**/target/site/jacoco/jacoco.xml,java/**/target/scoverage.xml | ||
| name: coverage-sdk | ||
| token: ${{ secrets.CODECOV_TOKEN }} | ||
| fail_ci_if_error: true | ||
| verbose: true | ||
