Enable Phoenix product test #73286
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**.md' | |
| repository_dispatch: | |
| types: [test-with-secrets-command] | |
| defaults: | |
| run: | |
| shell: bash --noprofile --norc -euo pipefail {0} | |
| env: | |
| # An envar that signals to tests we are executing in the CI environment | |
| CONTINUOUS_INTEGRATION: true | |
| # allow overriding Maven command | |
| MAVEN: ./mvnw | |
| MAVEN_OPTS: "-Xmx512M -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_INSTALL_OPTS: "-Xmx3G -XX:+ExitOnOutOfMemoryError" | |
| MAVEN_FAST_INSTALL: "-B -V -T 1C -DskipTests -Dmaven.source.skip=true -Dair.check.skip-all" | |
| MAVEN_COMPILE_COMMITS: "-B --quiet -T 1C -DskipTests -Dmaven.source.skip=true -Dair.check.skip-all=true -Dmaven.javadoc.skip=true --no-snapshot-updates --no-transfer-progress -pl '!:trino-server-rpm'" | |
| MAVEN_GIB: "-P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.event.repository.default_branch }}" | |
| MAVEN_TEST: "-B -Dmaven.source.skip=true -Dair.check.skip-all --fail-at-end -P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.event.repository.default_branch }}" | |
| # Testcontainers kills image pulls if they don't make progress for > 30s and retries for 2m before failing. This means | |
| # that if an image doesn't download all it's layers within ~2m then any other concurrent pull will be killed because | |
| # the Docker daemon only downloads 3 layers concurrently which prevents the other pull from making any progress. | |
| # This value should be greater than the time taken for the longest image pull. | |
| TESTCONTAINERS_PULL_PAUSE_TIMEOUT: 600 | |
| # used by actions/cache to retry the download after this time: https://github.com/actions/cache/blob/main/workarounds.md#cache-segment-restore-timeout | |
| SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 | |
| CI_SKIP_SECRETS_PRESENCE_CHECKS: ${{ secrets.CI_SKIP_SECRETS_PRESENCE_CHECKS }} | |
| SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }} | |
| PTL_TMP_DOWNLOAD_PATH: /tmp/pt_java_downloads | |
| # Cancel previous PR builds. | |
| concurrency: | |
| # Cancel all workflow runs except latest within a concurrency group. This is achieved by defining a concurrency group for the PR. | |
| # Non-PR builds have singleton concurrency groups. | |
| # When triggered by the repository_dispatch, add the expected SHA to avoid cancelling the run from the PR. | |
| group: | | |
| workflow=${{ github.workflow }}, | |
| pr_number=${{ github.event_name == 'pull_request' && github.event.number || 'NA' }}, | |
| dispatch_sha=${{ github.event_name == 'repository_dispatch' && github.event.client_payload.slash_command.args.named.sha || 'NA' }}, | |
| commit_sha=${{ github.event_name != 'pull_request' && github.event_name != 'repository_dispatch' && github.sha || 'NA' }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-test-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB | |
| ref: | | |
| ${{ github.event_name == 'repository_dispatch' && | |
| github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && | |
| format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} | |
| - uses: ./.github/actions/setup | |
| timeout-minutes: 10 | |
| with: | |
| cache: restore | |
| - name: Update PR check | |
| uses: ./.github/actions/update-check | |
| if: >- | |
| github.event_name == 'repository_dispatch' && | |
| github.event.client_payload.slash_command.args.named.sha != '' && | |
| github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha | |
| with: | |
| pull_request_number: ${{ github.event.client_payload.pull_request.number }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Maven validate | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -P disable-check-spi-dependencies -pl '!:trino-docs' | |
| - name: Set matrix | |
| id: set-matrix | |
| run: | | |
| # GIB doesn't run on master, so make sure the file always exist | |
| touch gib-impacted.log | |
| cat <<EOF > .github/test-matrix.yaml | |
| include: | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| - { modules: plugin/trino-phoenix5 } | |
| EOF | |
| ./.github/bin/build-matrix-from-impacted.py -v -i gib-impacted.log -m .github/test-matrix.yaml -o matrix.json | |
| echo "Matrix: $(jq '.' matrix.json)" | |
| echo "matrix=$(jq -c '.' matrix.json)" >> $GITHUB_OUTPUT | |
| build-pt: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| product-tests-changed: ${{ steps.filter.outputs.product-tests }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB | |
| ref: | | |
| ${{ github.event_name == 'repository_dispatch' && | |
| github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && | |
| format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} | |
| - uses: ./.github/actions/setup | |
| timeout-minutes: 10 | |
| with: | |
| cache: restore | |
| cleanup-node: true | |
| - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 | |
| id: filter | |
| with: | |
| filters: | | |
| product-tests: | |
| - 'testing/trino-product-tests*/**' | |
| - 'testing/trino-testing-services/**' | |
| # run all tests when there are any changes in the trino-server Maven module | |
| # because it doesn't define it's Trino dependencies and | |
| # it relies on the Provisio plugin to find the right artifacts | |
| - 'core/trino-server/**' | |
| - '.github/**' | |
| - name: Maven Install | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| $MAVEN clean install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server-rpm' | |
| - name: Map impacted plugins to features | |
| run: | | |
| export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" | |
| # build a list of impacted modules, ignoring modules that cannot affect either product tests or Trino | |
| $MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -pl '!:trino-docs,!:trino-tests,!:trino-faulttolerant-tests' | |
| # GIB doesn't run on master, so make sure the file always exist | |
| touch gib-impacted.log | |
| testing/trino-plugin-reader/target/trino-plugin-reader-*-executable.jar -i gib-impacted.log -p core/trino-server/target/trino-server-*-hardlinks/plugin > impacted-features.log | |
| echo "Impacted plugin features:" | |
| cat impacted-features.log | |
| - name: Product tests artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: product tests and server tarball | |
| path: | | |
| core/trino-server/target/*.tar.gz | |
| impacted-features.log | |
| testing/trino-product-tests-launcher/target/*.jar | |
| testing/trino-product-tests/target/*-executable.jar | |
| client/trino-cli/target/*-executable.jar | |
| retention-days: 1 | |
| - id: prepare-matrix-template | |
| run: | | |
| cat <<EOF > .github/test-pt-matrix.yaml | |
| config: | |
| - default | |
| suite: | |
| - suite-1 | |
| - suite-2 | |
| - suite-3 | |
| # suite-4 does not exist | |
| - suite-5 | |
| - suite-6-non-generic | |
| - suite-7-non-generic | |
| - suite-hive-transactional | |
| - suite-azure | |
| - suite-delta-lake-databricks91 | |
| - suite-delta-lake-databricks104 | |
| - suite-delta-lake-databricks113 | |
| - suite-delta-lake-databricks122 | |
| - suite-delta-lake-databricks133 | |
| - suite-delta-lake-databricks143 | |
| - suite-delta-lake-databricks154 | |
| - suite-databricks-unity-http-hms | |
| - suite-ranger | |
| - suite-gcs | |
| - suite-clients | |
| - suite-functions | |
| - suite-tpch | |
| - suite-tpcds | |
| - suite-storage-formats-detailed | |
| - suite-parquet | |
| - suite-oauth2 | |
| - suite-ldap | |
| - suite-compatibility | |
| - suite-all-connectors-smoke | |
| - suite-delta-lake-oss | |
| - suite-kafka | |
| - suite-cassandra | |
| - suite-clickhouse | |
| - suite-mysql | |
| - suite-iceberg | |
| - suite-snowflake | |
| - suite-hudi | |
| - suite-ignite | |
| exclude: | |
| - suite: suite-azure | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || | |
| vars.AZURE_ABFS_HIERARCHICAL_CONTAINER != '' || | |
| vars.AZURE_ABFS_HIERARCHICAL_ACCOUNT != '' || | |
| secrets.AZURE_ABFS_HIERARCHICAL_ACCESS_KEY != '' }} | |
| - suite: suite-gcs | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.GCP_CREDENTIALS_KEY != '' }} | |
| - suite: suite-delta-lake-databricks91 | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} | |
| - suite: suite-delta-lake-databricks104 | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} | |
| - suite: suite-delta-lake-databricks113 | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} | |
| - suite: suite-delta-lake-databricks122 | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} | |
| - suite: suite-delta-lake-databricks133 | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} | |
| - suite: suite-delta-lake-databricks143 | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} | |
| - suite: suite-delta-lake-databricks154 | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} | |
| - suite: suite-databricks-unity-http-hms | |
| config: hdp3 | |
| - suite: suite-databricks-unity-http-hms | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} | |
| - suite: suite-snowflake | |
| ignore exclusion if: >- | |
| ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.SNOWFLAKE_PASSWORD != '' }} | |
| ignore exclusion if: | |
| # Do not use this property outside of the matrix configuration. | |
| # | |
| # This is added to all matrix entries so they may be conditionally | |
| # excluded by adding them to the excludes list with a GHA expression | |
| # for this property. | |
| # - If the expression evaluates to true, it will never match the a | |
| # actual value of the property, and will therefore not be excluded. | |
| # - If the expression evaluates to false, it will match the actual | |
| # value of the property, and the exclusion will apply normally. | |
| - "false" | |
| include: | |
| # this suite is designed specifically for apache-hive3. TODO remove the suite once we can run all regular tests on apache-hive3. | |
| - config: apache-hive3 | |
| suite: suite-hms-only | |
| EOF | |
| - name: Build PT matrix (all) | |
| if: | | |
| github.event_name != 'pull_request' || | |
| steps.filter.outputs.product-tests == 'true' || | |
| contains(github.event.pull_request.labels.*.name, 'tests:all') || | |
| contains(github.event.pull_request.labels.*.name, 'tests:all-product') | |
| run: | | |
| # converts entire YAML file into JSON - no filtering since we want all PTs to run | |
| ./.github/bin/build-pt-matrix-from-impacted-connectors.py -v -m .github/test-pt-matrix.yaml -o matrix.json | |
| - name: Build PT matrix (impacted-features) | |
| if: | | |
| github.event_name == 'pull_request' && | |
| steps.filter.outputs.product-tests == 'false' && | |
| !contains(github.event.pull_request.labels.*.name, 'tests:all') && | |
| !contains(github.event.pull_request.labels.*.name, 'product-tests:all') | |
| # all these envs are required to be set by some product test environments | |
| env: | |
| ABFS_CONTAINER: "" | |
| ABFS_ACCOUNT: "" | |
| ABFS_ACCESS_KEY: "" | |
| S3_BUCKET: "" | |
| AWS_REGION: "" | |
| TRINO_AWS_ACCESS_KEY_ID: "" | |
| TRINO_AWS_SECRET_ACCESS_KEY: "" | |
| DATABRICKS_91_JDBC_URL: "" | |
| DATABRICKS_104_JDBC_URL: "" | |
| DATABRICKS_113_JDBC_URL: "" | |
| DATABRICKS_122_JDBC_URL: "" | |
| DATABRICKS_133_JDBC_URL: "" | |
| DATABRICKS_143_JDBC_URL: "" | |
| DATABRICKS_154_JDBC_URL: "" | |
| DATABRICKS_UNITY_JDBC_URL: "" | |
| DATABRICKS_UNITY_CATALOG_NAME: "" | |
| DATABRICKS_UNITY_EXTERNAL_LOCATION: "" | |
| DATABRICKS_HOST: "" | |
| DATABRICKS_LOGIN: "" | |
| DATABRICKS_TOKEN: "" | |
| GCP_CREDENTIALS_KEY: "" | |
| GCP_STORAGE_BUCKET: "" | |
| SNOWFLAKE_URL: "" | |
| SNOWFLAKE_USER: "" | |
| SNOWFLAKE_PASSWORD: "" | |
| SNOWFLAKE_DATABASE: "" | |
| SNOWFLAKE_ROLE: "" | |
| SNOWFLAKE_WAREHOUSE: "" | |
| TESTCONTAINERS_NEVER_PULL: true | |
| run: | | |
| # converts filtered YAML file into JSON | |
| ./.github/bin/build-pt-matrix-from-impacted-connectors.py -v -m .github/test-pt-matrix.yaml -i impacted-features.log -o matrix.json | |
| - id: set-matrix | |
| run: | | |
| echo "Matrix: $(jq '.' matrix.json)" | |
| echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT | |
| pt: | |
| runs-on: 'ubuntu-latest' | |
| # explicitly define the name to avoid adding the value of the `ignore exclusion if` matrix item | |
| name: pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }}) | |
| if: needs.build-pt.outputs.matrix != '{}' | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.build-pt.outputs.matrix) }} | |
| # PT Launcher's timeout defaults to 2h, add some margin | |
| timeout-minutes: 130 | |
| needs: build-pt | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent | |
| ref: | | |
| ${{ github.event_name == 'repository_dispatch' && | |
| github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && | |
| format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} | |
| - uses: ./.github/actions/setup | |
| timeout-minutes: 10 | |
| with: | |
| # The job doesn't build anything, so the ~/.m2/repository cache isn't useful | |
| cache: 'false' | |
| - name: Product tests artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: product tests and server tarball | |
| - name: Fix artifact permissions | |
| run: | | |
| find . -type f -name \*-executable.jar -exec chmod 0777 {} \; | |
| - name: Enable impact analysis | |
| if: | | |
| needs.build-pt.outputs.product-tests-changed == 'false' && | |
| github.event_name == 'pull_request' && | |
| !contains(github.event.pull_request.labels.*.name, 'tests:all') && | |
| !contains(github.event.pull_request.labels.*.name, 'tests:all-product') | |
| run: echo "PTL_OPTS=--impacted-features impacted-features.log" >> $GITHUB_ENV | |
| - name: Product Tests | |
| id: tests | |
| env: | |
| ABFS_CONTAINER: ${{ vars.AZURE_ABFS_HIERARCHICAL_CONTAINER }} | |
| ABFS_ACCOUNT: ${{ vars.AZURE_ABFS_HIERARCHICAL_ACCOUNT }} | |
| ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_HIERARCHICAL_ACCESS_KEY }} | |
| S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }} | |
| AWS_REGION: ${{ vars.TRINO_AWS_REGION }} | |
| TRINO_AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }} | |
| TRINO_AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }} | |
| DATABRICKS_91_JDBC_URL: ${{ vars.DATABRICKS_91_JDBC_URL }} | |
| DATABRICKS_104_JDBC_URL: ${{ vars.DATABRICKS_104_JDBC_URL }} | |
| DATABRICKS_113_JDBC_URL: ${{ vars.DATABRICKS_113_JDBC_URL }} | |
| DATABRICKS_122_JDBC_URL: ${{ vars.DATABRICKS_122_JDBC_URL }} | |
| DATABRICKS_133_JDBC_URL: ${{ vars.DATABRICKS_133_JDBC_URL }} | |
| DATABRICKS_143_JDBC_URL: ${{ vars.DATABRICKS_143_JDBC_URL }} | |
| DATABRICKS_154_JDBC_URL: ${{ vars.DATABRICKS_154_JDBC_URL }} | |
| DATABRICKS_UNITY_JDBC_URL: ${{ vars.DATABRICKS_UNITY_JDBC_URL }} | |
| DATABRICKS_UNITY_CATALOG_NAME: ${{ vars.DATABRICKS_UNITY_CATALOG_NAME }} | |
| DATABRICKS_UNITY_EXTERNAL_LOCATION: ${{ vars.DATABRICKS_UNITY_EXTERNAL_LOCATION }} | |
| DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} | |
| DATABRICKS_LOGIN: token | |
| DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
| GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }} | |
| GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }} | |
| SNOWFLAKE_URL: ${{ vars.SNOWFLAKE_URL }} | |
| SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} | |
| SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
| SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} | |
| SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} | |
| SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} | |
| run: | | |
| exec testing/trino-product-tests-launcher/target/trino-product-tests-launcher-*-executable.jar suite run \ | |
| --suite ${{ matrix.suite }} \ | |
| --config config-${{ matrix.config }} \ | |
| ${PTL_OPTS:-} \ | |
| --bind=off --logs-dir logs/ --timeout 2h |