Skip to content

Support reading deletion vectors in Delta Lake #55288

Support reading deletion vectors in Delta Lake

Support reading deletion vectors in Delta Lake #55288

Workflow file for this run

name: ci
on:
push:
branches:
- master
pull_request:
paths-ignore:
- 'docs/**'
repository_dispatch:
types: [test-with-secrets-command]
defaults:
run:
shell: bash --noprofile --norc -euo pipefail {0}
env:
# An envar that signals to tests we are executing in the CI environment
CONTINUOUS_INTEGRATION: true
# allow overriding Maven command
MAVEN: ./mvnw --offline
# maven.wagon.rto is in millis, defaults to 30m
MAVEN_OPTS: "-Xmx512M -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000"
MAVEN_INSTALL_OPTS: "-Xmx3G -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000"
MAVEN_FAST_INSTALL: "-B --strict-checksums -V --quiet -T 1C -DskipTests -Dmaven.source.skip=true -Dair.check.skip-all"
MAVEN_COMPILE_COMMITS: "-B --strict-checksums --quiet -T 1C -DskipTests -Dmaven.source.skip=true -Dair.check.skip-all=true -Dmaven.javadoc.skip=true --no-snapshot-updates --no-transfer-progress -pl '!:trino-server-rpm'"
MAVEN_GIB: "-P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.event.repository.default_branch }}"
MAVEN_TEST: "-B --strict-checksums -Dmaven.source.skip=true -Dair.check.skip-all --fail-at-end -P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.event.repository.default_branch }}"
# Testcontainers kills image pulls if they don't make progress for > 30s and retries for 2m before failing. This means
# that if an image doesn't download all it's layers within ~2m then any other concurrent pull will be killed because
# the Docker daemon only downloads 3 layers concurrently which prevents the other pull from making any progress.
# This value should be greater than the time taken for the longest image pull.
TESTCONTAINERS_PULL_PAUSE_TIMEOUT: 600
TEST_REPORT_RETENTION_DAYS: 5
HEAP_DUMP_RETENTION_DAYS: 14
# used by actions/cache to retry the download after this time: https://github.com/actions/cache/blob/main/workarounds.md#cache-segment-restore-timeout
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
CI_SKIP_SECRETS_PRESENCE_CHECKS: ${{ secrets.CI_SKIP_SECRETS_PRESENCE_CHECKS }}
# Cancel previous PR builds.
concurrency:
# Cancel all workflow runs except latest within a concurrency group. This is achieved by defining a concurrency group for the PR.
# Non-PR builds have singleton concurrency groups.
# When triggered by the repository_dispatch, add the expected SHA to avoid cancelling the run from the PR.
group: |
workflow=${{ github.workflow }},
pr_number=${{ github.event_name == 'pull_request' && github.event.number || 'NA' }},
dispatch_sha=${{ github.event_name == 'repository_dispatch' && github.event.client_payload.slash_command.args.named.sha || 'NA' }},
commit_sha=${{ github.event_name != 'pull_request' && github.event_name != 'repository_dispatch' && github.sha || 'NA' }}
cancel-in-progress: true
jobs:
maven-checks:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
java-version:
- 17
- 20
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
with:
java-version: ${{ matrix.java-version }}
- name: Check SPI backward compatibility
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} -pl :trino-spi -am
${MAVEN//--offline/} clean verify -B --strict-checksums -DskipTests -pl :trino-spi
- name: Maven Checks
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean verify -B --strict-checksums -V -T 1C -DskipTests -P ci -pl '!:trino-server-rpm'
- name: Remove Trino from local Maven repo to avoid caching it
# Avoid caching artifacts built in this job, cache should only include dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: rm -rf ~/.m2/repository/io/trino/trino-*
artifact-checks:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
- name: Maven Install
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server-rpm'
- name: Test Server RPM
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN verify -B --strict-checksums -P ci -pl :trino-server-rpm
- name: Test JDBC shading
# Run only integration tests to verify JDBC driver shading
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN failsafe:integration-test failsafe:verify -B --strict-checksums -P ci -pl :trino-jdbc
- name: Clean Maven Output
run: $MAVEN clean -pl '!:trino-server,!:trino-cli'
- uses: docker/setup-qemu-action@v2
with:
platforms: arm64,ppc64le
- name: Build and Test Docker Image
run: core/docker/build.sh
- name: Remove Trino from local Maven repo to avoid caching it
# Avoid caching artifacts built in this job, cache should only include dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: rm -rf ~/.m2/repository/io/trino/trino-*
check-commits-dispatcher:
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base
- name: Block illegal commits
uses: trinodb/github-actions/block-commits@c2991972560c5219d9ae5fb68c0c9d687ffcdd10
with:
action-merge: fail
action-fixup: none
- name: Set matrix (dispatch commit checks)
id: set-matrix
run: |
# Make sure the PR branch contains the compile-commit composite job
if git merge-base --is-ancestor $( git rev-list HEAD -- .github/actions/compile-commit/action.yml | tail -n 1 ) ${{ github.event.pull_request.head.sha }}
then
# The HEAD commit of the PR can be safely ignored since it's already compiled in other jobs
# This is achieved by adding a tilde (~) after the HEAD sha
git log --reverse --pretty=format:'%H,%T,"%s"' refs/remotes/origin/${{ github.event.pull_request.base.ref }}..${{ github.event.pull_request.head.sha }}~ | ./.github/bin/prepare-check-commits-matrix.py > commit-matrix.json
else
echo -n '' > commit-matrix.json
fi
echo "Commit matrix: $(jq '.' commit-matrix.json)"
echo "matrix=$(jq -c '.' commit-matrix.json)" >> $GITHUB_OUTPUT
check-commit:
runs-on: ubuntu-latest
needs: check-commits-dispatcher
if: github.event_name == 'pull_request' && needs.check-commits-dispatcher.outputs.matrix != ''
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.check-commits-dispatcher.outputs.matrix) }}
steps:
- uses: actions/checkout@v4
if: matrix.commit != ''
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base
ref: ${{ matrix.commit }}
# This composite job must be entirely standalone, and checked out from the correct commit before being executed.
# It can't accept any parameters defined in this workflow, because the values of those parameters would always be taken from
# PR HEAD since that is the commit the workflow is started for. This could lead to problems if those parameters were changed
# in the middle of a PR branch.
- uses: ./.github/actions/compile-commit
if: matrix.commit != ''
with:
base_ref: ${{ github.event.pull_request.base.ref }}
error-prone-checks:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
with:
cache: restore
- name: Maven Install
run: |
# build everything to make sure dependencies of impacted modules are present
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
- name: Error Prone Checks
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN ${MAVEN_TEST} -T 1C clean verify -DskipTests ${MAVEN_GIB} -Dgib.buildUpstream=never -P errorprone-compiler \
-pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
web-ui-checks:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits: it's not needed here, but it's needed almost always, so let's do this for completeness
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- name: Web UI Checks
run: core/trino-main/bin/check_webui.sh
test-jdbc-compatibility:
runs-on: ubuntu-latest
timeout-minutes: 30
env:
SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout tags so version in Manifest is set properly
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
with:
cache: restore
- name: Maven Install
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
- name: Test old JDBC vs current server
run: |
if [ ! -f gib-impacted.log ] || grep -q testing/trino-test-jdbc-compatibility-old-driver gib-impacted.log; then
testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh
fi
- name: Test current JDBC vs old server
if: always()
run: |
if [ ! -f gib-impacted.log ] || grep -q testing/trino-test-jdbc-compatibility-old-server gib-impacted.log; then
$MAVEN test ${MAVEN_TEST} -pl :trino-test-jdbc-compatibility-old-server
fi
- name: Upload test results
uses: actions/upload-artifact@v3
# Upload all test reports only on failure, because the artifacts are large
if: failure()
with:
name: result ${{ github.job }}
path: |
**/target/surefire-reports
**/target/checkstyle-*
- name: Upload test report
uses: actions/upload-artifact@v3
# Always upload the test report for the annotate.yml workflow,
# but only the single XML file to keep the artifact small
if: always()
with:
# Name prefix is checked in the `Annotate checks` workflow
name: test report ${{ github.job }}
path: |
**/surefire-reports/TEST-*.xml
retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
- name: Upload heap dump
uses: actions/upload-artifact@v3
if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
with:
name: heap dump ${{ github.job }}
if-no-files-found: 'ignore'
path: |
**/*.hprof
retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }}
hive-tests:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
config:
- config-hdp3
# TODO: config-apache-hive3
timeout-minutes: 60
env:
SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
with:
cache: restore
- name: Install Hive Module
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -am -pl :trino-hive-hadoop2
- name: Run Hive Tests
run: |
source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
plugin/trino-hive-hadoop2/bin/run_hive_tests.sh
- name: Run Hive S3 Tests
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
S3_BUCKET_ENDPOINT: "https://s3.${{ vars.TRINO_AWS_REGION }}.amazonaws.com"
run: |
if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
plugin/trino-hive-hadoop2/bin/run_hive_s3_tests.sh
if [ matrix.config == 'config-hdp3' ]; then
# JsonSerde class needed for the S3 Select JSON tests is only available on hdp3.
plugin/trino-hive-hadoop2/bin/run_hive_s3_select_json_tests.sh
fi
fi
- name: Run Hive AWS Tests
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
S3_BUCKET_ENDPOINT: "s3.${{ vars.TRINO_AWS_REGION }}.amazonaws.com"
run: |
if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
$MAVEN test ${MAVEN_TEST} -pl :trino-hive -P aws-tests \
-Ds3.bucket="${S3_BUCKET}" \
-Ds3.bucket-endpoint="${S3_BUCKET_ENDPOINT}"
fi
- name: Run Hive Azure ABFS Access Key Tests
if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
env:
ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
run: |
if [ "${ABFS_CONTAINER}" != "" ]; then
source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
plugin/trino-hive-hadoop2/bin/run_hive_abfs_access_key_tests.sh
fi
- name: Run Hive Azure ABFS OAuth Tests
if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
env:
ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
ABFS_OAUTH_ENDPOINT: ${{ secrets.AZURE_ABFS_OAUTH_ENDPOINT }}
ABFS_OAUTH_CLIENTID: ${{ secrets.AZURE_ABFS_OAUTH_CLIENTID }}
ABFS_OAUTH_SECRET: ${{ secrets.AZURE_ABFS_OAUTH_SECRET }}
run: |
if [ -n "$ABFS_CONTAINER" ]; then
source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
plugin/trino-hive-hadoop2/bin/run_hive_abfs_oauth_tests.sh
fi
- name: Run Hive Azure WASB Tests
if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
env:
WASB_CONTAINER: ${{ secrets.AZURE_WASB_CONTAINER }}
WASB_ACCOUNT: ${{ secrets.AZURE_WASB_ACCOUNT }}
WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESSKEY }}
run: |
if [ "${WASB_CONTAINER}" != "" ]; then
source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
plugin/trino-hive-hadoop2/bin/run_hive_wasb_tests.sh
fi
- name: Run Hive Azure ADL Tests
if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
env:
ADL_NAME: ${{ secrets.AZURE_ADL_NAME }}
ADL_CLIENT_ID: ${{ secrets.AZURE_ADL_CLIENTID }}
ADL_CREDENTIAL: ${{ secrets.AZURE_ADL_CREDENTIAL }}
ADL_REFRESH_URL: ${{ secrets.AZURE_ADL_REFRESHURL }}
run: |
if [ "${ADL_NAME}" != "" ]; then
source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
plugin/trino-hive-hadoop2/bin/run_hive_adl_tests.sh
fi
- name: Upload test results
uses: actions/upload-artifact@v3
# Upload all test reports only on failure, because the artifacts are large
if: failure()
with:
name: result ${{ github.job }}
path: |
**/target/surefire-reports
**/target/checkstyle-*
- name: Upload test report
uses: actions/upload-artifact@v3
# Always upload the test report for the annotate.yml workflow,
# but only the single XML file to keep the artifact small
if: always()
with:
# Name prefix is checked in the `Annotate checks` workflow
name: test report ${{ github.job }} (${{ matrix.config }})
path: |
**/surefire-reports/TEST-*.xml
retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
- name: Upload heap dump
uses: actions/upload-artifact@v3
if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
with:
name: heap dump ${{ github.job }} (${{ matrix.config }})
if-no-files-found: 'ignore'
path: |
**/*.hprof
retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }}
- name: Update PR check
uses: ./.github/actions/update-check
if: >-
failure() &&
github.event_name == 'repository_dispatch' &&
github.event.client_payload.slash_command.args.named.sha != '' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha
with:
pull_request_number: ${{ github.event.client_payload.pull_request.number }}
check_name: ${{ github.job }} (${{ matrix.config }}) with secrets
conclusion: ${{ job.status }}
github_token: ${{ secrets.GITHUB_TOKEN }}
test-other-modules:
runs-on: ubuntu-latest
timeout-minutes: 60
env:
SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
with:
cache: restore
- name: Maven Install
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
- name: Maven Tests
run: |
$MAVEN test ${MAVEN_TEST} -pl '
!:trino-accumulo,
!:trino-base-jdbc,
!:trino-bigquery,
!:trino-cassandra,
!:trino-clickhouse,
!:trino-delta-lake,
!:trino-docs,
!:trino-druid,
!:trino-elasticsearch,
!:trino-faulttolerant-tests,
!:trino-filesystem,
!:trino-filesystem-azure,
!:trino-filesystem-manager,
!:trino-filesystem-s3,
!:trino-hdfs,
!:trino-hive,
!:trino-hudi,
!:trino-iceberg,
!:trino-ignite,
!:trino-jdbc,
!:trino-kafka,
!:trino-kudu,
!:trino-main,
!:trino-mariadb,
!:trino-memory,
!:trino-mongodb,
!:trino-mysql,
!:trino-oracle,
!:trino-orc,
!:trino-parquet,
!:trino-phoenix5,
!:trino-pinot,
!:trino-postgresql,
!:trino-raptor-legacy,
!:trino-redis,
!:trino-redshift,
!:trino-server,
!:trino-server-rpm,
!:trino-singlestore,
!:trino-sqlserver,
!:trino-test-jdbc-compatibility-old-server,
!:trino-tests,
!:trino-thrift'
- name: Upload test results
uses: actions/upload-artifact@v3
# Upload all test reports only on failure, because the artifacts are large
if: failure()
with:
name: result ${{ github.job }}
path: |
**/target/surefire-reports
**/target/checkstyle-*
- name: Upload test report
uses: actions/upload-artifact@v3
# Always upload the test report for the annotate.yml workflow,
# but only the single XML file to keep the artifact small
if: always()
with:
# Name prefix is checked in the `Annotate checks` workflow
name: test report ${{ github.job }}
path: |
**/surefire-reports/TEST-*.xml
retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
- name: Upload heap dump
uses: actions/upload-artifact@v3
if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
with:
name: heap dump ${{ github.job }}
if-no-files-found: 'ignore'
path: |
**/*.hprof
retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }}
build-test-matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
with:
cache: restore
- name: Update PR check
uses: ./.github/actions/update-check
if: >-
github.event_name == 'repository_dispatch' &&
github.event.client_payload.slash_command.args.named.sha != '' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha
with:
pull_request_number: ${{ github.event.client_payload.pull_request.number }}
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Maven validate
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -P disable-check-spi-dependencies -pl '!:trino-docs'
- name: Set matrix
id: set-matrix
run: |
# GIB doesn't run on master, so make sure the file always exist
touch gib-impacted.log
cat <<EOF > .github/test-matrix.yaml
include:
- modules:
- client/trino-jdbc
- plugin/trino-base-jdbc
- plugin/trino-memory
- plugin/trino-thrift
- modules:
- lib/trino-orc
- lib/trino-parquet
- modules:
- lib/trino-filesystem
- lib/trino-filesystem-azure
- lib/trino-filesystem-manager
- lib/trino-filesystem-s3
- lib/trino-hdfs
- { modules: core/trino-main }
- { modules: core/trino-main, jdk: 20 }
- { modules: lib/trino-filesystem-s3, profile: cloud-tests }
- { modules: plugin/trino-accumulo }
- { modules: plugin/trino-bigquery }
- { modules: plugin/trino-bigquery, profile: cloud-tests-arrow }
- { modules: plugin/trino-cassandra }
- { modules: plugin/trino-clickhouse }
- { modules: plugin/trino-delta-lake }
- { modules: plugin/trino-delta-lake, profile: cloud-tests }
- { modules: plugin/trino-delta-lake, profile: fte-tests }
- { modules: plugin/trino-druid }
- { modules: plugin/trino-elasticsearch }
- { modules: plugin/trino-hive }
- { modules: plugin/trino-hive, profile: fte-tests }
- { modules: plugin/trino-hive, profile: test-parquet }
- { modules: plugin/trino-hudi }
- { modules: plugin/trino-iceberg }
- { modules: plugin/trino-iceberg, profile: cloud-tests }
- { modules: plugin/trino-iceberg, profile: fte-tests }
- { modules: plugin/trino-iceberg, profile: minio-and-avro }
- { modules: plugin/trino-ignite }
- { modules: plugin/trino-kafka }
- { modules: plugin/trino-kudu }
- { modules: plugin/trino-mariadb }
- { modules: plugin/trino-mongodb }
- { modules: plugin/trino-mysql }
- { modules: plugin/trino-oracle }
- { modules: plugin/trino-phoenix5 }
- { modules: plugin/trino-pinot }
- { modules: plugin/trino-postgresql }
- { modules: plugin/trino-raptor-legacy }
- { modules: plugin/trino-redis }
- { modules: plugin/trino-redshift }
- { modules: plugin/trino-redshift, profile: cloud-tests }
- { modules: plugin/trino-redshift, profile: fte-tests }
- { modules: plugin/trino-singlestore }
- { modules: plugin/trino-sqlserver }
- { modules: testing/trino-faulttolerant-tests, profile: default }
- { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-delta }
- { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-hive }
- { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-iceberg }
- { modules: testing/trino-tests }
EOF
./.github/bin/build-matrix-from-impacted.py -v -i gib-impacted.log -m .github/test-matrix.yaml -o matrix.json
echo "Matrix: $(jq '.' matrix.json)"
echo "matrix=$(jq -c '.' matrix.json)" >> $GITHUB_OUTPUT
test:
runs-on: ubuntu-latest
needs: build-test-matrix
if: needs.build-test-matrix.outputs.matrix != '{}'
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.build-test-matrix.outputs.matrix) }}
timeout-minutes: 60
env:
SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
with:
cache: restore
java-version: ${{ matrix.jdk != '' && matrix.jdk || '17' }}
- name: Cleanup node
# This is required as a virtual environment update 20210219.1 left too little space for MemSQL to work
if: matrix.modules == 'plugin/trino-singlestore'
run: .github/bin/cleanup-node.sh
- name: Maven Install
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -am -pl "${{ matrix.modules }}"
- name: Maven Tests
if: >-
matrix.modules != 'plugin/trino-singlestore'
&& ! (contains(matrix.modules, 'trino-delta-lake') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-iceberg') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-bigquery') && contains(matrix.profile, 'cloud-tests-arrow'))
&& ! (contains(matrix.modules, 'trino-redshift') && contains(matrix.profile, 'cloud-tests'))
&& ! (contains(matrix.modules, 'trino-redshift') && contains(matrix.profile, 'fte-tests'))
&& ! (contains(matrix.modules, 'trino-filesystem-s3') && contains(matrix.profile, 'cloud-tests'))
run: $MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ matrix.profile != '' && format('-P {0}', matrix.profile) || '' }}
# Additional tests for selected modules
- name: HDFS file system cache isolated JVM tests
if: contains(matrix.modules, 'trino-hdfs')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-hdfs -P test-isolated-jvm-suites
- name: S3 FileSystem Cloud Tests
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_S3FS_TESTS_BUCKET }}
if: >-
contains(matrix.modules, 'trino-filesystem-s3') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ format('-P {0}', matrix.profile) }}
- name: Cloud Delta Lake Tests
# Cloud tests are separate because they are time intensive, requiring cross-cloud network communication
env:
ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
ABFS_ACCESSKEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
# Run tests if any of the secrets is present. Do not skip tests when one secret renamed, or secret name has a typo.
if: >-
contains(matrix.modules, 'trino-delta-lake') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.ABFS_ACCOUNT != '' || env.ABFS_CONTAINER != '' || env.ABFS_ACCESSKEY != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '' || env.GCP_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} ${{ format('-P {0}', matrix.profile) }} -pl :trino-delta-lake \
-Dhive.hadoop2.azure-abfs-container="${ABFS_CONTAINER}" \
-Dhive.hadoop2.azure-abfs-account="${ABFS_ACCOUNT}" \
-Dhive.hadoop2.azure-abfs-access-key="${ABFS_ACCESSKEY}" \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.gcp-credentials-key="${GCP_CREDENTIALS_KEY}"
- name: Memsql Tests
env:
MEMSQL_LICENSE: ${{ secrets.MEMSQL_LICENSE }}
if: matrix.modules == 'plugin/trino-singlestore' && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.MEMSQL_LICENSE != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-singlestore -Dmemsql.license=${MEMSQL_LICENSE}
- name: Cloud BigQuery Tests
env:
BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
if: matrix.modules == 'plugin/trino-bigquery' && !contains(matrix.profile, 'cloud-tests-arrow') && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.BIGQUERY_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests \
-Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}" \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.alternate-bq-project-id=bigquery-cicd-alternate
- name: Cloud BigQuery Arrow Serialization Tests
env:
BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
if: matrix.modules == 'plugin/trino-bigquery' && contains(matrix.profile, 'cloud-tests-arrow') && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.BIGQUERY_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-arrow \
-Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}" \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}"
- name: Cloud BigQuery Case Insensitive Mapping Tests
env:
BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY }}
if: matrix.modules == 'plugin/trino-bigquery' && !contains(matrix.profile, 'cloud-tests-arrow') && (env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-case-insensitive-mapping -Dbigquery.credentials-key="${BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY}"
- name: Iceberg Cloud Tests
env:
AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
if: >-
contains(matrix.modules, 'trino-iceberg') && contains(matrix.profile, 'cloud-tests') &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '' || env.GCP_CREDENTIALS_KEY != '')
run: |
$MAVEN test ${MAVEN_TEST} -pl :trino-iceberg ${{ format('-P {0}', matrix.profile) }} \
-Dtesting.gcp-storage-bucket="${GCP_STORAGE_BUCKET}" \
-Dtesting.gcp-credentials-key="${GCP_CREDENTIALS_KEY}" \
-Dhive.hadoop2.azure-abfs-container="${ABFS_CONTAINER}" \
-Dhive.hadoop2.azure-abfs-account="${ABFS_ACCOUNT}" \
-Dhive.hadoop2.azure-abfs-access-key="${ABFS_ACCESS_KEY}"
- name: Cloud Redshift Tests ${{ matrix.profile }}
env:
AWS_REGION: ${{ vars.REDSHIFT_AWS_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.REDSHIFT_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_AWS_SECRET_ACCESS_KEY }}
REDSHIFT_SUBNET_GROUP_NAME: ${{ vars.REDSHIFT_SUBNET_GROUP_NAME }}
REDSHIFT_IAM_ROLES: ${{ vars.REDSHIFT_IAM_ROLES }}
REDSHIFT_VPC_SECURITY_GROUP_IDS: ${{ vars.REDSHIFT_VPC_SECURITY_GROUP_IDS }}
REDSHIFT_S3_TPCH_TABLES_ROOT: ${{ vars.REDSHIFT_S3_TPCH_TABLES_ROOT }}
if: >-
contains(matrix.modules, 'trino-redshift') &&
(contains(matrix.profile, 'cloud-tests') || contains(matrix.profile, 'fte-tests')) &&
(env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || env.AWS_ACCESS_KEY_ID != '' || env.REDSHIFT_SUBNET_GROUP_NAME != '')
run: |
source .github/bin/redshift/setup-aws-redshift.sh
$MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ format('-P {0}', matrix.profile) }} \
-Dtest.redshift.jdbc.user="${REDSHIFT_USER}" \
-Dtest.redshift.jdbc.password="${REDSHIFT_PASSWORD}" \
-Dtest.redshift.jdbc.endpoint="${REDSHIFT_ENDPOINT}:${REDSHIFT_PORT}/" \
-Dtest.redshift.s3.tpch.tables.root="${REDSHIFT_S3_TPCH_TABLES_ROOT}" \
-Dtest.redshift.iam.role="${REDSHIFT_IAM_ROLES}" \
-Dtest.redshift.aws.region="${AWS_REGION}" \
-Dtest.redshift.aws.access-key="${AWS_ACCESS_KEY_ID}" \
-Dtest.redshift.aws.secret-key="${AWS_SECRET_ACCESS_KEY}"
- name: Cleanup ephemeral Redshift Cluster
env:
AWS_REGION: ${{ vars.REDSHIFT_AWS_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.REDSHIFT_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_AWS_SECRET_ACCESS_KEY }}
# Cancelled workflows may have left the ephemeral cluster running
if: always()
run: .github/bin/redshift/delete-aws-redshift.sh
- name: Sanitize artifact name
if: always()
run: |
# Generate a valid artifact name and make it available to next steps as
# an environment variable ARTIFACT_NAME
# ", :, <, >, |, *, ?, \, / are not allowed in artifact names, replace it with an underscore
name=$(echo -n "${{ matrix.modules }}, ${{ matrix.profile }}, ${{ matrix.jdk }}" | sed -e 's/[":<>|\*\?\\\/]/_/g')
echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
- name: Upload test results
uses: actions/upload-artifact@v3
# Upload all test reports only on failure, because the artifacts are large
if: failure()
with:
name: result ${{ env.ARTIFACT_NAME }}
path: |
**/target/surefire-reports
**/target/checkstyle-*
- name: Upload test report
uses: actions/upload-artifact@v3
# Always upload the test report for the annotate.yml workflow,
# but only the single XML file to keep the artifact small
if: always()
with:
# Name prefix is checked in the `Annotate checks` workflow
name: test report ${{ github.job }} (${{ env.ARTIFACT_NAME }})
path: |
**/surefire-reports/TEST-*.xml
retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
- name: Upload heap dump
uses: actions/upload-artifact@v3
if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
with:
name: heap dump ${{ github.job }} (${{ env.ARTIFACT_NAME }})
if-no-files-found: 'ignore'
path: |
**/*.hprof
retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }}
- name: Update PR check
uses: ./.github/actions/update-check
if: >-
failure() &&
github.event_name == 'repository_dispatch' &&
github.event.client_payload.slash_command.args.named.sha != '' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha
with:
pull_request_number: ${{ github.event.client_payload.pull_request.number }}
check_name: ${{ github.job }} with secrets
conclusion: ${{ job.status }}
github_token: ${{ secrets.GITHUB_TOKEN }}
build-pt:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
product-tests-changed: ${{ steps.filter.outputs.product-tests }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
with:
cache: restore
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
product-tests:
- 'testing/trino-product-tests*/**'
- 'testing/trino-testing-services/**'
# run all tests when there are any changes in the trino-server Maven module
# because it doesn't define it's Trino dependencies and
# it relies on the Provisio plugin to find the right artifacts
- 'core/trino-server/**'
- '.github/**'
- name: Maven Install
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
$MAVEN clean install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server-rpm'
- name: Map impacted plugins to features
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
# build a list of impacted modules, ignoring modules that cannot affect either product tests or Trino
$MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -pl '!:trino-docs,!:trino-tests,!:trino-faulttolerant-tests'
# GIB doesn't run on master, so make sure the file always exist
touch gib-impacted.log
testing/trino-plugin-reader/target/trino-plugin-reader-*-executable.jar -i gib-impacted.log -p core/trino-server/target/trino-server-*-hardlinks/plugin > impacted-features.log
echo "Impacted plugin features:"
cat impacted-features.log
- name: Product tests artifact
uses: actions/upload-artifact@v3
with:
name: product tests and server tarball
path: |
core/trino-server/target/*.tar.gz
impacted-features.log
testing/trino-product-tests-launcher/target/*.jar
testing/trino-product-tests/target/*-executable.jar
client/trino-cli/target/*-executable.jar
retention-days: 1
- id: prepare-matrix-template
run: |
cat <<EOF > .github/test-pt-matrix.yaml
config:
- default
- hdp3
# TODO: config-apache-hive3
suite:
- suite-1
- suite-2
- suite-3
# suite-4 does not exist
- suite-5
- suite-azure
- suite-delta-lake-databricks91
- suite-delta-lake-databricks104
- suite-delta-lake-databricks113
- suite-delta-lake-databricks122
- suite-delta-lake-databricks133
- suite-gcs
- suite-clients
- suite-functions
- suite-tpch
- suite-storage-formats-detailed
exclude:
- config: default
ignore exclusion if: >-
${{ github.event_name != 'pull_request'
|| github.event.pull_request.head.repo.full_name == github.repository
|| contains(github.event.pull_request.labels.*.name, 'tests:all')
|| contains(github.event.pull_request.labels.*.name, 'tests:hive')
}}
- suite: suite-azure
config: default
- suite: suite-azure
ignore exclusion if: >-
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' ||
secrets.AZURE_ABFS_CONTAINER != '' ||
secrets.AZURE_ABFS_ACCOUNT != '' ||
secrets.AZURE_ABFS_ACCESSKEY != '' }}
- suite: suite-gcs
config: default
- suite: suite-gcs
ignore exclusion if: >-
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.GCP_CREDENTIALS_KEY != '' }}
- suite: suite-delta-lake-databricks91
config: hdp3
- suite: suite-delta-lake-databricks91
ignore exclusion if: >-
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
- suite: suite-delta-lake-databricks104
config: hdp3
- suite: suite-delta-lake-databricks104
ignore exclusion if: >-
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
- suite: suite-delta-lake-databricks113
config: hdp3
- suite: suite-delta-lake-databricks113
ignore exclusion if: >-
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
- suite: suite-delta-lake-databricks122
config: hdp3
- suite: suite-delta-lake-databricks122
ignore exclusion if: >-
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
- suite: suite-delta-lake-databricks133
config: hdp3
- suite: suite-delta-lake-databricks133
ignore exclusion if: >-
${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }}
ignore exclusion if:
# Do not use this property outside of the matrix configuration.
#
# This is added to all matrix entries so they may be conditionally
# excluded by adding them to the excludes list with a GHA expression
# for this property.
# - If the expression evaluates to true, it will never match the a
# actual value of the property, and will therefore not be excluded.
# - If the expression evaluates to false, it will match the actual
# value of the property, and the exclusion will apply normally.
- "false"
include:
# this suite is not meant to be run with different configs
- config: default
suite: suite-6-non-generic
# this suite is not meant to be run with different configs
- config: default
suite: suite-7-non-generic
# this suite is not meant to be run with different configs
- config: default
suite: suite-8-non-generic
# this suite is not meant to be run with different configs
- config: default
suite: suite-tpcds
# this suite is not meant to be run with different configs
- config: default
suite: suite-parquet
# this suite is not meant to be run with different configs
- config: default
suite: suite-oauth2
# this suite is not meant to be run with different configs
- config: default
suite: suite-ldap
# this suite is not meant to be run with different configs
- config: default
suite: suite-compatibility
# this suite is designed specifically for apache-hive3. TODO remove the suite once we can run all regular tests on apache-hive3.
- config: apache-hive3
suite: suite-hms-only
# this suite is not meant to be run with different configs
- config: default
suite: suite-all-connectors-smoke
# this suite is not meant to be run with different configs
- config: default
suite: suite-delta-lake-oss
# this suite is not meant to be run with different configs
- config: default
suite: suite-kafka
# this suite is not meant to be run with different configs
- config: default
suite: suite-cassandra
# this suite is not meant to be run with different configs
- config: default
suite: suite-clickhouse
# this suite is not meant to be run with different configs
- config: default
suite: suite-mysql
# this suite is not meant to be run with different configs
- config: default
suite: suite-iceberg
# this suite is not meant to be run with different configs
- config: default
suite: suite-hudi
# this suite is not meant to be run with different configs
- config: default
suite: suite-ignite
EOF
- name: Build PT matrix (all)
if: |
github.event_name != 'pull_request' ||
steps.filter.outputs.product-tests == 'true' ||
contains(github.event.pull_request.labels.*.name, 'tests:all') ||
contains(github.event.pull_request.labels.*.name, 'tests:all-product')
run: |
# converts entire YAML file into JSON - no filtering since we want all PTs to run
./.github/bin/build-pt-matrix-from-impacted-connectors.py -v -m .github/test-pt-matrix.yaml -o matrix.json
- name: Build PT matrix (impacted-features)
if: |
github.event_name == 'pull_request' &&
steps.filter.outputs.product-tests == 'false' &&
!contains(github.event.pull_request.labels.*.name, 'tests:all') &&
!contains(github.event.pull_request.labels.*.name, 'product-tests:all')
# all these envs are required to be set by some product test environments
env:
ABFS_CONTAINER:
ABFS_ACCOUNT:
ABFS_ACCESS_KEY:
S3_BUCKET:
AWS_REGION:
TRINO_AWS_ACCESS_KEY_ID:
TRINO_AWS_SECRET_ACCESS_KEY:
DATABRICKS_91_JDBC_URL:
DATABRICKS_104_JDBC_URL:
DATABRICKS_113_JDBC_URL:
DATABRICKS_122_JDBC_URL:
DATABRICKS_133_JDBC_URL:
DATABRICKS_LOGIN:
DATABRICKS_TOKEN:
GCP_CREDENTIALS_KEY:
GCP_STORAGE_BUCKET:
TESTCONTAINERS_NEVER_PULL: true
run: |
# converts filtered YAML file into JSON
./.github/bin/build-pt-matrix-from-impacted-connectors.py -v -m .github/test-pt-matrix.yaml -i impacted-features.log -o matrix.json
- id: set-matrix
run: |
echo "Matrix: $(jq '.' matrix.json)"
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
pt:
runs-on: ubuntu-latest
# explicitly define the name to avoid adding the value of the `ignore exclusion if` matrix item
name: pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }})
if: needs.build-pt.outputs.matrix != '{}'
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.build-pt.outputs.matrix) }}
# PT Launcher's timeout defaults to 2h, add some margin
timeout-minutes: 130
needs: build-pt
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent
ref: |
${{ github.event_name == 'repository_dispatch' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha &&
format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }}
- uses: ./.github/actions/setup
with:
cache: false
download_dependencies: false
- name: Product tests artifact
uses: actions/download-artifact@v3
with:
name: product tests and server tarball
- name: Fix artifact permissions
run: |
find . -type f -name \*-executable.jar -exec chmod 0777 {} \;
- name: Enable impact analysis
# don't enable this on pushes to master and in PRs in the main repository (not from forks)
# because these are most often used to run all tests with additional secrets
if: |
needs.build-pt.outputs.product-tests-changed == 'false' &&
github.event_name == 'pull_request' &&
github.event.pull_request.head.repo.full_name != github.repository &&
!contains(github.event.pull_request.labels.*.name, 'tests:all') &&
!contains(github.event.pull_request.labels.*.name, 'tests:all-product')
run: echo "PTL_OPTS=--impacted-features impacted-features.log" >> $GITHUB_ENV
- name: Product Tests
env:
ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }}
AWS_REGION: ${{ vars.TRINO_AWS_REGION }}
TRINO_AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
TRINO_AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
DATABRICKS_91_JDBC_URL: ${{ secrets.DATABRICKS_91_JDBC_URL }}
DATABRICKS_104_JDBC_URL: ${{ secrets.DATABRICKS_104_JDBC_URL }}
DATABRICKS_113_JDBC_URL: ${{ secrets.DATABRICKS_113_JDBC_URL }}
DATABRICKS_122_JDBC_URL: ${{ secrets.DATABRICKS_122_JDBC_URL }}
DATABRICKS_133_JDBC_URL: ${{ secrets.DATABRICKS_133_JDBC_URL }}
DATABRICKS_LOGIN: token
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }}
run: |
exec testing/trino-product-tests-launcher/target/trino-product-tests-launcher-*-executable.jar suite run \
--suite ${{ matrix.suite }} \
--config config-${{ matrix.config }} \
${PTL_OPTS:-} \
--bind=off --logs-dir logs/ --timeout 2h
- name: Upload test logs and results
uses: actions/upload-artifact@v3
# Upload all test reports only on failure, because the artifacts are large
if: failure()
with:
name: result pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }})
path: |
testing/trino-product-tests/target/*
logs/*
- name: Upload test report
uses: actions/upload-artifact@v3
# Always upload the test report for the annotate.yml workflow,
# but only the single XML file to keep the artifact small
if: always()
with:
# Name prefix is checked in the `Annotate checks` workflow
name: test report pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }})
path: testing/trino-product-tests/target/reports/**/testng-results.xml
retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
- name: Update PR check
uses: ./.github/actions/update-check
if: >-
failure() &&
github.event_name == 'repository_dispatch' &&
github.event.client_payload.slash_command.args.named.sha != '' &&
github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha
with:
pull_request_number: ${{ github.event.client_payload.pull_request.number }}
check_name: ${{ github.job }} with secrets
conclusion: ${{ job.status }}
github_token: ${{ secrets.GITHUB_TOKEN }}