ci.yml

name: ci

on:
  push:
    branches:
      - master
  pull_request:
    paths-ignore:
      - 'docs/**'

defaults:
  run:
    shell: bash --noprofile --norc -euo pipefail {0}

env:
  # An envar that signals to tests we are executing in the CI environment
  CONTINUOUS_INTEGRATION: true
  # allow overriding Maven command
  MAVEN: ./mvnw
  # maven.wagon.rto is in millis, defaults to 30m
  MAVEN_OPTS: "-Xmx512M -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000"
  MAVEN_INSTALL_OPTS: "-Xmx3G -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000"
  MAVEN_FAST_INSTALL: "-B --strict-checksums -V --quiet -T C1 -DskipTests -Dmaven.source.skip=true -Dair.check.skip-all"
  MAVEN_COMPILE_COMMITS: "-B --strict-checksums --quiet -T C1 -DskipTests -Dmaven.source.skip=true -Dair.check.skip-all=true -Dmaven.javadoc.skip=true --no-snapshot-updates --no-transfer-progress -pl '!:trino-server-rpm'"
  MAVEN_GIB: "-P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event.pull_request.base.ref }}"
  MAVEN_TEST: "-B --strict-checksums -Dmaven.source.skip=true -Dair.check.skip-all --fail-at-end -P gib -Dgib.referenceBranch=refs/remotes/origin/${{ github.event.pull_request.base.ref }}"
  RETRY: .github/bin/retry
  # Testcontainers kills image pulls if they don't make progress for > 30s and retries for 2m before failing. This means
  # that if an image doesn't download all it's layers within ~2m then any other concurrent pull will be killed because
  # the Docker daemon only downloads 3 layers concurrently which prevents the other pull from making any progress.
  # This value should be greater than the time taken for the longest image pull.
  TESTCONTAINERS_PULL_PAUSE_TIMEOUT: 600
  TEST_REPORT_RETENTION_DAYS: 5
  HEAP_DUMP_RETENTION_DAYS: 14
  # used by actions/cache to retry the download after this time: https://github.com/actions/cache/blob/main/workarounds.md#cache-segment-restore-timeout
  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5

# Cancel previous PR builds.
concurrency:
  # Cancel all workflow runs except latest within a concurrency group. This is achieved by defining a concurrency group for the PR.
  # Non-PR builds have singleton concurrency groups.
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.number || github.sha }}
  cancel-in-progress: true

jobs:
  maven-checks:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        java-version:
          - 17
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent
      - uses: ./.github/actions/setup
        with:
          java-version: ${{ matrix.java-version }}
      - name: Download all Trino dependencies to make sure cache is complete
        run: |
          $RETRY $MAVEN dependency:go-offline
      - name: Maven Checks
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $MAVEN clean install -B --strict-checksums -V -T C1 -DskipTests -P ci -pl '!:trino-server-rpm'
      - name: Test Server RPM
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $MAVEN verify -B --strict-checksums -P ci -pl :trino-server-rpm
      - name: Clean Maven Output
        run: $MAVEN clean -pl '!:trino-server,!:trino-cli'
      - uses: docker/setup-qemu-action@v2
        with:
          platforms: arm64,ppc64le
      - name: Test Docker Image
        run: core/docker/build.sh
      - name: Remove Trino from local Maven repo to avoid caching it
        # Avoid caching artifacts built in this job, cache should only include dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: rm -rf ~/.m2/repository/io/trino/trino-*

  check-commits:
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits to be able to determine merge base
          ref: ${{ github.event.pull_request.head.sha }} # checkout PR HEAD commit instead of merge commit
      - name: Check Commits
        uses: trinodb/github-actions/block-commits@c2991972560c5219d9ae5fb68c0c9d687ffcdd10
        with:
          action-merge: fail
          action-fixup: none
      - name: Set matrix
        id: set-matrix
        # The output from rev-list ends with a newline, so we have to filter out index -1 in jq since it's an empty string
        # The HEAD commit of the PR (index -2) can be safely ignored since it's already compiled in other jobs
        run: |
          export JQ_PIPELINE='split("\n") | .[0:-2] | map({base_ref: "${{ github.event.pull_request.base.ref }}", commit: .}) | select(length > 0) | {include: .}'
          git rev-list refs/remotes/origin/${{ github.event.pull_request.base.ref }}..HEAD | jq --raw-input --slurp "$JQ_PIPELINE" > commit-matrix.json
          echo "Commit matrix: $(jq '.' commit-matrix.json)"
          echo "matrix=$(jq -c '.' commit-matrix.json)" >> $GITHUB_OUTPUT

  check-commits-dispatcher:
    runs-on: ubuntu-latest
    needs: check-commits
    if: github.event_name == 'pull_request' && needs.check-commits.outputs.matrix != ''
    strategy:
      matrix: ${{ fromJson(needs.check-commits.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits to be able to determine merge base
          ref: ${{ matrix.commit }}
      # This composite job must be entirely standalone, and checked out from the correct commit before being executed.
      # It can't accept any parameters defined in this workflow, because the values of those parameters would always be taken from
      # PR HEAD since that is the commit the workflow is started for. This could lead to problems if those parameters were changed
      # in the middle of a PR branch.
      - uses: ./.github/actions/compile-commit
        with:
          base_ref: ${{ matrix.base_ref }}

  error-prone-checks:
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
      - uses: ./.github/actions/setup
      - name: Maven Package
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY $MAVEN clean package ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
      - name: Error Prone Checks
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          # Run Error Prone on one module with a retry to ensure all runtime dependencies are fetched
          $RETRY $MAVEN ${MAVEN_TEST} -T C1 clean verify -DskipTests -P gib,errorprone-compiler -am -pl ':trino-spi'
          # The main Error Prone run
          $MAVEN ${MAVEN_TEST} -T C1 clean verify -DskipTests -P gib,errorprone-compiler \
            -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
      - name: Clean local Maven repo
        # Avoid creating a cache entry because this job doesn't download all dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: rm -rf ~/.m2/repository

  web-ui-checks:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits: it's not needed here, but it's needed almost always, so let's do this for completeness
      - name: Web UI Checks
        run: core/trino-main/bin/check_webui.sh

  test-jdbc-compatibility:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    env:
      SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout tags so version in Manifest is set properly
      - uses: ./.github/actions/setup
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
      - name: Test old JDBC vs current server
        run: |
          if [ ! -f gib-impacted.log ] || grep -q testing/trino-test-jdbc-compatibility-old-driver gib-impacted.log; then
            testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh
          fi
      - name: Test current JDBC vs old server
        if: always()
        run: |
          if [ ! -f gib-impacted.log ] || grep -q testing/trino-test-jdbc-compatibility-old-server gib-impacted.log; then
            $MAVEN test ${MAVEN_TEST} -pl :trino-test-jdbc-compatibility-old-server
          fi
      - name: Upload test results
        uses: actions/upload-artifact@v3
        # Upload all test reports only on failure, because the artifacts are large
        if: failure()
        with:
          name: result ${{ github.job }}
          path: |
            **/target/surefire-reports
            **/target/checkstyle-*
      - name: Upload test report
        uses: actions/upload-artifact@v3
        # Always upload the test report for the annotate.yml workflow,
        # but only the single XML file to keep the artifact small
        if: always()
        with:
          # Name prefix is checked in the `Annotate checks` workflow
          name: test report ${{ github.job }}
          path: |
            **/surefire-reports/TEST-*.xml
          retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
      - name: Upload heap dump
        uses: actions/upload-artifact@v3
        if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
        with:
          name: heap dump ${{ github.job }}
          if-no-files-found: 'ignore'
          path: |
            **/*.hprof
          retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }}
      - name: Clean local Maven repo
        # Avoid creating a cache entry because this job doesn't download all dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: rm -rf ~/.m2/repository

  hive-tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        config:
          - config-hdp3
          # TODO: config-apache-hive3
    timeout-minutes: 60
    env:
      SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
      - uses: ./.github/actions/setup
      - name: Install Hive Module
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -am -pl :trino-hive-hadoop2
      - name: Run Hive Tests
        run: |
          source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
            plugin/trino-hive-hadoop2/bin/run_hive_tests.sh
      - name: Run Hive S3 Tests
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
          S3_BUCKET: "trino-ci-test"
          S3_BUCKET_ENDPOINT: "https://s3.us-east-2.amazonaws.com"
        run: |
          if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
            source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_s3_tests.sh
            if [ matrix.config == 'config-hdp3' ]; then
              # JsonSerde class needed for the S3 Select JSON tests is only available on hdp3.
              plugin/trino-hive-hadoop2/bin/run_hive_s3_select_json_tests.sh
            fi
          fi
      - name: Run Hive Glue Tests
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: us-east-2
        run: |
          if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then
            $MAVEN test ${MAVEN_TEST} -pl :trino-hive -P test-hive-glue
          fi
      - name: Run Hive Azure ABFS Access Key Tests
        if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
        env:
          ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
          ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
          ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
        run: |
          if [ "${ABFS_CONTAINER}" != "" ]; then
            source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_abfs_access_key_tests.sh
          fi
      - name: Run Hive Azure ABFS OAuth Tests
        if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
        env:
          ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
          ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
          ABFS_OAUTH_ENDPOINT: ${{ secrets.AZURE_ABFS_OAUTH_ENDPOINT }}
          ABFS_OAUTH_CLIENTID: ${{ secrets.AZURE_ABFS_OAUTH_CLIENTID }}
          ABFS_OAUTH_SECRET: ${{ secrets.AZURE_ABFS_OAUTH_SECRET }}
        run: |
          if [ -n "$ABFS_CONTAINER" ]; then
            source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_abfs_oauth_tests.sh
          fi
      - name: Run Hive Azure WASB Tests
        if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
        env:
          WASB_CONTAINER: ${{ secrets.AZURE_WASB_CONTAINER }}
          WASB_ACCOUNT: ${{ secrets.AZURE_WASB_ACCOUNT }}
          WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESSKEY }}
        run: |
          if [ "${WASB_CONTAINER}" != "" ]; then
            source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_wasb_tests.sh
          fi
      - name: Run Hive Azure ADL Tests
        if: matrix.config != 'config-empty' # Hive 1.x does not support Azure storage
        env:
          ADL_NAME: ${{ secrets.AZURE_ADL_NAME }}
          ADL_CLIENT_ID: ${{ secrets.AZURE_ADL_CLIENTID }}
          ADL_CREDENTIAL: ${{ secrets.AZURE_ADL_CREDENTIAL }}
          ADL_REFRESH_URL: ${{ secrets.AZURE_ADL_REFRESHURL }}
        run: |
          if [ "${ADL_NAME}" != "" ]; then
            source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
              plugin/trino-hive-hadoop2/bin/run_hive_adl_tests.sh
          fi
      - name: Run Hive Alluxio Tests
        run: |
          source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh &&
            plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh
      - name: Upload test results
        uses: actions/upload-artifact@v3
        # Upload all test reports only on failure, because the artifacts are large
        if: failure()
        with:
          name: result ${{ github.job }}
          path: |
            **/target/surefire-reports
            **/target/checkstyle-*
      - name: Upload test report
        uses: actions/upload-artifact@v3
        # Always upload the test report for the annotate.yml workflow,
        # but only the single XML file to keep the artifact small
        if: always()
        with:
          # Name prefix is checked in the `Annotate checks` workflow
          name: test report ${{ github.job }} (${{ matrix.config }})
          path: |
            **/surefire-reports/TEST-*.xml
          retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
      - name: Upload heap dump
        uses: actions/upload-artifact@v3
        if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
        with:
          name: heap dump ${{ github.job }} (${{ matrix.config }})
          if-no-files-found: 'ignore'
          path: |
            **/*.hprof
          retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }}
      - name: Clean local Maven repo
        # Avoid creating a cache entry because this job doesn't download all dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: rm -rf ~/.m2/repository

  test-other-modules:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
      - uses: ./.github/actions/setup
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -pl '!:trino-docs,!:trino-server,!:trino-server-rpm'
      - name: Maven Tests
        run: |
          $MAVEN test ${MAVEN_TEST} -pl '
            !:trino-accumulo,
            !:trino-bigquery,
            !:trino-cassandra,
            !:trino-clickhouse,
            !:trino-delta-lake,
            !:trino-docs,!:trino-server,!:trino-server-rpm,
            !:trino-druid,
            !:trino-elasticsearch,
            !:trino-faulttolerant-tests,
            !:trino-hive,
            !:trino-hudi,
            !:trino-iceberg,
            !:trino-jdbc,!:trino-base-jdbc,!:trino-thrift,!:trino-memory,
            !:trino-kafka,
            !:trino-kudu,
            !:trino-main,
            !:trino-mariadb,
            !:trino-mongodb,
            !:trino-mysql,
            !:trino-oracle,
            !:trino-phoenix5,
            !:trino-pinot,
            !:trino-postgresql,
            !:trino-raptor-legacy,
            !:trino-redis,
            !:trino-singlestore,
            !:trino-sqlserver,
            !:trino-test-jdbc-compatibility-old-server,
            !:trino-tests'
      - name: Upload test results
        uses: actions/upload-artifact@v3
        # Upload all test reports only on failure, because the artifacts are large
        if: failure()
        with:
          name: result ${{ github.job }}
          path: |
            **/target/surefire-reports
            **/target/checkstyle-*
      - name: Upload test report
        uses: actions/upload-artifact@v3
        # Always upload the test report for the annotate.yml workflow,
        # but only the single XML file to keep the artifact small
        if: always()
        with:
          # Name prefix is checked in the `Annotate checks` workflow
          name: test report ${{ github.job }}
          path: |
            **/surefire-reports/TEST-*.xml
          retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
      - name: Upload heap dump
        uses: actions/upload-artifact@v3
        if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
        with:
          name: heap dump ${{ github.job }}
          if-no-files-found: 'ignore'
          path: |
            **/*.hprof
          retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }}
      - name: Clean local Maven repo
        # Avoid creating a cache entry because this job doesn't download all dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: rm -rf ~/.m2/repository

  build-test-matrix:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
      - uses: ./.github/actions/setup
        with:
          cache: false
      - name: Maven validate
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY $MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -P disable-check-spi-dependencies -pl '!:trino-docs'
      - name: Set matrix
        id: set-matrix
        run: |
          # GIB doesn't run on master, so make sure the file always exist
          touch gib-impacted.log
          cat <<EOF > .github/test-matrix.yaml
          include:
            - { modules: [ client/trino-jdbc, plugin/trino-base-jdbc, plugin/trino-thrift, plugin/trino-memory ] }
            - { modules: core/trino-main }
            - { modules: plugin/trino-accumulo }
            - { modules: plugin/trino-bigquery }
            - { modules: plugin/trino-cassandra }
            - { modules: plugin/trino-clickhouse }
            - { modules: plugin/trino-delta-lake }
            - { modules: plugin/trino-delta-lake, profile: cloud-tests }
            - { modules: plugin/trino-delta-lake, profile: gcs-tests }
            - { modules: plugin/trino-druid }
            - { modules: plugin/trino-elasticsearch }
            - { modules: plugin/trino-hive }
            - { modules: plugin/trino-hive, profile: test-parquet }
            - { modules: plugin/trino-hudi }
            - { modules: plugin/trino-iceberg }
            - { modules: plugin/trino-iceberg, profile: cloud-tests }
            - { modules: plugin/trino-kafka }
            - { modules: plugin/trino-kudu }
            - { modules: plugin/trino-mariadb }
            - { modules: plugin/trino-mongodb }
            - { modules: plugin/trino-mysql }
            - { modules: plugin/trino-oracle }
            - { modules: plugin/trino-phoenix5 }
            - { modules: plugin/trino-pinot }
            - { modules: plugin/trino-postgresql }
            - { modules: plugin/trino-raptor-legacy }
            - { modules: plugin/trino-redis }
            - { modules: plugin/trino-singlestore }
            - { modules: plugin/trino-sqlserver }
            - { modules: testing/trino-faulttolerant-tests, profile: default }
            - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-delta }
            - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-hive-1 }
            - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-hive-2 }
            - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-iceberg }
            - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-postgresql }
            - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-mongodb }
            - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-mysql }
            - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-sqlserver }
            - { modules: testing/trino-tests }
          EOF
          ./.github/bin/build-matrix-from-impacted.py -v -i gib-impacted.log -m .github/test-matrix.yaml -o matrix.json
          echo "Matrix: $(jq '.' matrix.json)"
          echo "matrix=$(jq -c '.' matrix.json)" >> $GITHUB_OUTPUT

  test:
    runs-on: ubuntu-latest
    needs: build-test-matrix
    if: needs.build-test-matrix.outputs.matrix != '{}'
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.build-test-matrix.outputs.matrix) }}
    timeout-minutes: 60
    env:
      SECRETS_PRESENT: ${{ secrets.SECRETS_PRESENT }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
      - uses: ./.github/actions/setup
      - name: Cleanup node
        # This is required as a virtual environment update 20210219.1 left too little space for MemSQL to work
        if: matrix.modules == 'plugin/trino-singlestore'
        run: .github/bin/cleanup-node.sh
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -am -pl "${{ matrix.modules }}"
      - name: Maven Tests
        if: >-
          matrix.modules != 'plugin/trino-singlestore' &&
          !((contains(matrix.modules, 'trino-delta-lake') && (contains(matrix.profile, 'cloud-tests') || contains(matrix.profile, 'gcs-tests'))) || (contains(matrix.modules, 'trino-iceberg') && contains(matrix.profile, 'cloud-tests')))
        run: $MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ matrix.profile != '' && format('-P {0}', matrix.profile) || '' }}
      # Additional tests for selected modules
      - name: Cloud Delta Lake Tests
      # Cloud tests are separate because they are time intensive, requiring cross-cloud network communication
        env:
          ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
          ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
          ABFS_ACCESSKEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESSKEY }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRETKEY }}
          AWS_REGION: us-east-2
        # Run tests if any of the secrets is present. Do not skip tests when one secret renamed, or secret name has a typo.
        if: >-
          contains(matrix.modules, 'trino-delta-lake') && contains(matrix.profile, 'cloud-tests') &&
          (env.ABFS_ACCOUNT != '' || env.ABFS_CONTAINER != '' || env.ABFS_ACCESSKEY != '' || env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '')
        run: |
          $MAVEN test ${MAVEN_TEST} ${{ format('-P {0}', matrix.profile) }} -pl :trino-delta-lake \
            -Dhive.hadoop2.azure-abfs-container="${ABFS_CONTAINER}" \
            -Dhive.hadoop2.azure-abfs-account="${ABFS_ACCOUNT}" \
            -Dhive.hadoop2.azure-abfs-access-key="${ABFS_ACCESSKEY}"
      - name: GCS Delta Lake Tests
      # Cloud tests are separate because they are time intensive, requiring cross-cloud network communication
        env:
          GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
        # Run tests if any of the secrets is present. Do not skip tests when one secret renamed, or secret name has a typo.
        if: >-
          contains(matrix.modules, 'trino-delta-lake') && contains(matrix.profile, 'gcs-tests') && env.GCP_CREDENTIALS_KEY != ''
        run: |
          $MAVEN test ${MAVEN_TEST} -P gcs-tests -pl :trino-delta-lake \
            -Dtesting.gcp-storage-bucket="trino-ci-test" \
            -Dtesting.gcp-credentials-key="${GCP_CREDENTIALS_KEY}"
      - name: Memsql Tests
        env:
          MEMSQL_LICENSE: ${{ secrets.MEMSQL_LICENSE }}
        if: matrix.modules == 'plugin/trino-singlestore' && env.MEMSQL_LICENSE != ''
        run: |
          $MAVEN test ${MAVEN_TEST} -pl :trino-singlestore -Dmemsql.license=${MEMSQL_LICENSE}
      - name: Cloud BigQuery Tests
        env:
          BIGQUERY_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CREDENTIALS_KEY }}
        if: matrix.modules == 'plugin/trino-bigquery' && env.BIGQUERY_CREDENTIALS_KEY != ''
        run: |
          $MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests \
            -Dbigquery.credentials-key="${BIGQUERY_CREDENTIALS_KEY}" \
            -Dtesting.gcp-storage-bucket="trino-ci-test" \
            -Dtesting.alternate-bq-project-id=bigquery-cicd-alternate
      - name: Cloud BigQuery Case Insensitive Mapping Tests
        env:
          BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY: ${{ secrets.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY }}
        if: matrix.modules == 'plugin/trino-bigquery' && env.BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY != ''
        run: |
          $MAVEN test ${MAVEN_TEST} -pl :trino-bigquery -Pcloud-tests-case-insensitive-mapping -Dbigquery.credentials-key="${BIGQUERY_CASE_INSENSITIVE_CREDENTIALS_KEY}"
      - name: Iceberg Cloud Tests
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: us-east-2
          S3_BUCKET: trino-ci-test
          GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
        if: >-
          contains(matrix.modules, 'trino-iceberg') && contains(matrix.profile, 'cloud-tests') &&
          (env.AWS_ACCESS_KEY_ID != '' || env.AWS_SECRET_ACCESS_KEY != '' || env.GCP_CREDENTIALS_KEY != '')
        run: |
          $MAVEN test ${MAVEN_TEST} -pl :trino-iceberg ${{ format('-P {0}', matrix.profile) }} \
            -Ds3.bucket=${S3_BUCKET} \
            -Dtesting.gcp-storage-bucket="trino-ci-test-us-east" \
            -Dtesting.gcp-credentials-key="${GCP_CREDENTIALS_KEY}"
      - name: Sanitize artifact name
        if: always()
        run: |
          # Generate a valid artifact name and make it available to next steps as
          # an environment variable ARTIFACT_NAME
          # ", :, <, >, |, *, ?, \, / are not allowed in artifact names, replace it with an underscore
          name=$(echo -n "${{ matrix.modules }}" | sed -e 's/[":<>|\*\?\\\/]/_/g')
          echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV
      - name: Upload test results
        uses: actions/upload-artifact@v3
        # Upload all test reports only on failure, because the artifacts are large
        if: failure()
        with:
          name: result ${{ env.ARTIFACT_NAME }}
          path: |
            **/target/surefire-reports
            **/target/checkstyle-*
      - name: Upload test report
        uses: actions/upload-artifact@v3
        # Always upload the test report for the annotate.yml workflow,
        # but only the single XML file to keep the artifact small
        if: always()
        with:
          # Name prefix is checked in the `Annotate checks` workflow
          name: test report ${{ github.job }} (${{ env.ARTIFACT_NAME }})
          path: |
            **/surefire-reports/TEST-*.xml
          retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}
      - name: Upload heap dump
        uses: actions/upload-artifact@v3
        if: failure() && env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
        with:
          name: heap dump ${{ github.job }} (${{ env.ARTIFACT_NAME }})
          if-no-files-found: 'ignore'
          path: |
            **/*.hprof
          retention-days: ${{ env.HEAP_DUMP_RETENTION_DAYS }}
      - name: Clean local Maven repo
        # Avoid creating a cache entry because this job doesn't download all dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: rm -rf ~/.m2/repository

  build-pt:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      product-tests-changed: ${{ steps.filter.outputs.product-tests }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits to be able to determine merge base for GIB
      - uses: ./.github/actions/setup
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            product-tests:
              - 'testing/trino-product-tests*/**'
              - 'testing/trino-testing-services/**'
              # run all tests when there are any changes in the trino-server Maven module
              # because it doesn't define it's Trino dependencies and
              # it relies on the Provisio plugin to find the right artifacts
              - 'core/trino-server/**'
              - '.github/**'
      - name: Maven Install
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $RETRY $MAVEN clean install ${MAVEN_FAST_INSTALL} -pl '!:trino-docs,!:trino-server-rpm'
      - name: Map impacted plugins to features
        run: |
          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
          $MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -pl '!:trino-docs'
          # GIB doesn't run on master, so make sure the file always exist
          touch gib-impacted.log
          testing/trino-plugin-reader/target/trino-plugin-reader-*-executable.jar -i gib-impacted.log -p core/trino-server/target/trino-server-*-hardlinks/plugin > impacted-features.log
          echo "Impacted plugin features:"
          cat impacted-features.log
      - name: Product tests artifact
        uses: actions/upload-artifact@v3
        with:
          name: product tests and server tarball
          path: |
            core/trino-server/target/*.tar.gz
            impacted-features.log
            testing/trino-product-tests-launcher/target/*.jar
            testing/trino-product-tests/target/*-executable.jar
            client/trino-cli/target/*-executable.jar
          retention-days: 1
      - id: prepare-matrix-template
        run: |
          cat <<EOF > .github/test-pt-matrix.yaml
          config:
            - default
            - hdp3
            # TODO: config-apache-hive3
          suite:
            - suite-1
            - suite-2
            - suite-3
            # suite-4 does not exist
            - suite-5
            - suite-azure
            - suite-delta-lake-databricks73
            - suite-delta-lake-databricks91
            - suite-delta-lake-databricks104
            - suite-delta-lake-databricks113
            - suite-gcs
            - suite-clients
            - suite-functions
            - suite-tpch
            - suite-storage-formats-detailed
          exclude:
            - config: default
              ignore exclusion if: >-
                ${{ github.event_name != 'pull_request'
                 || github.event.pull_request.head.repo.full_name == github.repository
                 || contains(github.event.pull_request.labels.*.name, 'tests:all')
                 || contains(github.event.pull_request.labels.*.name, 'tests:hive')
                 }}

            - suite: suite-azure
              config: default
            - suite: suite-azure
              ignore exclusion if: >-
                ${{ secrets.AZURE_ABFS_CONTAINER != '' &&
                    secrets.AZURE_ABFS_ACCOUNT != '' &&
                    secrets.AZURE_ABFS_ACCESSKEY != '' }}

            - suite: suite-gcs
              config: default
            - suite: suite-gcs
              ignore exclusion if: >-
                ${{ secrets.GCP_CREDENTIALS_KEY != '' }}

            - suite: suite-delta-lake-databricks73
              config: hdp3
            - suite: suite-delta-lake-databricks73
              ignore exclusion if: >-
                ${{ secrets.DATABRICKS_TOKEN != '' }}
            - suite: suite-delta-lake-databricks91
              config: hdp3
            - suite: suite-delta-lake-databricks91
              ignore exclusion if: >-
                ${{ secrets.DATABRICKS_TOKEN != '' }}
            - suite: suite-delta-lake-databricks104
              config: hdp3
            - suite: suite-delta-lake-databricks104
              ignore exclusion if: >-
                ${{ secrets.DATABRICKS_TOKEN != '' }}
            - suite: suite-delta-lake-databricks113
              config: hdp3
            - suite: suite-delta-lake-databricks113
              ignore exclusion if: >-
                ${{ secrets.DATABRICKS_TOKEN != '' }}

          ignore exclusion if:
            # Do not use this property outside of the matrix configuration.
            #
            # This is added to all matrix entries so they may be conditionally
            # excluded by adding them to the excludes list with a GHA expression
            # for this property.
            # - If the expression evaluates to true, it will never match the a
            #   actual value of the property, and will therefore not be excluded.
            # - If the expression evaluates to false, it will match the actual
            #   value of the property, and the exclusion will apply normally.
            - "false"
          include:
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-6-non-generic
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-7-non-generic
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-8-non-generic
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-tpcds
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-parquet
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-oauth2
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-ldap
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-compatibility
            # this suite is designed specifically for apache-hive3. TODO remove the suite once we can run all regular tests on apache-hive3.
            - config: apache-hive3
              suite: suite-hms-only
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-all
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-delta-lake-oss
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-kafka
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-cassandra
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-clickhouse
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-mysql
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-iceberg
            # this suite is not meant to be run with different configs
            - config: default
              suite: suite-hudi
          EOF
      - name: Build PT matrix (all)
        if: |
          github.event_name != 'pull_request' ||
          steps.filter.outputs.product-tests == 'true' ||
          contains(github.event.pull_request.labels.*.name, 'tests:all') ||
          contains(github.event.pull_request.labels.*.name, 'tests:all-product')
        run: |
          # converts entire YAML file into JSON - no filtering since we want all PTs to run
          ./.github/bin/build-pt-matrix-from-impacted-connectors.py -v -m .github/test-pt-matrix.yaml -o matrix.json
      - name: Build PT matrix (impacted-features)
        if: |
          github.event_name == 'pull_request' &&
          steps.filter.outputs.product-tests == 'false' &&
          !contains(github.event.pull_request.labels.*.name, 'tests:all') &&
          !contains(github.event.pull_request.labels.*.name, 'product-tests:all')
        # all these envs are required to be set by some product test environments
        env:
          ABFS_CONTAINER:
          ABFS_ACCOUNT:
          ABFS_ACCESS_KEY:
          S3_BUCKET:
          AWS_REGION:
          TRINO_AWS_ACCESS_KEY_ID:
          TRINO_AWS_SECRET_ACCESS_KEY:
          DATABRICKS_73_JDBC_URL:
          DATABRICKS_91_JDBC_URL:
          DATABRICKS_104_JDBC_URL:
          DATABRICKS_113_JDBC_URL:
          DATABRICKS_LOGIN:
          DATABRICKS_TOKEN:
          GCP_CREDENTIALS_KEY:
          GCP_STORAGE_BUCKET:
        run: |
          # converts filtered YAML file into JSON
          ./.github/bin/build-pt-matrix-from-impacted-connectors.py -v -m .github/test-pt-matrix.yaml -i impacted-features.log -o matrix.json
      - id: set-matrix
        run: |
          echo "Matrix: $(jq '.' matrix.json)"
          echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
      - name: Clean local Maven repo
        # Avoid creating a cache entry because this job doesn't download all dependencies
        if: steps.cache.outputs.cache-hit != 'true'
        run: rm -rf ~/.m2/repository

  pt:
    runs-on: ubuntu-latest
    # explicitly define the name to avoid adding the value of the `ignore exclusion if` matrix item
    name: pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }})
    if: needs.build-pt.outputs.matrix != '{}'
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.build-pt.outputs.matrix) }}
    # PT Launcher's timeout defaults to 2h, add some margin
    timeout-minutes: 130
    needs: build-pt
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent
      - uses: ./.github/actions/setup
        with:
          cache: false
      - name: Product tests artifact
        uses: actions/download-artifact@v3
        with:
          name: product tests and server tarball
      - name: Fix artifact permissions
        run: |
          find . -type f -name \*-executable.jar -exec chmod 0777 {} \;
      - name: Enable impact analysis
        # don't enable this on pushes to master and in PRs in the main repository (not from forks)
        # because these are most often used to run all tests with additional secrets
        if: |
          needs.build-pt.outputs.product-tests-changed == 'false' &&
          github.event_name == 'pull_request' &&
          github.event.pull_request.head.repo.full_name != github.repository &&
          !contains(github.event.pull_request.labels.*.name, 'tests:all') &&
          !contains(github.event.pull_request.labels.*.name, 'tests:all-product')
        run: echo "PTL_OPTS=--impacted-features impacted-features.log" >> $GITHUB_ENV
      - name: Product Tests
        env:
          ABFS_CONTAINER: ${{ secrets.AZURE_ABFS_CONTAINER }}
          ABFS_ACCOUNT: ${{ secrets.AZURE_ABFS_ACCOUNT }}
          ABFS_ACCESS_KEY: ${{ secrets.AZURE_ABFS_ACCESSKEY }}
          S3_BUCKET: trino-ci-test
          AWS_REGION: us-east-2
          TRINO_AWS_ACCESS_KEY_ID: ${{ secrets.TRINO_AWS_ACCESS_KEY_ID }}
          TRINO_AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }}
          DATABRICKS_73_JDBC_URL: ${{ secrets.DATABRICKS_73_JDBC_URL }}
          DATABRICKS_91_JDBC_URL: ${{ secrets.DATABRICKS_91_JDBC_URL }}
          DATABRICKS_104_JDBC_URL: ${{ secrets.DATABRICKS_104_JDBC_URL }}
          DATABRICKS_113_JDBC_URL: ${{ secrets.DATABRICKS_113_JDBC_URL }}
          DATABRICKS_LOGIN: token
          DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
          GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }}
          GCP_STORAGE_BUCKET: trino-ci-test-us-east
        run: |
          exec testing/trino-product-tests-launcher/target/trino-product-tests-launcher-*-executable.jar suite run \
            --suite ${{ matrix.suite }} \
            --config config-${{ matrix.config }} \
            ${PTL_OPTS:-} \
            --bind=off --logs-dir logs/ --timeout 2h
      - name: Upload test logs and results
        uses: actions/upload-artifact@v3
        # Upload all test reports only on failure, because the artifacts are large
        if: failure()
        with:
          name: result pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }})
          path: |
            testing/trino-product-tests/target/*
            logs/*
      - name: Upload test report
        uses: actions/upload-artifact@v3
        # Always upload the test report for the annotate.yml workflow,
        # but only the single XML file to keep the artifact small
        if: always()
        with:
          # Name prefix is checked in the `Annotate checks` workflow
          name: test report pt (${{ matrix.config }}, ${{ matrix.suite }}, ${{ matrix.jdk }})
          path: testing/trino-product-tests/target/reports/**/testng-results.xml
          retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }}