Test-AITs.yml

name: Test - AITs

on:
  workflow_dispatch:
    inputs:
      agent-ref:
        description: "Specify agent branch/tag/sha (main is default)"
        required: false
        default: 'main'
      ait-ref:
        description: "Specify AIT branch/tag/sha (main is default)"
        required: false
        default: 'main'
      cache-ref:
        description: "Specify cache branch/tag/sha (main is default)"
        required: false
        default: 'main'
      single-test:
        description: "Specify a single test. If left blank, all tests will run. You can choose to specify a single test file or a test case in that file. For example you can write 'server/tomcat.py' to run all tomcat tests or run 'server/tomcat.py TomcatTest.test_tomcat' to run a specific test case."
        required: false
        default: ''
  workflow_call:
    inputs:
      agent-ref:
        description: "Specify agent branch/tag/sha (main is default)"
        required: false
        default: 'main'
        type: string
      ait-ref:
        description: "Specify AIT branch/tag/sha (main is default)"
        required: false
        default: 'main'
        type: string
      cache-ref:
        description: "Specify cache branch/tag/sha (main is default)"
        required: false
        default: 'main'
        type: string

jobs:
  build-agent:
    uses: ./.github/workflows/X-Reusable-BuildAgent.yml
    with:
      # inputs.agent-ref is for workflow dispatch/call, github.ref for PR or push, main if all else fail
      agent-ref: ${{ inputs.agent-ref || github.ref || 'main' }}
    secrets: inherit

  list-tests:
    name: List tests
    runs-on: ubuntu-20.04
    outputs:
      tests: ${{ steps.read-single-test.outputs.tests || steps.read-tests.outputs.tests }}
    steps:
      - name: Checkout AIT repo test
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4
        with:
          repository: newrelic/java-agent-integration-tests
          ref: ${{ inputs.ait-ref || 'main' }}
          token: ${{ secrets.AITPAT }}
          path: agent-integration-tests

      - id: read-tests
        name: List instrumentation tests
        if: ${{inputs.single-test == ''}}
        run: |
          excluded_tests=$(mktemp /tmp/excluded_tests.XXXXXXXX)
          echo "datastore/datastores.py" >> $excluded_tests
          echo "framework/jms/jms.py" >> $excluded_tests
          echo "r2dbc/mssql.py" >> $excluded_tests
          echo "server/mule.py" >> $excluded_tests
          echo "server/weblogic.py" >> $excluded_tests
          echo "basic_features/inf_tracing_test.py" >> $excluded_tests
          echo "security/lasp_msg_params.py" >> $excluded_tests
          echo "framework/pekko/pekko.py" >> $excluded_tests
          echo "framework/pekko/pekkohttp.py" >> $excluded_tests
          # The files below are not tests
          echo "trace/client.py" >> $excluded_tests
          echo "trace/server.py" >> $excluded_tests
          echo "trace/trace_base.py" >> $excluded_tests
          echo "trace/tracecontext/__init__.py" >> $excluded_tests
          echo "trace/tracecontext/traceparent.py" >> $excluded_tests
          echo "trace/tracecontext/tracestate.py" >> $excluded_tests
          cd agent-integration-tests/tests/java/functionality
          tmpfile=$(mktemp /tmp/dirs.XXXXXXXXXX)
          # list the tests
          find . -iname "*.py" | cut -d'/' -f 2- | grep -v -x -f $excluded_tests | sort > $tmpfile
          # checking if there is at least one test, xargs trims the output
          TEST_COUNT=$(cat $tmpfile | wc -l | xargs)
          if [[ "$TEST_COUNT" == "0" ]];
          then
            echo ":x: Failure: no test was found. There is probably something wrong with the script." >>$GITHUB_STEP_SUMMARY
            exit 1
          fi
          # converting the simple test list to a JSON array
          FILES=$(cat $tmpfile | jq -R -s -c 'split("\n")[:-1]')
          # creates an envar with the ait files in a JSON format
          TESTS=$((
            echo '{ "tests" : '
            echo $FILES
            echo " }"
          ) | jq -c .)
          # save the output of the job
          echo "tests=$TESTS" >> $GITHUB_OUTPUT

      - id: read-single-test
        name: Read single test
        if: ${{inputs.single-test != ''}}
        run: |
          # creates an envar with a single test in the same JSON format as the read-tests step above
          TESTS=$((
            echo '{ "tests" : '
            echo ' ["${{ inputs.single-test }}"] ' 
            echo " }"
          ) | jq -c .)
          # save the output of the job
          echo "tests=$TESTS" >> $GITHUB_OUTPUT

  tests:
    name: ${{ matrix.tests }}
    needs: [ build-agent, list-tests ]
    timeout-minutes: 120
    runs-on: ubuntu-20.04
    # Determine if easier to make the env strings below part of the matrix
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.list-tests.outputs.tests) }}
    steps:
      - name: Retrieve agent from cache
        id: retrieve-agent
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # pin@v4
        with:
          path: /home/runner/work/newrelic-java-agent/newrelic-java-agent/newrelic-agent/build/newrelicJar/newrelic.jar
          key: agent-jar-${{ github.run_id }}
          fail-on-cache-miss: true

      ## Ongoing tests with artifactory dependencies
      - name: Checkout AIT repo test
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4
        with:
          repository: newrelic/java-agent-integration-tests
          ref: ${{ inputs.ait-ref }}
          token: ${{ secrets.AITPAT }}
          path: agent-integration-tests

      # Apps repos/caches - this process could be a candidate for custom action
      - name: Checkout Cache 1
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4
        with:
          repository: newrelic/java-ait-cache-1
          ref: ${{ inputs.cache-ref || 'main' }}
          token: ${{ secrets.AITPAT }}
          path: appcache1
          lfs: true

      - name: Checkout Cache 2
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4
        with:
          repository: newrelic/java-ait-cache-2
          ref: ${{ inputs.cache-ref || 'main' }}
          token: ${{ secrets.AITPAT }}
          path: appcache2
          lfs: true

      - name: Checkout Cache 3
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4
        with:
          repository: newrelic/java-ait-cache-3
          ref: ${{ inputs.cache-ref || 'main' }}
          token: ${{ secrets.AITPAT }}
          path: appcache3
          lfs: true

      - name: Checkout Cache 4
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4
        with:
          repository: newrelic/java-ait-cache-4
          ref: ${{ inputs.cache-ref || 'main' }}
          token: ${{ secrets.AITPAT }}
          path: appcache4
          lfs: true

      - name: Checkout Cache 5
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4
        with:
          repository: newrelic/java-ait-cache-5
          ref: ${{ inputs.cache-ref || 'main' }}
          token: ${{ secrets.AITPAT }}
          path: appcache5
          lfs: true

      # Consolidate caches into one directory
      - name: Consolidate caches into Apps directory
        run: |
          mkdir apps
          mv appcache1/* apps/
          mv appcache2/* apps/
          mv appcache3/* apps/
          mv appcache4/* apps/

      # looks like docker is not properly sanitized between runs in GHA
      # so its disk space may be pilling up and blows up during our tests
      - name: Clean Docker image data
        run: docker rmi $(docker images -q)

      ## JDK Installs

      # Install Zulu
      - name: Set up Zulu versions
        uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # pin@v4
        with:
          distribution: 'zulu'
          java-version: |
            23
            21
            17
            11

      # Set the JDK variables for Zulu
      - name: Set the JDK variables for Zulu
        run: |
          echo "JDK_zulu_11=${JAVA_HOME_11_X64}" >> $GITHUB_ENV
          echo "JDK_zulu_17=${JAVA_HOME_17_X64}" >> $GITHUB_ENV
          echo "JDK_zulu_21=${JAVA_HOME_21_X64}" >> $GITHUB_ENV
          echo "JDK_zulu_23=${JAVA_HOME_23_X64}" >> $GITHUB_ENV

      - name: Set up Javas
        uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # pin@v4
        with:
          distribution: 'temurin'
          java-version: |
            23
            21
            17
            11
            8

      - name: Set up Toolchain
        shell: bash
        run: |
          mkdir -p $HOME/.m2 \
          && cat << EOF > $HOME/.m2/toolchains.xml
          <?xml version="1.0" encoding="UTF8"?>
          <!-- toolchains.xml is used by the pom in the Java11_test_webapp.  -->
          <toolchains>
            <toolchain>
            <type>jdk</type>
            <provides>
                    <version>8</version>
            </provides>
            <configuration>
                    <jdkHome>${JAVA_HOME_8_X64}</jdkHome>
            </configuration>
            </toolchain>
            <toolchain>
            <type>jdk</type>
            <provides>
                    <version>11</version>
            </provides>
            <configuration>
                    <jdkHome>${JAVA_HOME_11_X64}</jdkHome>
            </configuration>
            </toolchain>
            <toolchain>
            <type>jdk</type>
            <provides>
                    <version>17</version>
            </provides>
            <configuration>
                    <jdkHome>${JAVA_HOME_17_X64}</jdkHome>
            </configuration>
            </toolchain>
            <toolchain>
            <type>jdk</type>
            <provides>
                    <version>21</version>
            </provides>
            <configuration>
                    <jdkHome>${JAVA_HOME_21_X64}</jdkHome>
            </configuration>
            </toolchain>
          <toolchain>
            <type>jdk</type>
            <provides>
                    <version>23</version>
            </provides>
            <configuration>
                    <jdkHome>${JAVA_HOME_23_X64}</jdkHome>
            </configuration>
            </toolchain>
          </toolchains>
          EOF

      ## End JDK Install

      ## TESTING SECTION

      # Replication of steps from ait README

      - name: CD to agent-integration-tests dir.
        run: cd agent-integration-tests/

      - name: Set Test Name Env Var
        run: |
          TEST_NAME=$(echo ${{ matrix.tests }} | sed 's|/|-|g')
          echo "TEST_NAME="${TEST_NAME}"" >> $GITHUB_ENV

      ## WE LOSE THE VIRTUAL ENVIRONMENT ONCE WE LEAVE THE STEP
      ## TODO: This should really be a custom action, too many commands
      - name: Create virtualenv and run ${{ matrix.tests }}
        if: ${{ failure() || success() }}
        run: |
          cd agent-integration-tests
          echo "conf/testenv complains of the path below - creating symlink for now"
          ln -s ${GITHUB_WORKSPACE}/apps /home/runner/apps
          ln -s ${GITHUB_WORKSPACE}/newrelic-agent/build/newrelicJar/newrelic.jar ${GITHUB_WORKSPACE}/newrelic.jar
          echo "still complains of file not found"
          sudo apt-get install virtualenv
          virtualenv -p /usr/bin/python3.8 .
          . bin/activate
          bin/pip3 install -r conf/requirements.txt
          ZULU11=${JDK_zulu_11} \
          ZULU17=${JDK_zulu_17} \
          ZULU21=${JDK_zulu_21} \
          ZULU23=${JDK_zulu_23} \
          JAVA8JRE=${JAVA_HOME_8_X64} \
          JAVA11JRE=${JAVA_HOME_11_X64} \
          JAVA17JRE=${JAVA_HOME_17_X64} \
          JAVA21JRE=${JAVA_HOME_21_X64} \
          JAVA23JRE=${JAVA_HOME_23_X64} \
          conf/autoconfigure
          . conf/testenv java
          cat conf/java_local_config.yml
          sed -i 's|java_agent_dev_root: /home/runner/work/newrelic-java-agent/newrelic-java-agent|java_agent_dev_root: /home/runner/work/newrelic-java-agent/newrelic-java-agent/newrelic-agent/build/newrelicJar|' conf/java_local_config.yml
          sed -i 's|app_root: /home/runner/apps|app_root: /home/runner/work/newrelic-java-agent/newrelic-java-agent/apps|' conf/java_local_config.yml
          ## artifacts section for testing
          mkdir testing-artifacts
          touch testing-artifacts/shell-variables.txt
          touch testing-artifacts/env-variables.txt
          set | sort -f > testing-artifacts/shell-variables.txt
          printenv | sort -f > testing-artifacts/env-variables.txt
          cp conf/java_local_config.yml testing-artifacts/
          ## End testing artifacts section
          TEST_LOG_LEVEL=DEBUG TEST_SUPPRESS_METRIC_DEBUG=1 \
          PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python \
          ./bin/runtest.sh tests/java/functionality/${{ matrix.tests }}

      # Rename matrix item to remove problem characters
      - name: Rename Matrix item
        run: |
          MATRIX_ITEM=$(echo ${{ matrix.tests }} | sed 's|/|-|g')
          echo "MATRIX="${MATRIX_ITEM}"" >> $GITHUB_ENV

      - name: Capture testing artifacts
        if: ${{ failure() || success() }}
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # pin@v4
        with:
          name: ${{ github.workflow }}-${{ github.job }}-${{ env.MATRIX }}
          path: |
            agent-integration-tests/testing-artifacts/*

      - name: Upload negative value artifact
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # pin@v4
        with:
          path: ${{ github.workspace }}/negative_value_*.txt
          if-no-files-found: ignore

  notify-if-negative-values:
    needs: [ tests ]
    runs-on: ubuntu-20.04
    steps:
      - name: Download artifact(s) # surely there is a way to only download the negative_value_*.txt artifacts?
        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # pin@v4
        env:
          continue-on-error: true
      - name: Send failure message to Slack
        if: ${{ hashFiles('**/negative_value_*.txt') != '' }}
        id: slack
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # pin@v1.25.0
        with:
          payload: |
            {
              "message": "Negative value detected in AIT run.  Check the artifacts named 'negative_value_*.txt'.  This will tell you which test had the negative value, and the contents will list the events.  And, of course, more details can be found in the logs.",
              "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.NEGATIVE_VALUE_SLACK_WEBHOOK_URL }}