Test - AITs #296
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test - AITs | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| agent-ref: | |
| description: "Specify agent branch/tag/sha (main is default)" | |
| required: false | |
| default: 'main' | |
| ait-ref: | |
| description: "Specify AIT branch/tag/sha (main is default)" | |
| required: false | |
| default: 'main' | |
| cache-ref: | |
| description: "Specify cache branch/tag/sha (main is default)" | |
| required: false | |
| default: 'main' | |
| single-test: | |
| description: "Specify a single test. If left blank, all tests will run. You can choose to specify a single test file or a test case in that file. For example you can write 'server/tomcat.py' to run all tomcat tests or run 'server/tomcat.py TomcatTest.test_tomcat' to run a specific test case." | |
| required: false | |
| default: '' | |
| workflow_call: | |
| inputs: | |
| agent-ref: | |
| description: "Specify agent branch/tag/sha (main is default)" | |
| required: false | |
| default: 'main' | |
| type: string | |
| ait-ref: | |
| description: "Specify AIT branch/tag/sha (main is default)" | |
| required: false | |
| default: 'main' | |
| type: string | |
| cache-ref: | |
| description: "Specify cache branch/tag/sha (main is default)" | |
| required: false | |
| default: 'main' | |
| type: string | |
| jobs: | |
| build-agent: | |
| uses: ./.github/workflows/X-Reusable-BuildAgent.yml | |
| with: | |
| # inputs.agent-ref is for workflow dispatch/call, github.ref for PR or push, main if all else fail | |
| agent-ref: ${{ inputs.agent-ref || github.ref || 'main' }} | |
| secrets: inherit | |
| list-tests: | |
| name: List tests | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| tests: ${{ steps.read-single-test.outputs.tests || steps.read-tests.outputs.tests }} | |
| steps: | |
| - name: Checkout AIT repo test | |
| uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4 | |
| with: | |
| repository: newrelic/java-agent-integration-tests | |
| ref: ${{ inputs.ait-ref || 'main' }} | |
| token: ${{ secrets.AITPAT }} | |
| path: agent-integration-tests | |
| - id: read-tests | |
| name: List instrumentation tests | |
| if: ${{inputs.single-test == ''}} | |
| run: | | |
| excluded_tests=$(mktemp /tmp/excluded_tests.XXXXXXXX) | |
| echo "datastore/datastores.py" >> $excluded_tests | |
| echo "framework/jms/jms.py" >> $excluded_tests | |
| echo "r2dbc/mssql.py" >> $excluded_tests | |
| echo "server/mule.py" >> $excluded_tests | |
| echo "server/weblogic.py" >> $excluded_tests | |
| echo "basic_features/inf_tracing_test.py" >> $excluded_tests | |
| echo "security/lasp_msg_params.py" >> $excluded_tests | |
| echo "framework/pekko/pekko.py" >> $excluded_tests | |
| echo "framework/pekko/pekkohttp.py" >> $excluded_tests | |
| # The files below are not tests | |
| echo "trace/client.py" >> $excluded_tests | |
| echo "trace/server.py" >> $excluded_tests | |
| echo "trace/trace_base.py" >> $excluded_tests | |
| echo "trace/tracecontext/__init__.py" >> $excluded_tests | |
| echo "trace/tracecontext/traceparent.py" >> $excluded_tests | |
| echo "trace/tracecontext/tracestate.py" >> $excluded_tests | |
| cd agent-integration-tests/tests/java/functionality | |
| tmpfile=$(mktemp /tmp/dirs.XXXXXXXXXX) | |
| # list the tests | |
| find . -iname "*.py" | cut -d'/' -f 2- | grep -v -x -f $excluded_tests | sort > $tmpfile | |
| # checking if there is at least one test, xargs trims the output | |
| TEST_COUNT=$(cat $tmpfile | wc -l | xargs) | |
| if [[ "$TEST_COUNT" == "0" ]]; | |
| then | |
| echo ":x: Failure: no test was found. There is probably something wrong with the script." >>$GITHUB_STEP_SUMMARY | |
| exit 1 | |
| fi | |
| # converting the simple test list to a JSON array | |
| FILES=$(cat $tmpfile | jq -R -s -c 'split("\n")[:-1]') | |
| # creates an envar with the ait files in a JSON format | |
| TESTS=$(( | |
| echo '{ "tests" : ' | |
| echo $FILES | |
| echo " }" | |
| ) | jq -c .) | |
| # save the output of the job | |
| echo "tests=$TESTS" >> $GITHUB_OUTPUT | |
| - id: read-single-test | |
| name: Read single test | |
| if: ${{inputs.single-test != ''}} | |
| run: | | |
| # creates an envar with a single test in the same JSON format as the read-tests step above | |
| TESTS=$(( | |
| echo '{ "tests" : ' | |
| echo ' ["${{ inputs.single-test }}"] ' | |
| echo " }" | |
| ) | jq -c .) | |
| # save the output of the job | |
| echo "tests=$TESTS" >> $GITHUB_OUTPUT | |
| tests: | |
| name: ${{ matrix.tests }} | |
| needs: [ build-agent, list-tests ] | |
| timeout-minutes: 120 | |
| runs-on: ubuntu-20.04 | |
| # Determine if easier to make the env strings below part of the matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.list-tests.outputs.tests) }} | |
| steps: | |
| - name: Retrieve agent from cache | |
| id: retrieve-agent | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # pin@v4 | |
| with: | |
| path: /home/runner/work/newrelic-java-agent/newrelic-java-agent/newrelic-agent/build/newrelicJar/newrelic.jar | |
| key: agent-jar-${{ github.run_id }} | |
| fail-on-cache-miss: true | |
| ## Ongoing tests with artifactory dependencies | |
| - name: Checkout AIT repo test | |
| uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4 | |
| with: | |
| repository: newrelic/java-agent-integration-tests | |
| ref: ${{ inputs.ait-ref }} | |
| token: ${{ secrets.AITPAT }} | |
| path: agent-integration-tests | |
| # Apps repos/caches - this process could be a candidate for custom action | |
| - name: Checkout Cache 1 | |
| uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4 | |
| with: | |
| repository: newrelic/java-ait-cache-1 | |
| ref: ${{ inputs.cache-ref || 'main' }} | |
| token: ${{ secrets.AITPAT }} | |
| path: appcache1 | |
| lfs: true | |
| - name: Checkout Cache 2 | |
| uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4 | |
| with: | |
| repository: newrelic/java-ait-cache-2 | |
| ref: ${{ inputs.cache-ref || 'main' }} | |
| token: ${{ secrets.AITPAT }} | |
| path: appcache2 | |
| lfs: true | |
| - name: Checkout Cache 3 | |
| uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4 | |
| with: | |
| repository: newrelic/java-ait-cache-3 | |
| ref: ${{ inputs.cache-ref || 'main' }} | |
| token: ${{ secrets.AITPAT }} | |
| path: appcache3 | |
| lfs: true | |
| - name: Checkout Cache 4 | |
| uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # pin@v4 | |
| with: | |
| repository: newrelic/java-ait-cache-4 | |
| ref: ${{ inputs.cache-ref || 'main' }} | |
| token: ${{ secrets.AITPAT }} | |
| path: appcache4 | |
| lfs: true | |
| # Consolidate caches into one directory | |
| - name: Consolidate caches into Apps directory | |
| run: | | |
| mkdir apps | |
| mv appcache1/* apps/ | |
| mv appcache2/* apps/ | |
| mv appcache3/* apps/ | |
| mv appcache4/* apps/ | |
| # looks like docker is not properly sanitized between runs in GHA | |
| # so its disk space may be pilling up and blows up during our tests | |
| - name: Clean Docker image data | |
| run: docker rmi $(docker images -q) | |
| ## JDK Installs | |
| # Install Zulu | |
| - name: Set up Zulu versions | |
| uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # pin@v4 | |
| with: | |
| distribution: 'zulu' | |
| java-version: | | |
| 22 | |
| 21 | |
| 17 | |
| 11 | |
| # Set the JDK variables for Zulu | |
| - name: Set the JDK variables for Zulu | |
| run: | | |
| echo "JDK_zulu_11=${JAVA_HOME_11_X64}" >> $GITHUB_ENV | |
| echo "JDK_zulu_17=${JAVA_HOME_17_X64}" >> $GITHUB_ENV | |
| echo "JDK_zulu_21=${JAVA_HOME_21_X64}" >> $GITHUB_ENV | |
| echo "JDK_zulu_22=${JAVA_HOME_22_X64}" >> $GITHUB_ENV | |
| - name: Set up Javas | |
| uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9 # pin@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: | | |
| 22 | |
| 21 | |
| 17 | |
| 11 | |
| 8 | |
| - name: Set up Toolchain | |
| shell: bash | |
| run: | | |
| mkdir -p $HOME/.m2 \ | |
| && cat << EOF > $HOME/.m2/toolchains.xml | |
| <?xml version="1.0" encoding="UTF8"?> | |
| <!-- toolchains.xml is used by the pom in the Java11_test_webapp. --> | |
| <toolchains> | |
| <toolchain> | |
| <type>jdk</type> | |
| <provides> | |
| <version>8</version> | |
| </provides> | |
| <configuration> | |
| <jdkHome>${JAVA_HOME_8_X64}</jdkHome> | |
| </configuration> | |
| </toolchain> | |
| <toolchain> | |
| <type>jdk</type> | |
| <provides> | |
| <version>11</version> | |
| </provides> | |
| <configuration> | |
| <jdkHome>${JAVA_HOME_11_X64}</jdkHome> | |
| </configuration> | |
| </toolchain> | |
| <toolchain> | |
| <type>jdk</type> | |
| <provides> | |
| <version>17</version> | |
| </provides> | |
| <configuration> | |
| <jdkHome>${JAVA_HOME_17_X64}</jdkHome> | |
| </configuration> | |
| </toolchain> | |
| <toolchain> | |
| <type>jdk</type> | |
| <provides> | |
| <version>21</version> | |
| </provides> | |
| <configuration> | |
| <jdkHome>${JAVA_HOME_21_X64}</jdkHome> | |
| </configuration> | |
| </toolchain> | |
| <toolchain> | |
| <type>jdk</type> | |
| <provides> | |
| <version>22</version> | |
| </provides> | |
| <configuration> | |
| <jdkHome>${JAVA_HOME_22_X64}</jdkHome> | |
| </configuration> | |
| </toolchain> | |
| </toolchains> | |
| EOF | |
| ## End JDK Install | |
| ## TESTING SECTION | |
| # Replication of steps from ait README | |
| - name: CD to agent-integration-tests dir. | |
| run: cd agent-integration-tests/ | |
| - name: Set Test Name Env Var | |
| run: | | |
| TEST_NAME=$(echo ${{ matrix.tests }} | sed 's|/|-|g') | |
| echo "TEST_NAME="${TEST_NAME}"" >> $GITHUB_ENV | |
| ## WE LOSE THE VIRTUAL ENVIRONMENT ONCE WE LEAVE THE STEP | |
| ## TODO: This should really be a custom action, too many commands | |
| - name: Create virtualenv and run ${{ matrix.tests }} | |
| if: ${{ failure() || success() }} | |
| run: | | |
| cd agent-integration-tests | |
| echo "conf/testenv complains of the path below - creating symlink for now" | |
| ln -s ${GITHUB_WORKSPACE}/apps /home/runner/apps | |
| ln -s ${GITHUB_WORKSPACE}/newrelic-agent/build/newrelicJar/newrelic.jar ${GITHUB_WORKSPACE}/newrelic.jar | |
| echo "still complains of file not found" | |
| sudo apt-get install virtualenv | |
| virtualenv -p /usr/bin/python3.8 . | |
| . bin/activate | |
| bin/pip3 install -r conf/requirements.txt | |
| ZULU11=${JDK_zulu_11} \ | |
| ZULU17=${JDK_zulu_17} \ | |
| ZULU21=${JDK_zulu_21} \ | |
| ZULU22=${JDK_zulu_22} \ | |
| JAVA8JRE=${JAVA_HOME_8_X64} \ | |
| JAVA11JRE=${JAVA_HOME_11_X64} \ | |
| JAVA17JRE=${JAVA_HOME_17_X64} \ | |
| JAVA21JRE=${JAVA_HOME_21_X64} \ | |
| JAVA22JRE=${JAVA_HOME_22_X64} \ | |
| conf/autoconfigure | |
| . conf/testenv java | |
| cat conf/java_local_config.yml | |
| sed -i 's|java_agent_dev_root: /home/runner/work/newrelic-java-agent/newrelic-java-agent|java_agent_dev_root: /home/runner/work/newrelic-java-agent/newrelic-java-agent/newrelic-agent/build/newrelicJar|' conf/java_local_config.yml | |
| sed -i 's|app_root: /home/runner/apps|app_root: /home/runner/work/newrelic-java-agent/newrelic-java-agent/apps|' conf/java_local_config.yml | |
| ## artifacts section for testing | |
| mkdir testing-artifacts | |
| touch testing-artifacts/shell-variables.txt | |
| touch testing-artifacts/env-variables.txt | |
| set | sort -f > testing-artifacts/shell-variables.txt | |
| printenv | sort -f > testing-artifacts/env-variables.txt | |
| cp conf/java_local_config.yml testing-artifacts/ | |
| ## End testing artifacts section | |
| TEST_LOG_LEVEL=DEBUG TEST_SUPPRESS_METRIC_DEBUG=1 \ | |
| PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python \ | |
| ./bin/runtest.sh tests/java/functionality/${{ matrix.tests }} | |
| # Rename matrix item to remove problem characters | |
| - name: Rename Matrix item | |
| run: | | |
| MATRIX_ITEM=$(echo ${{ matrix.tests }} | sed 's|/|-|g') | |
| echo "MATRIX="${MATRIX_ITEM}"" >> $GITHUB_ENV | |
| - name: Capture testing artifacts | |
| if: ${{ failure() || success() }} | |
| uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # pin@v4 | |
| with: | |
| name: ${{ github.workflow }}-${{ github.job }}-${{ env.MATRIX }} | |
| path: | | |
| agent-integration-tests/testing-artifacts/* | |
| - name: Upload negative value artifact | |
| uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # pin@v4 | |
| with: | |
| path: ${{ github.workspace }}/negative_value_*.txt | |
| if-no-files-found: ignore | |
| notify-if-negative-values: | |
| needs: [ tests ] | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Download artifact(s) # surely there is a way to only download the negative_value_*.txt artifacts? | |
| uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # pin@v4 | |
| env: | |
| continue-on-error: true | |
| - name: Send failure message to Slack | |
| if: ${{ hashFiles('**/negative_value_*.txt') != '' }} | |
| id: slack | |
| uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # [email protected] | |
| with: | |
| payload: | | |
| { | |
| "message": "Negative value detected in AIT run. Check the artifacts named 'negative_value_*.txt'. This will tell you which test had the negative value, and the contents will list the events. And, of course, more details can be found in the logs.", | |
| "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.NEGATIVE_VALUE_SLACK_WEBHOOK_URL }} |