Merge branch 'master' into nirebu/avoid-string-dup

.github/workflows/lib-injection.yml

@@ -51,7 +51,7 @@ jobs:
     steps:
       - name: lib-injection test runner
         id: lib-injection-test-runner
-        uses: DataDog/system-tests/lib-injection/runner@4eb1ec01b61c1c7e84f219127a91d07f03b0f85e
+        uses: DataDog/system-tests/lib-injection/runner@495d99b7a5d62ae8e2b7de39ec16334cbeb16523
         with:
           docker-registry: ghcr.io
           docker-registry-username: ${{ github.repository_owner }}

.github/workflows/system-tests.yml

@@ -12,76 +12,292 @@ env:
   REGISTRY: ghcr.io
 
 jobs:
-  system-tests:
+  build-harness:
     strategy:
       fail-fast: false
       matrix:
+        image:
+          - runner
+          - agent
+    runs-on: ubuntu-latest
+    name: Build (${{ matrix.image }})
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Pull released image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest system_tests/${{ matrix.image }}:latest
+      - name: Build image
+        run: ./build.sh --images ${{ matrix.image }}
+      - name: List images
+        run: |
+          docker image list
+      - name: Log in to the Container registry
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
+      - name: Tag image for CI run
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Push image for CI run
+        run: |
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Tag image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+      - name: Push image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run: |
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest
+
+  build-apps:
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - weblog
+        library:
+          - ruby
+        app:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
+    runs-on: ubuntu-latest
+    name: Build (${{ matrix.app }})
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Checkout dd-trace-rb
+        uses: actions/checkout@v3
+        with:
+          path: 'binaries/dd-trace-rb'
+      - name: Pull released image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest
+      - name: Log in to the Container registry
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
+      - name: Build
+        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }}
+      - name: Tag image for CI run
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Push image for CI run
+        run: |
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
+      - name: Tag image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run:
+          docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest
+      - name: Push image for release
+        if: ${{ github.ref == 'refs/heads/master' }}
+        run: |
+          docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest
+
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        library:
+          - ruby
+        app:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
+        scenario:
+          - DEFAULT
+          - APPSEC_DISABLED
+          - APPSEC_IP_BLOCKING
+          - APPSEC_REQUEST_BLOCKING
         include:
           - library: ruby
-            weblog-variant: rack
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
           - library: ruby
-            weblog-variant: sinatra14
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
           - library: ruby
-            weblog-variant: sinatra20
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
           - library: ruby
-            weblog-variant: sinatra21
+            app: rack
+            scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
           - library: ruby
-            weblog-variant: rails32
+            app: rack
+            scenario: APPSEC_CUSTOM_RULES
           - library: ruby
-            weblog-variant: rails40
+            app: rack
+            scenario: APPSEC_MISSING_RULES
           - library: ruby
-            weblog-variant: rails41
+            app: rack
+            scenario: APPSEC_CORRUPTED_RULES
           - library: ruby
-            weblog-variant: rails42
+            app: rack
+            scenario: APPSEC_LOW_WAF_TIMEOUT
           - library: ruby
-            weblog-variant: rails50
+            app: rack
+            scenario: APPSEC_CUSTOM_OBFUSCATION
           - library: ruby
-            weblog-variant: rails51
+            app: rack
+            scenario: APPSEC_RATE_LIMITER
           - library: ruby
-            weblog-variant: rails52
+            app: rack
+            scenario: SAMPLING
           - library: ruby
-            weblog-variant: rails60
-          - library: ruby
-            weblog-variant: rails61
-          - library: ruby
-            weblog-variant: rails70
+            app: rack
+            scenario: PROFILING
     runs-on: ubuntu-latest
-    name: System Tests (${{ matrix.weblog-variant }})
+    needs:
+      - build-harness
+      - build-apps
+    name: Test (${{ matrix.app }}, ${{ matrix.scenario }})
     steps:
+      - name: Setup python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
       - name: Checkout
         uses: actions/checkout@v3
         with:
           repository: 'DataDog/system-tests'
-
-      - name: Checkout dd-trace-rb
-        uses: actions/checkout@v3
-        with:
-          path: 'binaries/dd-trace-rb'
-
-      - name: Log in to the Container registry
+      - name: Pull agent image
         run: |
-          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
-
-      - name: Build
-        run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.weblog-variant }}
-
-      - name: Run default scenario
-        run: ./run.sh
-        env:
-          DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-
-      - name: Run APPSEC_CUSTOM_RULES scenario
-        run: ./run.sh APPSEC_CUSTOM_RULES
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest
+      - name: Pull runner image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} system_tests/runner:latest
+      - name: Pull app image
+        run: |
+          docker pull ghcr.io/datadog/dd-trace-rb/system-tests/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
+          docker tag ghcr.io/datadog/dd-trace-rb/system-tests/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest
+      - name: List images
+        run: |
+          docker image list
+      - name: Run scenario
+        run: ./run.sh ${{ matrix.scenario }}
         env:
           DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
-
-      - name: Archive logs
+      - name: Archive logs (per scenario)
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
-          name: system-tests-${{ matrix.library }}-${{ matrix.weblog-variant }}-logs-${{ github.run_id }}-${{ github.sha }}
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
+          path: logs*
+      - name: Archive logs (aggregated)
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
           path: logs*
 
+  aggregate:
+    strategy:
+      fail-fast: false
+      matrix:
+        library:
+          - ruby
+        app:
+          - rack
+          - sinatra14
+          - sinatra20
+          - sinatra21
+          - rails32
+          - rails40
+          - rails41
+          - rails42
+          - rails50
+          - rails51
+          - rails52
+          - rails60
+          - rails61
+          - rails70
+    runs-on: ubuntu-latest
+    needs:
+      - test
+    if: ${{ always() }}
+    name: Aggregate (${{ matrix.app }})
+    steps:
+      - name: Setup python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+      - name: Retrieve logs
+        uses: actions/download-artifact@v3
+        with:
+          name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
+          path: .
       - name: Print fancy log report
-        if: ${{ always() }}
-        run: python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY
+        run: |
+          find logs*
+          python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY
+
+  cleanup:
+    strategy:
+      fail-fast: false
+      matrix:
+        image:
+          - runner
+          - agent
+          - weblog-rack
+          - weblog-sinatra14
+          - weblog-sinatra20
+          - weblog-sinatra21
+          - weblog-rails32
+          - weblog-rails40
+          - weblog-rails41
+          - weblog-rails42
+          - weblog-rails50
+          - weblog-rails51
+          - weblog-rails52
+          - weblog-rails60
+          - weblog-rails61
+          - weblog-rails70
+    runs-on: ubuntu-latest
+    needs:
+      - test
+    if: ${{ always() }}
+    name: Cleanup (${{ matrix.image }})
+    steps:
+      - name: Log in to the Container registry
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
+      - uses: actions/delete-package-versions@v4
+        with:
+          package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}'
+          package-name: 'system-tests/${{ matrix.image }}'
+          package-type: 'container'
+        continue-on-error: true

.github/workflows/test-macos.yaml

@@ -8,7 +8,6 @@ jobs:
         os:
           - macos-latest
         ruby:
-          - '2.1'
           - '2.2'
           - '2.3'
           - '2.4'

.github/workflows/yard.yml

@@ -0,0 +1,46 @@
+name: Deploy Yard documentation to Pages
+
+on:
+  push:
+    branches: ["release"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2'
+          bundler-cache: true
+      - name: Generate YARD documentation
+        run: bundle exec rake docs
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v1
+        with:
+          # Upload generated YARD directory
+          path: 'doc/'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2