Merge branch 'master' into tests/blackbox

.github/workflows/ci-tests.yml

@@ -26,42 +26,100 @@ env:
   PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
 
 jobs:
+  golangci-lint:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
+    steps:
+      - name: "Checkout PR"
+        uses: TykTechnologies/github-actions/.github/actions/checkout-pr@main
+        with:
+          token: ${{ secrets.ORG_GH_TOKEN }}
+
+      - name: "Get base ref"
+        run: |
+          git fetch origin ${{ github.base_ref }}
+          git rev-parse origin/${{ github.base_ref }}
+
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/golangci-lint
+          key: 'golangci-lint-${{ runner.os }}-${{ hashFiles(''**/go.sum'') }}'
+          restore-keys: |
+            golangci-lint-${{ runner.os }}-
+
+      - name: Setup CI Tooling
+        uses: shrink/actions-docker-extract@v3
+        with:
+          image: tykio/ci-tools:latest
+          path: /usr/local/bin/golangci-lint
+          destination: /usr/local/bin
+
+      - run: golangci-lint version && golangci-lint cache status
+
+      - name: golangci-lint
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          golangci-lint run --out-format colored-line-number,checkstyle:golangci-lint-report.json --issues-exit-code=0 --new-from-rev=origin/${{ github.base_ref }} -v ./...
+
+      - name: golangci-lint-on-push
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          golangci-lint run --out-format checkstyle:golangci-lint-report.json --issues-exit-code=0 -v ./...
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: golangcilint
+          retention-days: 1
+          path: |
+            golangci-lint-report.json
+
   test:
     name: Go ${{ matrix.go-version }} Redis ${{ matrix.redis-version }}
-    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
+    needs: golangci-lint
+    # Runs on is pinned to a version that provides python 3.10.
+    # See: https://github.com/actions/runner-images?tab=readme-ov-file#available-images
+    # Avoid using ubuntu-latest as it would upgrade python unattended.
+    runs-on: ubuntu-22.04
     strategy:
-      fail-fast: true
-      # This workflow isn't designed to be run as a pipeline, several issues:
-      #
-      # - contains golangci-lint jobs, sonarcloud (would duplicate)
-      # - cache config not suitable for multiple pipelines
-      # - python tests should be separate job, or no job
-      #
-      # Keep it to a single job run from the matrix as configured
-      # until we get a chance to redesign the pipeline properly.
+      fail-fast: false
       matrix:
         redis-version: [7]
-        python-version: ["3.11"]
         go-version: [1.22.x]
 
     env:
       REDIS_IMAGE: redis:${{ matrix.redis-version }}
 
     steps:
+      - name: "Reclaim some runner space"
+        run: sudo rm -rf /usr/local/bin/* /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
       - name: Checkout Tyk
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.ref }}
 
-      - name: Setup Golang
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
-
+      # Regardless that the base image provides a python release, we need
+      # setup-python so it properly configures the python3-venv.
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Print runtime python version
+        run: python3 -c 'import sys; print("%d.%d" % (sys.version_info[0], sys.version_info[1]))'
+
+      - name: Print runtime pip version
+        run: pip -V && pip3 -V
 
       - name: Setup CI Tooling
         uses: shrink/actions-docker-extract@v3
@@ -70,7 +128,13 @@ jobs:
           path: /usr/local/bin/.
           destination: /usr/local/bin
 
-      - name: Cache
+      - name: Setup Golang
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Restore Golang Cache
         uses: actions/cache@v4
         with:
           path: |
@@ -109,33 +173,63 @@ jobs:
             exit 1
           fi
 
-      - name: Fetch base branch
-        if: ${{ github.event_name == 'pull_request' }}
-        run: git fetch origin ${{ github.base_ref }}
-
-      - name: Print CPU info
-        run: grep '^model name' /proc/cpuinfo
+      - name: Bring up test services
+        run: task services:up
 
-      - name: Print Go env
-        run: go env
+      - name: Preflight Python tests
+        if: runner.debug == '1'
+        run: TYK_LOGLEVEL=debug go test -p 1 -parallel 1 -race -v ./dlpython ./coprocess/...
 
       - name: Run Gateway Tests
         id: ci-tests
         run: |
           task test:e2e-combined args="-race -timeout=15m"
           task test:coverage
 
-      # golangci-lint actions *require* issues-exit-code=0 to pass data along to sonarcloud
-      # rather than erroring out on github issues directly with out-format github.
-      - name: golangci-lint
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          golangci-lint run --out-format checkstyle --issues-exit-code=0 --new-from-rev=origin/${{ github.base_ref }} ./... > golanglint.xml
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: coverage
+          retention-days: 1
+          path: coverage/gateway-all.cov
 
-      - name: golangci-lint-on-push
-        if: ${{ github.event_name == 'push' }}
-        run: |
-          golangci-lint run --out-format checkstyle --issues-exit-code=0 ./... > golanglint.xml
+      - uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: testjson
+          retention-days: 1
+          path: coverage/gateway-all.json
+
+  sonar-cloud-analysis:
+    runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
+    needs: [test, golangci-lint]
+    steps:
+      - name: "Checkout repository"
+        uses: TykTechnologies/github-actions/.github/actions/checkout-pr@main
+        with:
+          token: ${{ secrets.ORG_GH_TOKEN }}
+
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage
+
+      - name: Download golangcilint artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: golangcilint
+
+      - name: Check reports existence
+        id: check_files
+        uses: andstor/file-existence-action@v1
+        with:
+          files: 'coverage/gateway-all.cov, golangci-lint-report.json'
+
+      - name: Install Dependencies
+        env:
+          TOKEN: '${{ secrets.ORG_GH_TOKEN }}'
+        run: git config --global url."https://${TOKEN}@github.com".insteadOf "https://github.com"
 
       - name: SonarCloud Scan
         if: always()
@@ -145,12 +239,12 @@ jobs:
             -Dsonar.organization=tyktechnologies
             -Dsonar.projectKey=TykTechnologies_tyk
             -Dsonar.sources=.
-            -Dsonar.exclusions=**/testdata/*,test/**,coprocess/**/*,ci/**,smoke-tests/**,apidef/oas/schema/schema.gen.go
+            -Dsonar.exclusions=**/testdata/*,test/**,coprocess/**/*,ci/**,smoke-tests/**,apidef/oas/schema/schema.gen.go,templates/**
             -Dsonar.coverage.exclusions=**/*_test.go,**/mock/*
             -Dsonar.test.inclusions=**/*_test.go
             -Dsonar.tests=.
             -Dsonar.go.coverage.reportPaths=coverage/gateway-all.cov
-            -Dsonar.go.golangci-lint.reportPaths=golanglint.xml
+            -Dsonar.go.golangci-lint.reportPaths=golangci-lint-report.json
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -18,6 +18,7 @@ concurrency:
 jobs:
   analyze:
     name: Analyze
+    if: ${{ !github.event.pull_request.draft }}
     runs-on: ubuntu-latest
 
     strategy:
@@ -31,7 +32,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         # We must fetch at least the immediate parents so that if this is
         # a pull request then we can checkout the head.
@@ -43,7 +44,7 @@ jobs:
       if: ${{ github.event_name == 'pull_request' }}
 
     - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v5
       with:
         go-version-file: go.mod
 

.github/workflows/plugin-compiler-build.yml

@@ -8,7 +8,7 @@ on:
       - master
       - release-**
     tags:
-      - 'v*'
+      - "v*"
 
 env:
   GOLANG_CROSS: 1.22-bullseye
@@ -20,11 +20,17 @@ concurrency:
 jobs:
   docker-build:
     runs-on: ubuntu-latest
+    if: ${{ !github.event.pull_request.draft }}
     permissions:
       id-token: write
     steps:
+      - name: "Reclaim some runner space"
+        run: sudo rm -rf /usr/local/bin/* /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
 
       - name: Configure AWS Credentials
         id: configure-aws
@@ -74,3 +80,35 @@ jobs:
             BASE-IMAGE=tykio/golang-cross:${{ env.GOLANG_CROSS }}
             GITHUB_SHA=${{ github.sha }}
             GITHUB_TAG=${{ github.ref_name }}
+
+      - name: Set docker metadata EE
+        id: set-metadata-ee
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            tykio/tyk-plugin-compiler-ee,enable=${{ startsWith(github.ref, 'refs/tags') }}
+            ${{ steps.login-ecr.outputs.registry }}/tyk-plugin-compiler-ee
+          labels: |
+            org.opencontainers.image.title=tyk-plugin-compiler-ee
+            org.opencontainers.image.description=Plugin compiler for the Tyk API Gateway Enterprise Edition
+          tags: |
+            type=ref,event=pr
+            type=semver,pattern=v{{version}}
+            type=semver,pattern=v{{major}}.{{minor}}
+            type=semver,pattern={{raw}}
+            type=sha,format=long
+
+      - name: Build and push to dockerhub/ECR EE
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ci/images/plugin-compiler/Dockerfile
+          platforms: linux/amd64
+          push: true
+          labels: ${{ steps.set-metadata-ee.outputs.labels }}
+          tags: ${{ steps.set-metadata-ee.outputs.tags }}
+          build-args: |
+            BASE-IMAGE=tykio/golang-cross:${{ env.GOLANG_CROSS }}
+            GITHUB_SHA=${{ github.sha }}
+            GITHUB_TAG=${{ github.ref_name }}
+            BUILD_TAG=ee