dev: run integration tests in ci (#658)

* initial go

* feat: sort of make it run?

* cheat: make ci temporarily shorter

* fix: install playwright correctly

* fix: pass secrets to CI

* try again

* list

* succesfully run dirtily

* feat: go back to running tests in the action runner itself, not in a container

* chore: remove logging of heap usage during build

* dev: remove build-base job: did not do anything

* dev: setup db before running migrations in e2e

* dev: cache build steps in ECR

* dev: try to improve cache?

* dev: use output: standalone

* fix: move sentry instrumentation to instrumentation.ts

* fix: build jobs correctly

* fix: also make evaluations output:standalone

* fix: properly set target

* fix: restore correct reference to integration-tests image

* fix: correctly copy the static and public files

* fix: AA

* fix: AA

* fix: why why why why why why

* dev: test to see if networking is just completely borked in gha

* fix: use the correct test command

* dev: test with live correct image

* dev: use test image again, but in a network this time

* dev: try to mimick core image even more

* dev: try to use docker internal network?

* dev: log out netstat

* dev: use correct image again

* dev: sleep?

* dev: add healthcheck to integration tests and wait until healthy to run tests

* dev: try out integration tests fr fr fr

* fix: remove test image

* dev: seed the db, duh

* dev: skip validation for seeding

* fix: make login test work better

* fix: make pub edit test work again

* dev: up test timeout to 20seconds in CI

* fix: restore full workflow to use newly built image

* chore: dont lint playwright results and cleanup a bit

* chore: clean up ci.yml

* fix: use server side redirect instead of client side for form creation

* dev: use correct SMTP port in CI

* fix: do not use env in playwright tests

* fix: CI can be boolean as well

* fix: cleanup gh action files

* fix: symbolic change to trigger ci

* fix: even more symbolic change

* chore: ??

* fix: fix workflow file

* chore: cleanup on_pr

* fix: remove invalid secrets key from workflow dispatch

* fix: do not use strange global type

* feat: enable other ci again, and add turbo cache

* fix: add hacky timeout to externalForm test

* fix: restore data-testId form text form elements

* dev: test to see if not caching the pnpm fetch is faster

* dev: move turbo cache to a subdir

* dev: setup jobs correctly

* dev: disable cache to see if that's just faster

* chore: update pnpm-lock

* fix: set CI for integration test image so that emails can be sent unsecurely

* dev: cache test run

* dev: run build first to make sure db has enough time to spin up

* dev: also cache preconstruct build?

* fix: restore p:build

* dev: clean up dockerfile

* dev: use 2 retries instead of 0

* dev: use more workers for playwright

* fix: get rid of default command

* dev: make tests 30sec timeout by default, and only run one worker to reduce flakiness

.github/workflows/awsdeploy.yml

@@ -31,11 +31,6 @@ on:
                 type: string
             image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8
                 type: string # use this to force a container version.
-        secrets:
-            AWS_ACCESS_KEY_ID:
-                required: true
-            AWS_SECRET_ACCESS_KEY:
-                required: true
 
 jobs:
     deploy-core:

.github/workflows/ci.yml

@@ -1,10 +1,17 @@
 name: "CI"
 on:
     workflow_call:
+        inputs:
+            image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8
+                type: string # use this to force a container version.
     workflow_dispatch:
 
 env:
     CI: true
+    AWS_REGION: us-east-1
+
+    ECR_REPOSITORY_PREFIX: pubpub-v7
+    CONTAINER_NAME: core
 
 jobs:
     ci:
@@ -16,20 +23,63 @@ jobs:
             - name: Checkout
               uses: actions/checkout@v4
 
-            - name: Build docker image
-              run: docker compose --profile test build
+            - name: Install Node.js
+              uses: actions/setup-node@v4
+              with:
+                  node-version: 20
+
+            - uses: pnpm/action-setup@v4
+              name: Install pnpm
+              with:
+                  run_install: false
+
+            - name: Get pnpm store directory
+              id: get-store-path
+              shell: bash
+              run: |
+                  echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
+
+            - name: Setup pnpm cache
+              uses: actions/cache@v4
+              with:
+                  path: ${{ steps.get-store-path.outputs.STORE_PATH }}
+                  key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+                  restore-keys: |
+                      ${{ runner.os }}-pnpm-store-
+
+            - name: Cache turbo
+              uses: actions/cache@v4
+              with:
+                  path: .turbo
+                  key: ${{ runner.os }}-turbo-${{ github.sha }}
+                  restore-keys: |
+                      ${{ runner.os }}-turbo-
+
+            - name: Start up DB
+              run: docker compose --profile test up -d
+
+            - name: Install dependencies
+              run: pnpm install --frozen-lockfile --prefer-offline
+
+            - name: p:build
+              run: pnpm p:build
 
             - name: Run migrations
-              run: docker compose run core pnpm --filter core migrate-test
+              run: pnpm --filter core migrate-test
+              env:
+                  DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+
+            - name: generate prisma
+              run: pnpm --filter core prisma generate
 
             - name: Run prettier
-              run: docker compose run core pnpm format
+              run: pnpm format
 
             - name: Run eslint
-              run: docker compose run core pnpm lint
+              run: pnpm lint
 
             - name: Run typecheck
-              run: docker compose run core pnpm type-check
+              run: pnpm type-check
 
             - name: Run vitest tests
-              run: docker compose run core
+              run: pnpm test-run

.github/workflows/deploy-template.yml

@@ -3,55 +3,50 @@
 name: aws ecs deploy template
 
 on:
-  workflow_call:
-    inputs:
-      service: # example: core
-        required: true
-        type: string
-      proper-name: # example: blake
-        required: true
-        type: string
-      environment: # example: staging
-        required: true
-        type: string
-      repo-name-override:
-        type: string
-      image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8
-        type: string      # use this to force a container version.
-    secrets:
-      AWS_ACCESS_KEY_ID:
-        required: true
-      AWS_SECRET_ACCESS_KEY:
-        required: true
-  workflow_dispatch:
-    inputs:
-      service: # example: core
-        required: true
-        type: string
-      proper-name: # example: blake
-        required: true
-        type: string
-      environment: # example: staging
-        required: true
-        type: string
-      repo-name-override:
-        type: string
-      image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8
-        type: string      # use this to force a container version.
-    secrets:
-      AWS_ACCESS_KEY_ID:
-        required: true
-      AWS_SECRET_ACCESS_KEY:
-        required: true
+    workflow_call:
+        inputs:
+            service: # example: core
+                required: true
+                type: string
+            proper-name: # example: blake
+                required: true
+                type: string
+            environment: # example: staging
+                required: true
+                type: string
+            repo-name-override:
+                type: string
+            image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8
+                type: string # use this to force a container version.
+        secrets:
+            AWS_ACCESS_KEY_ID:
+                required: true
+            AWS_SECRET_ACCESS_KEY:
+                required: true
+    workflow_dispatch:
+        inputs:
+            service: # example: core
+                required: true
+                type: string
+            proper-name: # example: blake
+                required: true
+                type: string
+            environment: # example: staging
+                required: true
+                type: string
+            repo-name-override:
+                type: string
+            image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8
+                type: string # use this to force a container version.
 
 env:
-  AWS_REGION: us-east-1
-  ECR_REPOSITORY_PREFIX: pubpub-v7
-  ECR_REPOSITORY_NAME_OVERRIDE: ${{ inputs.repo-name-override }}
-  ECS_SERVICE: ${{ inputs.proper-name }}-${{inputs.service}}
-  ECS_CLUSTER: ${{inputs.proper-name}}-ecs-cluster-${{inputs.environment}}
-  ECS_TASK_DEFINITION_TEMPLATE: ${{ inputs.proper-name }}-${{inputs.service}}
-  CONTAINER_NAME: ${{inputs.service}}
+    AWS_REGION: us-east-1
+    ECR_REPOSITORY_PREFIX: pubpub-v7
+    ECR_REPOSITORY_NAME_OVERRIDE: ${{ inputs.repo-name-override }}
+    ECS_SERVICE: ${{ inputs.proper-name }}-${{inputs.service}}
+    ECS_CLUSTER: ${{inputs.proper-name}}-ecs-cluster-${{inputs.environment}}
+    ECS_TASK_DEFINITION_TEMPLATE: ${{ inputs.proper-name }}-${{inputs.service}}
+    CONTAINER_NAME: ${{inputs.service}}
 
 jobs:
     deploy:

.github/workflows/e2e.yml

@@ -0,0 +1,119 @@
+on:
+    workflow_call:
+        inputs:
+            image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8
+                type: string # use this to force a container version.
+        secrets:
+            AWS_ACCESS_KEY_ID:
+                required: true
+            AWS_SECRET_ACCESS_KEY:
+                required: true
+
+env:
+    CI: true
+    AWS_REGION: us-east-1
+
+    ECR_REPOSITORY_PREFIX: pubpub-v7
+    CONTAINER_NAME: core
+
+jobs:
+    integration-tests:
+        name: Integration tests
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Install Node.js
+              uses: actions/setup-node@v4
+              with:
+                  node-version: 20
+
+            - uses: pnpm/action-setup@v4
+              name: Install pnpm
+              with:
+                  run_install: false
+
+            - name: Get pnpm store directory
+              id: get-store-path
+              shell: bash
+              run: |
+                  echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
+
+            - name: Setup pnpm cache
+              uses: actions/cache@v4
+              with:
+                  path: ${{ steps.get-store-path.outputs.STORE_PATH }}
+                  key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+                  restore-keys: |
+                      ${{ runner.os }}-pnpm-store-
+
+            - name: Install dependencies
+              run: pnpm install --frozen-lockfile --prefer-offline
+
+            - name: Start up DB
+              run: docker compose -f docker-compose.test.yml --profile test up -d
+
+            - name: p:build
+              run: pnpm p:build
+
+            - name: Run migrations
+              run: pnpm --filter core prisma migrate deploy
+              env:
+                  DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+
+            - name: generate prisma
+              run: pnpm --filter core prisma generate
+
+            - name: seed db
+              run: pnpm --filter core prisma db seed
+              env:
+                  SKIP_VALIDATION: true
+                  DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
+
+            - name: Configure AWS credentials
+              uses: aws-actions/configure-aws-credentials@v4
+              with:
+                  role-to-assume: ${{ vars.IAM_ROLE_TO_ASSUME }}
+                  aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+                  aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                  aws-region: ${{ env.AWS_REGION }}
+
+            - name: Get image tag based on SHA
+              id: gettag
+              # use shell substitution
+              run: echo "tag=$(git describe --always --abbrev=40 --dirty)" >> $GITHUB_OUTPUT
+
+            - name: Login to Amazon ECR
+              id: login-ecr
+              uses: aws-actions/amazon-ecr-login@v2
+
+            - name: Get image labels
+              id: label
+              env:
+                  ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+                  IMAGE_TAG: ${{ steps.gettag.outputs.tag }}
+              run: |
+                  echo "core_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-core}:$IMAGE_TAG" >> $GITHUB_OUTPUT
+                  echo "jobs_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-jobs}:$IMAGE_TAG" >> $GITHUB_OUTPUT
+                  echo "base_label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX:$IMAGE_TAG" >> $GITHUB_OUTPUT
+
+            - run: pnpx playwright install chromium --with-deps
+
+            - name: Start up core
+              run: docker compose -f docker-compose.test.yml --profile integration up -d
+              env:
+                  INTEGRATION_TESTS_IMAGE: ${{steps.label.outputs.core_label}}
+                  JOBS_IMAGE: ${{steps.label.outputs.jobs_label}}
+
+            - name: Log out Container ID for health check
+              id: log-container-id
+              run: echo "CONTAINER_ID=$(docker compose -f docker-compose.test.yml ps integration-tests -q)" >> $GITHUB_OUTPUT
+
+            - name: Wait until container is healthy
+              run: while [ "`docker inspect -f {{.State.Health.Status}} ${{steps.log-container-id.outputs.CONTAINER_ID}}`" != "healthy" ]; do     sleep .2; done
+
+            - name: Run integration tests
+              run: pnpm --filter core playwright:test
+              env:
+                  INTEGRATION_TEST_HOST: localhost

.github/workflows/ecrbuild-all.yml

@@ -23,13 +23,14 @@ jobs:
                   sha_short=$(git describe --always --abbrev=40 --dirty)
                   echo "Building containers with tag:"
                   echo "$sha_short"
-    # TODO: restore the commented `needs` lines once we implement
-    # Docker build caching in github actions
-    build-base:
-        uses: ./.github/workflows/ecrbuild-template.yml
-        secrets:
-            AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-            AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+    # TODO: restore the commented `needs` lines
+    # and the `build-base` job
+    # once we implement Docker build caching in github actions
+    # build-base:
+    #     uses: ./.github/workflows/ecrbuild-template.yml
+    #     secrets:
+    #         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    #         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
 
     build-core:
         uses: ./.github/workflows/ecrbuild-template.yml
@@ -60,6 +61,7 @@ jobs:
         #   - build-base
         with:
             package: jobs
+            target: jobs
         secrets:
             AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
             AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}