perf(db-mongodb): improve performance of all operations, up to 50% faster #12841
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| # <workflow_name>-<branch_name>-<true || commit_sha if branch is protected> | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref_protected && github.sha || ''}} | |
| cancel-in-progress: true | |
| env: | |
| NODE_VERSION: 22.6.0 | |
| PNPM_VERSION: 9.7.1 | |
| DO_NOT_TRACK: 1 # Disable Turbopack telemetry | |
| NEXT_TELEMETRY_DISABLED: 1 # Disable Next telemetry | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| outputs: | |
| needs_build: ${{ steps.filter.outputs.needs_build }} | |
| templates: ${{ steps.filter.outputs.templates }} | |
| steps: | |
| # https://github.com/actions/virtual-environments/issues/1187 | |
| - name: tune linux network | |
| run: sudo ethtool -K eth0 tx off rx off | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 25 | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| needs_build: | |
| - '.github/workflows/**' | |
| - 'packages/**' | |
| - 'test/**' | |
| - 'pnpm-lock.yaml' | |
| - 'package.json' | |
| - 'templates/**' | |
| templates: | |
| - 'templates/**' | |
| - name: Log all filter results | |
| run: | | |
| echo "needs_build: ${{ steps.filter.outputs.needs_build }}" | |
| echo "templates: ${{ steps.filter.outputs.templates }}" | |
| lint: | |
| if: > | |
| github.event_name == 'pull_request' && !contains(github.event.pull_request.title, 'no-lint') && !contains(github.event.pull_request.title, 'skip-lint') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # https://github.com/actions/virtual-environments/issues/1187 | |
| - name: tune linux network | |
| run: sudo ethtool -K eth0 tx off rx off | |
| - name: Setup Node@${{ env.NODE_VERSION }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| run_install: false | |
| - name: Get pnpm store directory | |
| shell: bash | |
| run: | | |
| echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV | |
| - name: Setup pnpm cache | |
| uses: actions/cache@v4 | |
| timeout-minutes: 720 | |
| with: | |
| path: ${{ env.STORE_PATH }} | |
| key: pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| pnpm-store- | |
| pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| - run: pnpm install | |
| - name: Lint staged | |
| run: | | |
| git diff --name-only --diff-filter=d origin/${GITHUB_BASE_REF}...${GITHUB_SHA} | |
| npx lint-staged --diff="origin/${GITHUB_BASE_REF}...${GITHUB_SHA}" | |
| build: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.needs_build == 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 25 | |
| # https://github.com/actions/virtual-environments/issues/1187 | |
| - name: tune linux network | |
| run: sudo ethtool -K eth0 tx off rx off | |
| - name: Setup Node@${{ env.NODE_VERSION }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| run_install: false | |
| - name: Get pnpm store directory | |
| shell: bash | |
| run: | | |
| echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV | |
| - name: Setup pnpm cache | |
| uses: actions/cache@v4 | |
| timeout-minutes: 720 | |
| with: | |
| path: ${{ env.STORE_PATH }} | |
| key: pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| pnpm-store- | |
| pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| - run: pnpm install | |
| - run: pnpm run build:all | |
| env: | |
| DO_NOT_TRACK: 1 # Disable Turbopack telemetry | |
| - name: Cache build | |
| uses: actions/cache@v4 | |
| timeout-minutes: 10 | |
| with: | |
| path: ./* | |
| key: ${{ github.sha }}-${{ github.run_number }} | |
| tests-unit: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| # https://github.com/actions/virtual-environments/issues/1187 | |
| - name: tune linux network | |
| run: sudo ethtool -K eth0 tx off rx off | |
| - name: Setup Node@${{ env.NODE_VERSION }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| run_install: false | |
| - name: Restore build | |
| uses: actions/cache@v4 | |
| timeout-minutes: 10 | |
| with: | |
| path: ./* | |
| key: ${{ github.sha }}-${{ github.run_number }} | |
| - name: Unit Tests | |
| run: pnpm test:unit | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=8096 | |
| tests-int: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| name: int-${{ matrix.database }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| database: | |
| - mongodb | |
| - postgres | |
| - postgres-custom-schema | |
| - postgres-uuid | |
| - supabase | |
| - sqlite | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: payloadtests | |
| AWS_ENDPOINT_URL: http://127.0.0.1:4566 | |
| AWS_ACCESS_KEY_ID: localstack | |
| AWS_SECRET_ACCESS_KEY: localstack | |
| AWS_REGION: us-east-1 | |
| services: | |
| postgres: | |
| image: ${{ (startsWith(matrix.database, 'postgres') ) && 'postgis/postgis:16-3.4' || '' }} | |
| env: | |
| # must specify password for PG Docker container image, see: https://registry.hub.docker.com/_/postgres?tab=description&page=1&name=10 | |
| POSTGRES_USER: ${{ env.POSTGRES_USER }} | |
| POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} | |
| POSTGRES_DB: ${{ env.POSTGRES_DB }} | |
| ports: | |
| - 5432:5432 | |
| # needed because the postgres container does not provide a healthcheck | |
| options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 25 | |
| # https://github.com/actions/virtual-environments/issues/1187 | |
| - name: tune linux network | |
| run: sudo ethtool -K eth0 tx off rx off | |
| - name: Setup Node@${{ env.NODE_VERSION }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| run_install: false | |
| - run: pnpm install | |
| - name: Start LocalStack | |
| run: pnpm docker:start | |
| - name: Install Supabase CLI | |
| uses: supabase/setup-cli@v1 | |
| with: | |
| version: latest | |
| if: matrix.database == 'supabase' | |
| - name: Initialize Supabase | |
| run: | | |
| supabase init | |
| supabase start | |
| if: matrix.database == 'supabase' | |
| - name: Configure PostgreSQL | |
| run: | | |
| psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:5432/$POSTGRES_DB" -c "CREATE ROLE runner SUPERUSER LOGIN;" | |
| psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:5432/$POSTGRES_DB" -c "SELECT version();" | |
| echo "POSTGRES_URL=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:5432/$POSTGRES_DB" >> $GITHUB_ENV | |
| if: startsWith(matrix.database, 'postgres') | |
| - name: Configure PostgreSQL with custom schema | |
| run: | | |
| psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:5432/$POSTGRES_DB" -c "CREATE SCHEMA custom;" | |
| if: matrix.database == 'postgres-custom-schema' | |
| - name: Configure Supabase | |
| run: | | |
| echo "POSTGRES_URL=postgresql://postgres:[email protected]:54322/postgres" >> $GITHUB_ENV | |
| if: matrix.database == 'supabase' | |
| - name: Integration Tests | |
| run: pnpm test:int | |
| env: | |
| NODE_OPTIONS: --max-old-space-size=8096 | |
| PAYLOAD_DATABASE: ${{ matrix.database }} | |
| POSTGRES_URL: ${{ env.POSTGRES_URL }} | |
| tests-e2e: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| name: e2e-${{ matrix.suite }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # find test -type f -name 'e2e.spec.ts' | sort | xargs dirname | xargs -I {} basename {} | |
| suite: | |
| - _community | |
| - access-control | |
| - admin__e2e__1 | |
| - admin__e2e__2 | |
| - admin__e2e__3 | |
| - admin-root | |
| - auth | |
| - auth-basic | |
| - field-error-states | |
| - fields-relationship | |
| - fields | |
| - fields__collections__Blocks | |
| - fields__collections__Array | |
| - fields__collections__Relationship | |
| - fields__collections__RichText | |
| - fields__collections__Lexical__e2e__main | |
| - fields__collections__Lexical__e2e__blocks | |
| - fields__collections__Date | |
| - fields__collections__Number | |
| - fields__collections__Point | |
| - fields__collections__Tabs | |
| - fields__collections__Text | |
| - fields__collections__Upload | |
| - live-preview | |
| - localization | |
| - locked-documents | |
| - i18n | |
| - plugin-cloud-storage | |
| - plugin-form-builder | |
| - plugin-nested-docs | |
| - plugin-seo | |
| - versions | |
| - uploads | |
| env: | |
| SUITE_NAME: ${{ matrix.suite }} | |
| steps: | |
| # https://github.com/actions/virtual-environments/issues/1187 | |
| - name: tune linux network | |
| run: sudo ethtool -K eth0 tx off rx off | |
| - name: Setup Node@${{ env.NODE_VERSION }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| run_install: false | |
| - name: Restore build | |
| uses: actions/cache@v4 | |
| timeout-minutes: 10 | |
| with: | |
| path: ./* | |
| key: ${{ github.sha }}-${{ github.run_number }} | |
| - name: Start LocalStack | |
| run: pnpm docker:start | |
| if: ${{ matrix.suite == 'plugin-cloud-storage' }} | |
| - name: Store Playwright's Version | |
| run: | | |
| # Extract the version number using a more targeted regex pattern with awk | |
| PLAYWRIGHT_VERSION=$(pnpm ls @playwright/test --depth=0 | awk '/@playwright\/test/ {print $2}') | |
| echo "Playwright's Version: $PLAYWRIGHT_VERSION" | |
| echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV | |
| - name: Cache Playwright Browsers for Playwright's Version | |
| id: cache-playwright-browsers | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }} | |
| - name: Setup Playwright - Browsers and Dependencies | |
| if: steps.cache-playwright-browsers.outputs.cache-hit != 'true' | |
| run: pnpm exec playwright install --with-deps chromium | |
| - name: Setup Playwright - Dependencies-only | |
| if: steps.cache-playwright-browsers.outputs.cache-hit == 'true' | |
| run: pnpm exec playwright install-deps chromium | |
| - name: E2E Tests | |
| run: PLAYWRIGHT_JSON_OUTPUT_NAME=results_${{ matrix.suite }}.json pnpm test:e2e:prod:ci ${{ matrix.suite }} | |
| env: | |
| PLAYWRIGHT_JSON_OUTPUT_NAME: results_${{ matrix.suite }}.json | |
| NEXT_TELEMETRY_DISABLED: 1 | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: test-results-${{ matrix.suite }} | |
| path: test/test-results/ | |
| if-no-files-found: ignore | |
| retention-days: 1 | |
| # Disabled until this is fixed: https://github.com/daun/playwright-report-summary/issues/156 | |
| # - uses: daun/playwright-report-summary@v3 | |
| # with: | |
| # report-file: results_${{ matrix.suite }}.json | |
| # report-tag: ${{ matrix.suite }} | |
| # job-summary: true | |
| # Build listed templates with packed local packages | |
| build-templates: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| strategy: | |
| matrix: | |
| include: | |
| - template: blank | |
| database: mongodb | |
| - template: website | |
| database: mongodb | |
| - template: with-payload-cloud | |
| database: mongodb | |
| - template: with-vercel-mongodb | |
| database: mongodb | |
| # Postgres | |
| - template: with-postgres | |
| database: postgres | |
| - template: with-vercel-postgres | |
| database: postgres | |
| # Re-enable once PG conncection is figured out | |
| # - template: with-vercel-website | |
| # database: postgres | |
| name: ${{ matrix.template }}-${{ matrix.database }} | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: payloadtests | |
| steps: | |
| # https://github.com/actions/virtual-environments/issues/1187 | |
| - name: tune linux network | |
| run: sudo ethtool -K eth0 tx off rx off | |
| - name: Setup Node@${{ env.NODE_VERSION }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| run_install: false | |
| - name: Restore build | |
| uses: actions/cache@v4 | |
| timeout-minutes: 10 | |
| with: | |
| path: ./* | |
| key: ${{ github.sha }}-${{ github.run_number }} | |
| - name: Start PostgreSQL | |
| uses: CasperWA/[email protected] | |
| with: | |
| postgresql version: '14' # See https://hub.docker.com/_/postgres for available versions | |
| postgresql db: ${{ env.POSTGRES_DB }} | |
| postgresql user: ${{ env.POSTGRES_USER }} | |
| postgresql password: ${{ env.POSTGRES_PASSWORD }} | |
| if: matrix.database == 'postgres' | |
| - name: Wait for PostgreSQL | |
| run: sleep 30 | |
| if: matrix.database == 'postgres' | |
| - name: Configure PostgreSQL | |
| run: | | |
| psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:5432/$POSTGRES_DB" -c "CREATE ROLE runner SUPERUSER LOGIN;" | |
| psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:5432/$POSTGRES_DB" -c "SELECT version();" | |
| echo "POSTGRES_URL=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:5432/$POSTGRES_DB" >> $GITHUB_ENV | |
| if: matrix.database == 'postgres' | |
| - name: Start MongoDB | |
| uses: supercharge/[email protected] | |
| with: | |
| mongodb-version: 6.0 | |
| - name: Build Template | |
| run: | | |
| pnpm run script:pack --dest templates/${{ matrix.template }} | |
| pnpm runts scripts/build-template-with-local-pkgs.ts ${{ matrix.template }} $POSTGRES_URL | |
| tests-type-generation: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| # https://github.com/actions/virtual-environments/issues/1187 | |
| - name: tune linux network | |
| run: sudo ethtool -K eth0 tx off rx off | |
| - name: Setup Node@${{ env.NODE_VERSION }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| run_install: false | |
| - name: Restore build | |
| uses: actions/cache@v4 | |
| timeout-minutes: 10 | |
| with: | |
| path: ./* | |
| key: ${{ github.sha }}-${{ github.run_number }} | |
| - name: Generate Payload Types | |
| run: pnpm dev:generate-types fields | |
| - name: Generate GraphQL schema file | |
| run: pnpm dev:generate-graphql-schema graphql-schema-gen | |
| all-green: | |
| name: All Green | |
| if: always() | |
| runs-on: ubuntu-latest | |
| needs: | |
| - lint | |
| - build | |
| - tests-unit | |
| - tests-int | |
| - tests-e2e | |
| steps: | |
| - if: ${{ always() && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} | |
| run: exit 1 | |
| publish-canary: | |
| name: Publish Canary | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.all-green.result == 'success' && github.ref_name == 'main' }} | |
| needs: | |
| - all-green | |
| steps: | |
| # debug github.ref output | |
| - run: | | |
| echo github.ref: ${{ github.ref }} | |
| echo isV3: ${{ github.ref == 'refs/heads/main' }} |