Deploy to Production

.github/workflows/deploy.yml

@@ -104,8 +104,8 @@ jobs:
           NODE_OPTIONS: --dns-result-order=ipv4first
           GATSBY_ROLLBAR_TOKEN: ${{ secrets.GATSBY_ROLLBAR_TOKEN }}
           SKIP_PAGE_CREATOR: ${{ vars.SKIP_PAGE_CREATOR }}
-          CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }}
-          CLOUDFLARE_R2_BUCKET_NAME: ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }}
+          CLOUDFLARE_R2_ACCOUNT_ID: ${{ vars.CLOUDFLARE_R2_ACCOUNT_ID }}
+          CLOUDFLARE_R2_BUCKET_NAME: ${{ vars.CLOUDFLARE_R2_BUCKET_NAME }}
           GATSBY_CLOUDFLARE_R2_PUBLIC_BUCKET_URL: ${{ vars.GATSBY_CLOUDFLARE_R2_PUBLIC_BUCKET_URL }}
           CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
           CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}

.github/workflows/test-build.yml

@@ -101,8 +101,8 @@ jobs:
           NODE_OPTIONS: --dns-result-order=ipv4first
           GATSBY_ROLLBAR_TOKEN: ${{ secrets.GATSBY_ROLLBAR_TOKEN }}
           SKIP_PAGE_CREATOR: ${{ vars.SKIP_PAGE_CREATOR }}
-          CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }}
-          CLOUDFLARE_R2_BUCKET_NAME: ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }}
+          CLOUDFLARE_R2_ACCOUNT_ID: ${{ vars.CLOUDFLARE_R2_ACCOUNT_ID }}
+          CLOUDFLARE_R2_BUCKET_NAME: ${{ vars.CLOUDFLARE_R2_BUCKET_NAME }}
           GATSBY_CLOUDFLARE_R2_PUBLIC_BUCKET_URL: ${{ vars.GATSBY_CLOUDFLARE_R2_PUBLIC_BUCKET_URL }}
           CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
           CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
@@ -124,5 +124,5 @@ jobs:
         with:
           path: |
             site/gatsby-site/public
-            site/gatsby-site/.cache/functions
+            site/gatsby-site/netlify/functions
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ inputs.sha }}-${{ inputs.cache-modifier }}

.github/workflows/test-playwright-full.yml

@@ -95,8 +95,8 @@ jobs:
           PRISMIC_ACCESS_TOKEN: ${{ secrets.PRISMIC_ACCESS_TOKEN }}
           GATSBY_ROLLBAR_TOKEN: ${{ secrets.GATSBY_ROLLBAR_TOKEN }}
           SKIP_PAGE_CREATOR: ${{ vars.SKIP_PAGE_CREATOR }}
-          CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }}
-          CLOUDFLARE_R2_BUCKET_NAME: ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }}
+          CLOUDFLARE_R2_ACCOUNT_ID: ${{ vars.CLOUDFLARE_R2_ACCOUNT_ID }}
+          CLOUDFLARE_R2_BUCKET_NAME: ${{ vars.CLOUDFLARE_R2_BUCKET_NAME }}
           GATSBY_CLOUDFLARE_R2_PUBLIC_BUCKET_URL: ${{ vars.GATSBY_CLOUDFLARE_R2_PUBLIC_BUCKET_URL }}
           CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
           CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
@@ -122,6 +122,14 @@ jobs:
           E2E_ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }}
           IS_EMPTY_ENVIRONMENT: ${{ vars.IS_EMPTY_ENVIRONMENT }}
           MONGODB_CONNECTION_STRING: mongodb://127.0.0.1:4110/
+          REALM_API_APP_ID: ${{ vars.REALM_API_APP_ID }}
+          REALM_API_GROUP_ID: ${{ vars.REALM_API_GROUP_ID }}
+          REALM_APP_ID: ${{ vars.GATSBY_REALM_APP_ID }}
+          REALM_API_PUBLIC_KEY: ${{ secrets.REALM_API_PUBLIC_KEY }}
+          REALM_API_PRIVATE_KEY: ${{ secrets.REALM_API_PRIVATE_KEY }}
+          REALM_GRAPHQL_API_KEY: ${{ secrets.REALM_GRAPHQL_API_KEY }}
+          API_MONGODB_CONNECTION_STRING: mongodb://127.0.0.1:4110/
+          ROLLBAR_POST_SERVER_ITEM_ACCESS_TOKEN: ${{ secrets.GATSBY_ROLLBAR_TOKEN }}
           GATSBY_AVAILABLE_LANGUAGES: ${{ vars.GATSBY_AVAILABLE_LANGUAGES }}
           SHARD_INDEX: ${{ matrix.shardIndex }}
           SHARD_TOTAL: ${{ matrix.shardTotal }}

.github/workflows/test-playwright.yml

@@ -63,7 +63,7 @@ jobs:
         with:
           path: |
             site/gatsby-site/public
-            site/gatsby-site/.cache/functions
+            site/gatsby-site/netlify/functions
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ inputs.sha }}-${{ inputs.cache-modifier }}
 
       - name: Install playwright browsers
@@ -78,6 +78,14 @@ jobs:
           E2E_ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }}
           IS_EMPTY_ENVIRONMENT: ${{ vars.IS_EMPTY_ENVIRONMENT }}
           GATSBY_AVAILABLE_LANGUAGES: ${{ vars.GATSBY_AVAILABLE_LANGUAGES }}
+          REALM_API_APP_ID: ${{ vars.REALM_API_APP_ID }}
+          REALM_API_GROUP_ID: ${{ vars.REALM_API_GROUP_ID }}
+          REALM_APP_ID: ${{ vars.GATSBY_REALM_APP_ID }}
+          REALM_API_PUBLIC_KEY: ${{ secrets.REALM_API_PUBLIC_KEY }}
+          REALM_API_PRIVATE_KEY: ${{ secrets.REALM_API_PRIVATE_KEY }}
+          REALM_GRAPHQL_API_KEY: ${{ secrets.REALM_GRAPHQL_API_KEY }}
+          API_MONGODB_CONNECTION_STRING: ${{ secrets.API_MONGODB_CONNECTION_STRING }}
+          ROLLBAR_POST_SERVER_ITEM_ACCESS_TOKEN: ${{ secrets.GATSBY_ROLLBAR_TOKEN }}
           SHARD_INDEX: ${{ matrix.shardIndex }}
           SHARD_TOTAL: ${{ matrix.shardTotal }}
           TEST_FOLDER: playwright/e2e/

.github/workflows/test.yml

@@ -73,7 +73,7 @@ jobs:
         with:
           path: |
             site/gatsby-site/public
-            site/gatsby-site/.cache/functions
+            site/gatsby-site/netlify/functions
           key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ inputs.sha }}-${{ inputs.cache-modifier }}
 
       - name: Extract branch name
@@ -91,7 +91,7 @@ jobs:
           parallel: true
           group: "Cypress e2e tests"
           tag: ${{ steps.extract_branch.outputs.branch }}
-          start: node node_modules/.bin/gatsby serve -p 8000 -H 127.0.0.1
+          start: npx -y pm2 start npm --name "web-server" -- run serve && npx pm2 logs "web-server"
           wait-on: http://127.0.0.1:8000
           wait-on-timeout: 60
         env:
@@ -118,8 +118,8 @@ jobs:
           INSTRUMENT: true
           # Since this is triggered on a pull request, we set the commit message to the pull request title
           COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }}
-          CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }}
-          CLOUDFLARE_R2_BUCKET_NAME: ${{ secrets.CLOUDFLARE_R2_BUCKET_NAME }}
+          CLOUDFLARE_R2_ACCOUNT_ID: ${{ vars.CLOUDFLARE_R2_ACCOUNT_ID }}
+          CLOUDFLARE_R2_BUCKET_NAME: ${{ vars.CLOUDFLARE_R2_BUCKET_NAME }}
           GATSBY_CLOUDFLARE_R2_PUBLIC_BUCKET_URL: ${{ vars.GATSBY_CLOUDFLARE_R2_PUBLIC_BUCKET_URL }}
           CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }}
           CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}

README.md

@@ -132,6 +132,10 @@ See [mongo.md](mongo.md)
 
 ## Setting up a development environment
 
+## Important Notice
+
+This project is currently undergoing a significant restructuring as we transition away from the recently deprecated Atlas GraphQL endpoint. Please note that some parts of the documentation may be outdated. For the most up-to-date information and guidance, please follow [this link](site/gatsby-site/README.md) to the latest documentation.
+
 Depending on what feature you are working on, there will be different systems you'll need to set up after you've forked and cloned this repository:
 
 ### Basic setup

site/gatsby-site/.gitignore

@@ -12,3 +12,5 @@ node_modules
 /playwright-report/
 /blob-report/
 /playwright/.cache/
+
+/netlify/functions/lookupIndex.json

site/gatsby-site/README.md

@@ -0,0 +1,71 @@
+
+### Setting Up a Local Development Environment
+
+To set up a local development environment for the AIID project, follow these steps:
+
+1. **Navigate to the Gatsby Site Directory**
+
+   Open your terminal and navigate to the `site/gatsby-site` directory:
+
+   ```bash
+   cd site/gatsby-site
+   ```
+
+2. **Install Dependencies**
+
+   Run the following command to install all necessary dependencies:
+
+   ```bash
+   npm install
+   ```
+
+3. **Configure Environment Variables**
+
+   Create a `.env` file in the root of the `gatsby-site` directory. Add the following environment variables to the file, replacing the placeholders with your actual credentials:
+
+   ```env
+   REALM_API_APP_ID=  # Application ID for MongoDB Realm API
+   REALM_API_GROUP_ID=  # Group ID for MongoDB Realm API
+   REALM_API_PRIVATE_KEY=  # Private key for accessing the MongoDB Realm API
+   REALM_API_PUBLIC_KEY=  # Public key for accessing the MongoDB Realm API
+   REALM_GRAPHQL_API_KEY=  # API key for accessing the Realm GraphQL API
+   REALM_APP_ID=  # App ID used to access MongoDB Realm services
+   API_MONGODB_CONNECTION_STRING=mongodb://127.0.0.1:4110  # MongoDB connection string
+   ROLLBAR_POST_SERVER_ITEM_ACCESS_TOKEN=  # Token for sending error reports to Rollbar from the server
+   GATSBY_REALM_APP_ID=  # Application ID used in the Gatsby frontend for MongoDB Realm, same as REALM_APP_ID
+   GATSBY_ALGOLIA_APP_ID=  # Application ID for Algolia search integration in the Gatsby app
+   GATSBY_ALGOLIA_SEARCH_KEY=  # Public search key for Algolia, used in the Gatsby frontend
+   ALGOLIA_ADMIN_KEY=  # Admin key for managing the Algolia index
+   MONGODB_CONNECTION_STRING=mongodb://127.0.0.1:4110  # MongoDB connection string
+   MONGODB_REPLICA_SET=  # Name of the MongoDB replica set for high availability
+   MONGODB_TRANSLATIONS_CONNECTION_STRING=mongodb://127.0.0.1:4110  # MongoDB connection string for the translations database
+   GOOGLE_MAPS_API_KEY=  # API key for accessing Google Maps services
+   GATSBY_AVAILABLE_LANGUAGES=  # List of languages available for the Gatsby app (e.g., en, es, fr)
+   GOOGLE_TRANSLATE_API_KEY=  # API key for accessing Google Translate services
+   GATSBY_ROLLBAR_TOKEN=  # Token for Rollbar error tracking in the Gatsby frontend
+   CLOUDFLARE_R2_ACCOUNT_ID=  # Account ID for Cloudflare R2 storage service
+   CLOUDFLARE_R2_ACCESS_KEY_ID=  # Access key ID for Cloudflare R2 storage
+   CLOUDFLARE_R2_SECRET_ACCESS_KEY=  # Secret access key for Cloudflare R2 storage
+   CLOUDFLARE_R2_BUCKET_NAME=  # Name of the Cloudflare R2 bucket for storage
+   GATSBY_CLOUDFLARE_R2_PUBLIC_BUCKET_URL=  # Public URL for accessing the Cloudflare R2 bucket from the Gatsby app
+   ```
+
+   Ensure that each variable is set correctly to match your development environment's requirements.
+
+4. **Start a Memory Mongo Instance**
+
+   To start a memory MongoDB instance, run the following command:
+
+   ```bash
+   npm run start:memory-mongo
+   ```
+
+5. **Start Gatsby and Netlify Development Server**
+
+   Finally, start the Gatsby development server along with Netlify dev using:
+
+   ```bash
+   npm run start
+   ```
+
+Follow these steps to get your local environment up and running for development with the AIID project. Make sure to replace the placeholder values in the `.env` file with your actual credentials to ensure proper functionality.

site/gatsby-site/_redirects

@@ -0,0 +1,7 @@
+# This file is only used when starting a local dev server via `npm run start` or `npm run serve`
+# when deploying please take a look at deploy-netlify.toml
+
+/api/graphql /.netlify/functions/graphql 200
+/api/parseNews /.netlify/functions/parseNews 200
+/api/semanticallyRelated /.netlify/functions/semanticallyRelated 200
+/api/lookupbyurl /.netlify/functions/lookupbyurl 200