set initial mem limit, create lot of pages

benchmarks/memory/Dockerfile

@@ -1,7 +1,7 @@
 FROM node:14-buster
 ENV NODE_ENV=production
 RUN apt-get update -y && apt-get upgrade -y && apt-get install git curl npm -y
-RUN npm i -g gatsby-dev-cli
+RUN npm i -g gatsby-cli gatsby-dev-cli
 WORKDIR /usr/src/app
 RUN echo "\n\necho \"Welcome to the Gatsby Memory benchmark container!\\n  - /usr/src/gatsby : Your local gatsby repo\\n  - /usr/src/app : The memory benchmark gatsby site\\n\"" > /root/.bashrc
 

benchmarks/memory/gatsby-node.js

@@ -1,35 +1,8 @@
-// const {
-//   takeHeapSnapshot,
-// } = require(`./node_modules/gatsby/dist/utils/debug-memory.js`)
+const { cpuCoreCount } = require(`gatsby-core-utils`)
 
-// exports.createSchemaCustomization = ({ actions }) => {
-//   actions.createTypes(`
-//     type Test implements Node @dontInfer {
-//       id: ID!
-//       nodeNum: Int!
-//       nodeNumStr: String!
-//       pageNum: Int!
-//       pageNumStr: String!
-//       fooBar: String!
-//       fooBar2: String!
-//       fooBarArray: [TestFooBarArray!]
-//       text: String!
-//       random: Int!
-//       randomPage: Int!
-//     }
-//     type TestFooBarArray {
-//       fooBar: String!
-//     }
-//     type SitePage implements Node @dontInfer {
-//       id: ID!
-//     }
-//   `)
-// }
-const NUM_NODES = 200
+const NUM_NODES = parseInt(process.env.NUM_NODES || 300, 10)
 
 exports.sourceNodes = async ({ actions }) => {
-  // await takeHeapSnapshot(`sourceNodes-1`)
-
   for (let i = 0; i < NUM_NODES; i++) {
     const largeSizeObj = {}
     for (let j = 1; j <= 1024; j++) {
@@ -58,31 +31,90 @@ exports.sourceNodes = async ({ actions }) => {
   }
 
   await new Promise(resolve => setTimeout(resolve, 100))
+}
 
-  // await takeHeapSnapshot(`sourceNodes-2`)
+const printedMessages = new Set()
+exports.createResolvers = ({ createResolvers }) => {
+  createResolvers({
+    Query: {
+      workerInfo: {
+        type: `String`,
+        args: {
+          label: `String!`,
+        },
+        resolve: (_, args) => {
+          const msg = `${args.label} on ${
+            process.env.GATSBY_WORKER_ID
+              ? `worker #${process.env.GATSBY_WORKER_ID}`
+              : `main`
+          }`
+          if (!printedMessages.has(msg)) {
+            printedMessages.add(msg)
+            console.log(msg)
+          }
+          return msg
+        },
+      },
+    },
+  })
 }
 
-// exports.onCreateNode = ({ node, actions, getNode }) => {
-//   if (node.internal.type === `TestChild`) {
-//     const grandpa = getNode(node.parent)
-//     console.log({ grandpa })
+const WORKER_BATCH_SIZE = 50
+exports.createPages = async ({ actions, graphql }) => {
+  const numWorkers = Math.max(1, cpuCoreCount() - 1)
+
+  // we do want ALL available workers to execute each query type
+  const minNumOfPagesToSaturateAllWorkers = WORKER_BATCH_SIZE * numWorkers
+
+  const { data } = await graphql(`
+    {
+      allTest {
+        nodes {
+          id
+          idClone
+        }
+      }
+    }
+  `)
+
+  // we might need to "duplicate" pages if node count is less than number of needed pages
+  const repeatCount = Math.min(
+    1,
+    Math.ceil(minNumOfPagesToSaturateAllWorkers / data.allTest.nodes.length)
+  )
 
-//     actions.createNode({
-//       id: `${node.id} << test child2`,
-//       parent: node.id,
-//       internal: {
-//         type: `TestGrandChild`,
-//         contentDigest: `wa`,
-//       },
-//     })
-//   }
-// }
+  function createEnoughToSaturate(cb) {
+    let counter = 0
+    for (let i = 0; i < repeatCount; i++) {
+      for (const node of data.allTest.nodes) {
+        const { template, context } = cb(node)
 
-exports.createPages = async ({ getNode, action, graphql }) => {
-  debugger
+        actions.createPage({
+          path: `/${template}/${counter++}`,
+          component: require.resolve(`./src/templates/${template}`),
+          context,
+        })
+      }
+    }
+  }
 
-  const node = getNode(`memory-1`)
-  // console.log({ node })
-  // console.info(`just using node`, node.id)
-  // await takeHeapSnapshot(`create-pages`)
+  // fast path (eq: { id: x })
+  createEnoughToSaturate(node => {
+    return {
+      template: `eq_id`,
+      context: {
+        id: node.id,
+      },
+    }
+  })
+
+  // (eq: { idClone: x })
+  createEnoughToSaturate(node => {
+    return {
+      template: `eq_field`,
+      context: {
+        id: node.id,
+      },
+    }
+  })
 }

benchmarks/memory/scripts/docker-start

@@ -10,6 +10,11 @@ DOCKER_ID=$(\
   docker run -td \
   --mount type=bind,source="$(pwd)/../..",target=/usr/src/gatsby \
   --mount type=bind,source="$(pwd)",target=/usr/src/app \
+  --publish 9229:9229 \
+  --publish 8000:8000 \
+  --publish 9000:9000 \
+  --memory="2g" \
+  --memory-swap="2g" \
   gatsby-memory \
   | head -c 12 \
 )

benchmarks/memory/src/pages/eq_field.js → benchmarks/memory/src/templates/eq_field.js

@@ -10,10 +10,11 @@ export default function Home({ data }) {
 }
 
 export const q = graphql`
-  {
-    test(idClone: { eq: "memory-2" }) {
+  query ($id: String!) {
+    test(idClone: { eq: $id }) {
       id
       fooBar
     }
+    workerInfo(label: "eq-field")
   }
 `

benchmarks/memory/src/pages/eq_id.js → benchmarks/memory/src/templates/eq_id.js

@@ -10,10 +10,11 @@ export default function Home({ data }) {
 }
 
 export const q = graphql`
-  {
-    test(id: { eq: "memory-2" }) {
+  query ($id: String!) {
+    test(id: { eq: $id }) {
       id
       fooBar
     }
+    workerInfo(label: "eq-id")
   }
 `