From 435885a8e0c03b43518fe29c4cd293a5e6009565 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 21 Jan 2022 14:19:49 +0100 Subject: [PATCH] set initial mem limit, create lot of pages --- benchmarks/memory/Dockerfile | 2 +- benchmarks/memory/gatsby-node.js | 132 +++++++++++------- benchmarks/memory/scripts/docker-start | 5 + .../src/{pages => templates}/double_eq.js | 0 .../src/{pages => templates}/eq_field.js | 5 +- .../memory/src/{pages => templates}/eq_id.js | 5 +- 6 files changed, 94 insertions(+), 55 deletions(-) rename benchmarks/memory/src/{pages => templates}/double_eq.js (100%) rename benchmarks/memory/src/{pages => templates}/eq_field.js (73%) rename benchmarks/memory/src/{pages => templates}/eq_id.js (74%) diff --git a/benchmarks/memory/Dockerfile b/benchmarks/memory/Dockerfile index 91ff1cb53a400..7601139ba9f4b 100644 --- a/benchmarks/memory/Dockerfile +++ b/benchmarks/memory/Dockerfile @@ -1,7 +1,7 @@ FROM node:14-buster ENV NODE_ENV=production RUN apt-get update -y && apt-get upgrade -y && apt-get install git curl npm -y -RUN npm i -g gatsby-dev-cli +RUN npm i -g gatsby-cli gatsby-dev-cli WORKDIR /usr/src/app RUN echo "\n\necho \"Welcome to the Gatsby Memory benchmark container!\\n - /usr/src/gatsby : Your local gatsby repo\\n - /usr/src/app : The memory benchmark gatsby site\\n\"" > /root/.bashrc diff --git a/benchmarks/memory/gatsby-node.js b/benchmarks/memory/gatsby-node.js index 8e1a737d19d72..5bf3fa5cc05e3 100644 --- a/benchmarks/memory/gatsby-node.js +++ b/benchmarks/memory/gatsby-node.js @@ -1,35 +1,8 @@ -// const { -// takeHeapSnapshot, -// } = require(`./node_modules/gatsby/dist/utils/debug-memory.js`) +const { cpuCoreCount } = require(`gatsby-core-utils`) -// exports.createSchemaCustomization = ({ actions }) => { -// actions.createTypes(` -// type Test implements Node @dontInfer { -// id: ID! -// nodeNum: Int! -// nodeNumStr: String! -// pageNum: Int! -// pageNumStr: String! -// fooBar: String! -// fooBar2: String! -// fooBarArray: [TestFooBarArray!] -// text: String! -// random: Int! -// randomPage: Int! -// } -// type TestFooBarArray { -// fooBar: String! -// } -// type SitePage implements Node @dontInfer { -// id: ID! -// } -// `) -// } -const NUM_NODES = 200 +const NUM_NODES = parseInt(process.env.NUM_NODES || 300, 10) exports.sourceNodes = async ({ actions }) => { - // await takeHeapSnapshot(`sourceNodes-1`) - for (let i = 0; i < NUM_NODES; i++) { const largeSizeObj = {} for (let j = 1; j <= 1024; j++) { @@ -58,31 +31,90 @@ exports.sourceNodes = async ({ actions }) => { } await new Promise(resolve => setTimeout(resolve, 100)) +} - // await takeHeapSnapshot(`sourceNodes-2`) +const printedMessages = new Set() +exports.createResolvers = ({ createResolvers }) => { + createResolvers({ + Query: { + workerInfo: { + type: `String`, + args: { + label: `String!`, + }, + resolve: (_, args) => { + const msg = `${args.label} on ${ + process.env.GATSBY_WORKER_ID + ? `worker #${process.env.GATSBY_WORKER_ID}` + : `main` + }` + if (!printedMessages.has(msg)) { + printedMessages.add(msg) + console.log(msg) + } + return msg + }, + }, + }, + }) } -// exports.onCreateNode = ({ node, actions, getNode }) => { -// if (node.internal.type === `TestChild`) { -// const grandpa = getNode(node.parent) -// console.log({ grandpa }) +const WORKER_BATCH_SIZE = 50 +exports.createPages = async ({ actions, graphql }) => { + const numWorkers = Math.max(1, cpuCoreCount() - 1) + + // we do want ALL available workers to execute each query type + const minNumOfPagesToSaturateAllWorkers = WORKER_BATCH_SIZE * numWorkers + + const { data } = await graphql(` + { + allTest { + nodes { + id + idClone + } + } + } + `) + + // we might need to "duplicate" pages if node count is less than number of needed pages + const repeatCount = Math.min( + 1, + Math.ceil(minNumOfPagesToSaturateAllWorkers / data.allTest.nodes.length) + ) -// actions.createNode({ -// id: `${node.id} << test child2`, -// parent: node.id, -// internal: { -// type: `TestGrandChild`, -// contentDigest: `wa`, -// }, -// }) -// } -// } + function createEnoughToSaturate(cb) { + let counter = 0 + for (let i = 0; i < repeatCount; i++) { + for (const node of data.allTest.nodes) { + const { template, context } = cb(node) -exports.createPages = async ({ getNode, action, graphql }) => { - debugger + actions.createPage({ + path: `/${template}/${counter++}`, + component: require.resolve(`./src/templates/${template}`), + context, + }) + } + } + } - const node = getNode(`memory-1`) - // console.log({ node }) - // console.info(`just using node`, node.id) - // await takeHeapSnapshot(`create-pages`) + // fast path (eq: { id: x }) + createEnoughToSaturate(node => { + return { + template: `eq_id`, + context: { + id: node.id, + }, + } + }) + + // (eq: { idClone: x }) + createEnoughToSaturate(node => { + return { + template: `eq_field`, + context: { + id: node.id, + }, + } + }) } diff --git a/benchmarks/memory/scripts/docker-start b/benchmarks/memory/scripts/docker-start index 6528dd2a8f7d3..f68497e2524f3 100755 --- a/benchmarks/memory/scripts/docker-start +++ b/benchmarks/memory/scripts/docker-start @@ -10,6 +10,11 @@ DOCKER_ID=$(\ docker run -td \ --mount type=bind,source="$(pwd)/../..",target=/usr/src/gatsby \ --mount type=bind,source="$(pwd)",target=/usr/src/app \ + --publish 9229:9229 \ + --publish 8000:8000 \ + --publish 9000:9000 \ + --memory="2g" \ + --memory-swap="2g" \ gatsby-memory \ | head -c 12 \ ) diff --git a/benchmarks/memory/src/pages/double_eq.js b/benchmarks/memory/src/templates/double_eq.js similarity index 100% rename from benchmarks/memory/src/pages/double_eq.js rename to benchmarks/memory/src/templates/double_eq.js diff --git a/benchmarks/memory/src/pages/eq_field.js b/benchmarks/memory/src/templates/eq_field.js similarity index 73% rename from benchmarks/memory/src/pages/eq_field.js rename to benchmarks/memory/src/templates/eq_field.js index 5d85bbc4c0d03..c881ada4c05f3 100644 --- a/benchmarks/memory/src/pages/eq_field.js +++ b/benchmarks/memory/src/templates/eq_field.js @@ -10,10 +10,11 @@ export default function Home({ data }) { } export const q = graphql` - { - test(idClone: { eq: "memory-2" }) { + query ($id: String!) { + test(idClone: { eq: $id }) { id fooBar } + workerInfo(label: "eq-field") } ` diff --git a/benchmarks/memory/src/pages/eq_id.js b/benchmarks/memory/src/templates/eq_id.js similarity index 74% rename from benchmarks/memory/src/pages/eq_id.js rename to benchmarks/memory/src/templates/eq_id.js index cde339ed799ab..3bca139fc3c26 100644 --- a/benchmarks/memory/src/pages/eq_id.js +++ b/benchmarks/memory/src/templates/eq_id.js @@ -10,10 +10,11 @@ export default function Home({ data }) { } export const q = graphql` - { - test(id: { eq: "memory-2" }) { + query ($id: String!) { + test(id: { eq: $id }) { id fooBar } + workerInfo(label: "eq-id") } `