From 353b60dae24ab7d0a392ab5d808687fe6d0c7455 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 10 Oct 2018 02:12:22 -0700 Subject: [PATCH 01/63] add createContentDigest helper for gatsby-transformer-remark --- packages/gatsby-transformer-remark/package.json | 2 +- .../__tests__/__snapshots__/gatsby-node.js.snap | 8 ++++---- .../src/__tests__/extend-node.js | 2 ++ .../src/__tests__/gatsby-node.js | 8 ++++++++ .../src/on-node-create.js | 16 ++++++++++------ 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/packages/gatsby-transformer-remark/package.json b/packages/gatsby-transformer-remark/package.json index 20f948b559e27..09f7723945f22 100644 --- a/packages/gatsby-transformer-remark/package.json +++ b/packages/gatsby-transformer-remark/package.json @@ -41,7 +41,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-remark", "scripts": { diff --git a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/gatsby-node.js.snap index 663f38f8d4a04..72f7f255f7039 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/gatsby-node.js.snap +++ b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -15,7 +15,7 @@ Array [ "internal": Object { "content": "Where oh where is my little pony? ", - "contentDigest": "e33e9489dc1352bfac2577e99155c1b3", + "contentDigest": "contentDigest", "type": "MarkdownRemark", }, "parent": "whatever", @@ -42,7 +42,7 @@ Array [ "internal": Object { "content": "Where oh where is my little pony? ", - "contentDigest": "e33e9489dc1352bfac2577e99155c1b3", + "contentDigest": "contentDigest", "type": "MarkdownRemark", }, "parent": "whatever", @@ -97,7 +97,7 @@ Sed eu gravida mauris. Suspendisse potenti. Praesent sit amet egestas mi, sed he Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat mattis dui nec pretium. Donec vel consectetur est. Nam sagittis, libero vitae pretium pharetra, velit est dignissim erat, at cursus quam massa vitae ligula. Suspendisse potenti. In hac habitasse platea dictumst. Donec sit amet finibus justo. Mauris ante dolor, pulvinar vitae feugiat eu, rhoncus nec diam. In ut accumsan diam, faucibus fringilla odio. Nunc id ultricies turpis. Quisque justo quam, tristique sit amet interdum quis, facilisis at mi. Fusce porttitor vel sem ut condimentum. Praesent at libero congue, vulputate elit ut, rhoncus erat. ", - "contentDigest": "e9f6c76d1bc3cb15f402ed2c24b815bd", + "contentDigest": "contentDigest", "type": "MarkdownRemark", }, "parent": "whatever", @@ -148,7 +148,7 @@ Sed eu gravida mauris. Suspendisse potenti. Praesent sit amet egestas mi, sed he Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat mattis dui nec pretium. Donec vel consectetur est. Nam sagittis, libero vitae pretium pharetra, velit est dignissim erat, at cursus quam massa vitae ligula. Suspendisse potenti. In hac habitasse platea dictumst. Donec sit amet finibus justo. Mauris ante dolor, pulvinar vitae feugiat eu, rhoncus nec diam. In ut accumsan diam, faucibus fringilla odio. Nunc id ultricies turpis. Quisque justo quam, tristique sit amet interdum quis, facilisis at mi. Fusce porttitor vel sem ut condimentum. Praesent at libero congue, vulputate elit ut, rhoncus erat. ", - "contentDigest": "e9f6c76d1bc3cb15f402ed2c24b815bd", + "contentDigest": "contentDigest", "type": "MarkdownRemark", }, "parent": "whatever", diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 51fcb01df9803..34a08d4f8e679 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -106,11 +106,13 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) = const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) + const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode({ node, loadNodeContent, actions, createNodeId, + createContentDigest, }, { ...additionalParameters } ) diff --git a/packages/gatsby-transformer-remark/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-remark/src/__tests__/gatsby-node.js index 70709a35f3c90..c8a5f9597660e 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/gatsby-node.js @@ -41,12 +41,14 @@ Where oh where is my little pony? const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) + const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode({ node, loadNodeContent, actions, createNodeId, + createContentDigest, }).then(() => { expect(createNode.mock.calls).toMatchSnapshot() expect( @@ -83,6 +85,7 @@ Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat ma const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) + const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode( { @@ -90,6 +93,7 @@ Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat ma loadNodeContent, actions, createNodeId, + createContentDigest, }, { excerpt_separator: `` } ).then(() => { @@ -194,6 +198,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) + const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) onCreateNode( { @@ -201,6 +206,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid loadNodeContent, actions, createNodeId, + createContentDigest, }, { excerpt_separator: `` } ) @@ -250,12 +256,14 @@ Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat ma const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) + const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) onCreateNode({ node, loadNodeContent, actions, createNodeId, + createContentDigest, }) }) }) diff --git a/packages/gatsby-transformer-remark/src/on-node-create.js b/packages/gatsby-transformer-remark/src/on-node-create.js index 76b2931ca82ad..211a74ded150e 100644 --- a/packages/gatsby-transformer-remark/src/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/on-node-create.js @@ -1,9 +1,16 @@ const grayMatter = require(`gray-matter`) -const crypto = require(`crypto`) const _ = require(`lodash`) module.exports = async function onCreateNode( - { node, getNode, loadNodeContent, actions, createNodeId, reporter }, + { + node, + getNode, + loadNodeContent, + actions, + createNodeId, + createContentDigest, + reporter, + }, pluginOptions ) { const { createNode, createParentChildLink } = actions @@ -56,10 +63,7 @@ module.exports = async function onCreateNode( markdownNode.fileAbsolutePath = node.absolutePath } - markdownNode.internal.contentDigest = crypto - .createHash(`md5`) - .update(JSON.stringify(markdownNode)) - .digest(`hex`) + markdownNode.internal.contentDigest = createContentDigest(markdownNode) createNode(markdownNode) createParentChildLink({ parent: node, child: markdownNode }) From 9c0e1870adaacd67fac23f553739caae18988a0a Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 10 Oct 2018 07:18:28 -0700 Subject: [PATCH 02/63] add createContentDigest helper for gatsby-source-filesystem --- .../gatsby-source-filesystem/package.json | 2 +- .../src/create-file-node.js | 12 ++++----- .../src/create-remote-file-node.js | 25 ++++++------------- .../src/gatsby-node.js | 10 +++++++- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json index 1184c3bd3ad24..951782a449ae6 100644 --- a/packages/gatsby-source-filesystem/package.json +++ b/packages/gatsby-source-filesystem/package.json @@ -32,7 +32,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem", "scripts": { diff --git a/packages/gatsby-source-filesystem/src/create-file-node.js b/packages/gatsby-source-filesystem/src/create-file-node.js index e64e86d53fa90..7157c401c4b6a 100644 --- a/packages/gatsby-source-filesystem/src/create-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-file-node.js @@ -5,11 +5,11 @@ const mime = require(`mime`) const prettyBytes = require(`pretty-bytes`) const md5File = require(`bluebird`).promisify(require(`md5-file`)) -const crypto = require(`crypto`) exports.createFileNode = async ( pathToFile, createNodeId, + createContentDigest, pluginOptions = {} ) => { const slashed = slash(pathToFile) @@ -27,12 +27,10 @@ exports.createFileNode = async ( const stats = await fs.stat(slashedFile.absolutePath) let internal if (stats.isDirectory()) { - const contentDigest = crypto - .createHash(`md5`) - .update( - JSON.stringify({ stats: stats, absolutePath: slashedFile.absolutePath }) - ) - .digest(`hex`) + const contentDigest = createContentDigest({ + stats: stats, + absolutePath: slashedFile.absolutePath, + }) internal = { contentDigest, type: `Directory`, diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index fc29a6fbdae59..d2d3561fda63f 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -1,6 +1,5 @@ const fs = require(`fs-extra`) const got = require(`got`) -const crypto = require(`crypto`) const path = require(`path`) const { isWebUri } = require(`valid-url`) const Queue = require(`better-queue`) @@ -46,20 +45,6 @@ const cacheId = url => `create-remote-file-node-${url}` * utils * *********/ -/** - * createHash - * -- - * - * Create an md5 hash of the given str - * @param {Stringq} str - * @return {String} - */ -const createHash = str => - crypto - .createHash(`md5`) - .update(str) - .digest(`hex`) - const CACHE_DIR = `.cache` const FS_PLUGIN_DIR = `gatsby-source-filesystem` @@ -172,6 +157,7 @@ async function processRemoteNode({ store, cache, createNode, + createContentDigest, auth = {}, createNodeId, }) { @@ -195,7 +181,7 @@ async function processRemoteNode({ } // Create the temp and permanent file names for the url. - const digest = createHash(url) + const digest = createContentDigest(url) const ext = getRemoteFileExtension(url) const tmpFilename = createFilePath(programDir, `tmp-${digest}`, ext) @@ -221,7 +207,12 @@ async function processRemoteNode({ } // Create the file node. - const fileNode = await createFileNode(filename, createNodeId, {}) + const fileNode = await createFileNode( + filename, + createNodeId, + createContentDigest, + {} + ) fileNode.internal.description = `File "${url}"` // Override the default plugin as gatsby-source-filesystem needs to // be the owner of File nodes or there'll be conflicts if any other diff --git a/packages/gatsby-source-filesystem/src/gatsby-node.js b/packages/gatsby-source-filesystem/src/gatsby-node.js index 9cdcd27a51621..b8e72054d333f 100644 --- a/packages/gatsby-source-filesystem/src/gatsby-node.js +++ b/packages/gatsby-source-filesystem/src/gatsby-node.js @@ -48,7 +48,15 @@ const createFSMachine = () => }) exports.sourceNodes = ( - { actions, getNode, createNodeId, hasNodeChanged, reporter, emitter }, + { + actions, + getNode, + createNodeId, + createContentDigest, + hasNodeChanged, + reporter, + emitter, + }, pluginOptions ) => { const { createNode, deleteNode } = actions From d3b39313c6a604fe8598244eb32496a05413530a Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 10 Oct 2018 07:30:32 -0700 Subject: [PATCH 03/63] Added createContentDigest in test --- .../gatsby-source-filesystem/src/__tests__/create-file-node.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js b/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js index 3224b1ad833c4..01ec6b90eed70 100644 --- a/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js +++ b/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js @@ -8,9 +8,11 @@ describe(`create-file-node`, () => { it(`creates a file node`, () => { const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) + const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) return createFileNode( path.resolve(`${__dirname}/fixtures/file.json`), createNodeId, + createContentDigest, {} ) }) From 4c13027541f3fd183b497e13bbb96e6638751879 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 10 Oct 2018 07:40:47 -0700 Subject: [PATCH 04/63] Added missing createContentDigest param --- packages/gatsby-source-filesystem/src/gatsby-node.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/gatsby-source-filesystem/src/gatsby-node.js b/packages/gatsby-source-filesystem/src/gatsby-node.js index b8e72054d333f..cd9f8d8b0922c 100644 --- a/packages/gatsby-source-filesystem/src/gatsby-node.js +++ b/packages/gatsby-source-filesystem/src/gatsby-node.js @@ -111,6 +111,7 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/ const fileNodePromise = createFileNode( path, createNodeId, + createContentDigest, pluginOptions ).then(fileNode => { createNode(fileNode) From 3bfd43a2dc036bc9eacc9170496fce0c3885aae2 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Mon, 15 Oct 2018 11:12:14 -0700 Subject: [PATCH 05/63] update createContentDigest helper tests for gatsby-transformer-remark --- .../src/__tests__/__snapshots__/gatsby-node.js.snap | 8 ++++---- .../src/__tests__/gatsby-node.js | 5 +---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/gatsby-node.js.snap index 72f7f255f7039..663f38f8d4a04 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/gatsby-node.js.snap +++ b/packages/gatsby-transformer-remark/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -15,7 +15,7 @@ Array [ "internal": Object { "content": "Where oh where is my little pony? ", - "contentDigest": "contentDigest", + "contentDigest": "e33e9489dc1352bfac2577e99155c1b3", "type": "MarkdownRemark", }, "parent": "whatever", @@ -42,7 +42,7 @@ Array [ "internal": Object { "content": "Where oh where is my little pony? ", - "contentDigest": "contentDigest", + "contentDigest": "e33e9489dc1352bfac2577e99155c1b3", "type": "MarkdownRemark", }, "parent": "whatever", @@ -97,7 +97,7 @@ Sed eu gravida mauris. Suspendisse potenti. Praesent sit amet egestas mi, sed he Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat mattis dui nec pretium. Donec vel consectetur est. Nam sagittis, libero vitae pretium pharetra, velit est dignissim erat, at cursus quam massa vitae ligula. Suspendisse potenti. In hac habitasse platea dictumst. Donec sit amet finibus justo. Mauris ante dolor, pulvinar vitae feugiat eu, rhoncus nec diam. In ut accumsan diam, faucibus fringilla odio. Nunc id ultricies turpis. Quisque justo quam, tristique sit amet interdum quis, facilisis at mi. Fusce porttitor vel sem ut condimentum. Praesent at libero congue, vulputate elit ut, rhoncus erat. ", - "contentDigest": "contentDigest", + "contentDigest": "e9f6c76d1bc3cb15f402ed2c24b815bd", "type": "MarkdownRemark", }, "parent": "whatever", @@ -148,7 +148,7 @@ Sed eu gravida mauris. Suspendisse potenti. Praesent sit amet egestas mi, sed he Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat mattis dui nec pretium. Donec vel consectetur est. Nam sagittis, libero vitae pretium pharetra, velit est dignissim erat, at cursus quam massa vitae ligula. Suspendisse potenti. In hac habitasse platea dictumst. Donec sit amet finibus justo. Mauris ante dolor, pulvinar vitae feugiat eu, rhoncus nec diam. In ut accumsan diam, faucibus fringilla odio. Nunc id ultricies turpis. Quisque justo quam, tristique sit amet interdum quis, facilisis at mi. Fusce porttitor vel sem ut condimentum. Praesent at libero congue, vulputate elit ut, rhoncus erat. ", - "contentDigest": "contentDigest", + "contentDigest": "e9f6c76d1bc3cb15f402ed2c24b815bd", "type": "MarkdownRemark", }, "parent": "whatever", diff --git a/packages/gatsby-transformer-remark/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-remark/src/__tests__/gatsby-node.js index c8a5f9597660e..d0a20e5088b7e 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/gatsby-node.js @@ -12,6 +12,7 @@ const { const { inferObjectStructureFromNodes, } = require(`../../../gatsby/src/schema/infer-graphql-type`) +const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) describe(`Process markdown content correctly`, () => { const node = { @@ -41,7 +42,6 @@ Where oh where is my little pony? const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) - const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode({ node, @@ -85,7 +85,6 @@ Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat ma const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) - const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode( { @@ -198,7 +197,6 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) - const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) onCreateNode( { @@ -256,7 +254,6 @@ Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat ma const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) - const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) onCreateNode({ node, From 46e277a4f78f3151a797b807b82d7924bf3b10ee Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Mon, 15 Oct 2018 12:05:30 -0700 Subject: [PATCH 06/63] update createContentDigest helper tests for gatsby-source-filesystem --- .../gatsby-source-filesystem/src/__tests__/create-file-node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js b/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js index 01ec6b90eed70..701df26ebf8f8 100644 --- a/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js +++ b/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js @@ -1,6 +1,7 @@ const path = require(`path`) const { createFileNode } = require(`../create-file-node`) +const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) // FIXME: This test needs to not use snapshots because of file differences // and locations across users and CI systems @@ -8,7 +9,6 @@ describe(`create-file-node`, () => { it(`creates a file node`, () => { const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) - const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) return createFileNode( path.resolve(`${__dirname}/fixtures/file.json`), createNodeId, From d8e9689886b94b452903f463d5bfb7f6b796aa1e Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Mon, 15 Oct 2018 12:07:37 -0700 Subject: [PATCH 07/63] update createContentDigest helper tests for gatsby-transformer-csv --- .../__snapshots__/gatsby-node.js.snap | 20 +++++++++---------- .../src/__tests__/gatsby-node.js | 3 +-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap index 876a82914a97d..14f2b875a35e2 100644 --- a/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap +++ b/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -9,7 +9,7 @@ Array [ "funny": "yup", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "5005aee6b2557974cae0aabc712e125a", "type": "TestCsv", }, "parent": "whatever", @@ -22,7 +22,7 @@ Array [ "funny": "nope", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "0af87da7e572c2d6d8493bcd642c2c78", "type": "TestCsv", }, "parent": "whatever", @@ -41,7 +41,7 @@ Array [ "funny": "yup", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "5005aee6b2557974cae0aabc712e125a", "type": "TestCsv", }, "parent": "whatever", @@ -70,7 +70,7 @@ false,\\"nope\\"", "funny": "nope", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "0af87da7e572c2d6d8493bcd642c2c78", "type": "TestCsv", }, "parent": "whatever", @@ -103,7 +103,7 @@ Array [ "field2": "funny", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "49605f5e32a2250d7ca740c5d2e79784", "type": "TestCsv", }, "parent": "whatever", @@ -116,7 +116,7 @@ Array [ "field2": "yup", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "2bf0bc0e8539ef5f6bbcf67c8b733233", "type": "TestCsv", }, "parent": "whatever", @@ -129,7 +129,7 @@ Array [ "field2": "nope", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "f5f617ce4243d639de924543dec5600e", "type": "TestCsv", }, "parent": "whatever", @@ -148,7 +148,7 @@ Array [ "field2": "funny", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "49605f5e32a2250d7ca740c5d2e79784", "type": "TestCsv", }, "parent": "whatever", @@ -177,7 +177,7 @@ false,nope", "field2": "yup", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "2bf0bc0e8539ef5f6bbcf67c8b733233", "type": "TestCsv", }, "parent": "whatever", @@ -206,7 +206,7 @@ false,nope", "field2": "nope", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "contentDigest", + "contentDigest": "f5f617ce4243d639de924543dec5600e", "type": "TestCsv", }, "parent": "whatever", diff --git a/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js index 5cf7bd33dbfac..2cc931dde8d38 100644 --- a/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js @@ -2,6 +2,7 @@ const Promise = require(`bluebird`) const json2csv = require(`json2csv`) const { onCreateNode } = require(`../gatsby-node`) +const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) describe(`Process nodes correctly`, () => { const node = { @@ -30,7 +31,6 @@ describe(`Process nodes correctly`, () => { const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) - const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode({ node, @@ -54,7 +54,6 @@ describe(`Process nodes correctly`, () => { const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) - const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode( { From 096f1c8d0f0bdacdffcb70550d7166db0d45fedc Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Mon, 15 Oct 2018 13:11:05 -0700 Subject: [PATCH 08/63] add createContentDigest helper for internal-plugins/internal-data-bridge --- .../internal-data-bridge/gatsby-node.js | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js index a5af127a384de..11ff24b02ee84 100644 --- a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js +++ b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js @@ -1,4 +1,3 @@ -const crypto = require(`crypto`) const moment = require(`moment`) const chokidar = require(`chokidar`) const systemPath = require(`path`) @@ -7,6 +6,7 @@ const _ = require(`lodash`) const { emitter } = require(`../../redux`) const { boundActionCreators } = require(`../../redux/actions`) const { getNode } = require(`../../redux`) +const createContentDigest = require(`../../utils/create-content-digest`) function transformPackageJson(json) { const transformDeps = deps => @@ -58,10 +58,7 @@ exports.sourceNodes = ({ actions, store }) => { children: [], internal: { type: `SitePage`, - contentDigest: crypto - .createHash(`md5`) - .update(JSON.stringify(page)) - .digest(`hex`), + contentDigest: createContentDigest(page), }, }) @@ -75,10 +72,7 @@ exports.sourceNodes = ({ actions, store }) => { parent: `SOURCE`, children: [], internal: { - contentDigest: crypto - .createHash(`md5`) - .update(JSON.stringify(plugin)) - .digest(`hex`), + contentDigest: createContentDigest(plugin), type: `SitePlugin`, }, }) @@ -108,10 +102,7 @@ exports.sourceNodes = ({ actions, store }) => { parent: `SOURCE`, children: [], internal: { - contentDigest: crypto - .createHash(`md5`) - .update(JSON.stringify(node)) - .digest(`hex`), + contentDigest: createContentDigest(node), type: `Site`, }, }) @@ -152,10 +143,7 @@ exports.onCreatePage = ({ page, actions }) => { children: [], internal: { type: `SitePage`, - contentDigest: crypto - .createHash(`md5`) - .update(JSON.stringify(pageWithoutUpdated)) - .digest(`hex`), + contentDigest: createContentDigest(pageWithoutUpdated), description: page.pluginCreatorId === `Plugin default-site-plugin` ? `Your site's "gatsby-node.js"` From 9d28bec601491f4a85474186d2cfcf39aa2cd338 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Mon, 15 Oct 2018 13:17:56 -0700 Subject: [PATCH 09/63] add createContentDigest helper for gatsby-source-hacker-news --- .../gatsby-source-hacker-news/src/gatsby-node.js | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/gatsby-source-hacker-news/src/gatsby-node.js b/packages/gatsby-source-hacker-news/src/gatsby-node.js index e53b2e2b06a9e..efb96ce4db52d 100644 --- a/packages/gatsby-source-hacker-news/src/gatsby-node.js +++ b/packages/gatsby-source-hacker-news/src/gatsby-node.js @@ -1,5 +1,4 @@ const axios = require(`axios`) -const crypto = require(`crypto`) const url = require(`url`) const _ = require(`lodash`) @@ -12,6 +11,7 @@ exports.sourceNodes = async ({ actions, getNode, createNodeId, + createContentDigest, hasNodeChanged, }) => { const { createNode } = actions @@ -117,10 +117,7 @@ fragment commentsFragment on HackerNewsItem { storyNode.by = storyNode.by.id // Get content digest of node. - const contentDigest = crypto - .createHash(`md5`) - .update(JSON.stringify(storyNode)) - .digest(`hex`) + const contentDigest = createContentDigest(storyNode) storyNode.internal.contentDigest = contentDigest createNode(storyNode) @@ -147,10 +144,7 @@ fragment commentsFragment on HackerNewsItem { const nodeStr = JSON.stringify(commentNode) // Get content digest of comment node. - const contentDigest = crypto - .createHash(`md5`) - .update(nodeStr) - .digest(`hex`) + const contentDigest = createContentDigest(nodeStr) commentNode.internal.contentDigest = contentDigest commentNode.internal.content = nodeStr From f379bb6eb372241ce644d364618a09e110b4e1fb Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Mon, 15 Oct 2018 13:22:21 -0700 Subject: [PATCH 10/63] add createContentDigest helper for gatsby-transformer-excel --- .../src/__tests__/gatsby-node.js | 4 ++++ .../gatsby-transformer-excel/src/gatsby-node.js | 13 +++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/gatsby-transformer-excel/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-excel/src/__tests__/gatsby-node.js index ba878f2429394..b273743aafa38 100644 --- a/packages/gatsby-transformer-excel/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-excel/src/__tests__/gatsby-node.js @@ -2,6 +2,7 @@ const Promise = require(`bluebird`) const XLSX = require(`xlsx`) const { onCreateNode } = require(`../gatsby-node`) +const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) describe(`Process nodes correctly`, () => { const node = { @@ -35,6 +36,7 @@ describe(`Process nodes correctly`, () => { loadNodeContent, actions, createNodeId, + createContentDigest, }).then(() => { expect(createNode.mock.calls).toMatchSnapshot() expect(createParentChildLink.mock.calls).toMatchSnapshot() @@ -60,6 +62,7 @@ describe(`Process nodes correctly`, () => { loadNodeContent, actions, createNodeId, + createContentDigest, }, { raw: false } ).then(() => { @@ -87,6 +90,7 @@ describe(`Process nodes correctly`, () => { loadNodeContent, actions, createNodeId, + createContentDigest, }, { rawOutput: false } ).then(() => { diff --git a/packages/gatsby-transformer-excel/src/gatsby-node.js b/packages/gatsby-transformer-excel/src/gatsby-node.js index eb537e6e24719..d37c390e38123 100644 --- a/packages/gatsby-transformer-excel/src/gatsby-node.js +++ b/packages/gatsby-transformer-excel/src/gatsby-node.js @@ -1,7 +1,6 @@ const XLSX = require(`xlsx`) const fs = require(`fs-extra`) const _ = require(`lodash`) -const crypto = require(`crypto`) // read files as `binary` from file system function _loadNodeContent(fileNode, fallback) { @@ -11,7 +10,7 @@ function _loadNodeContent(fileNode, fallback) { } async function onCreateNode( - { node, actions, loadNodeContent, createNodeId }, + { node, actions, loadNodeContent, createNodeId, createContentDigest }, options = {} ) { const { createNode, createParentChildLink } = actions @@ -46,10 +45,7 @@ async function onCreateNode( if (_.isArray(parsedContent)) { const csvArray = parsedContent.map((obj, i) => { const objStr = JSON.stringify(obj) - const contentDigest = crypto - .createHash(`md5`) - .update(objStr) - .digest(`hex`) + const contentDigest = createContentDigest(objStr) return { ...obj, @@ -75,10 +71,7 @@ async function onCreateNode( const shObj = { name: n, idx: idx } const shStr = JSON.stringify(shObj) - const contentDigest = crypto - .createHash(`md5`) - .update(shStr) - .digest(`hex`) + const contentDigest = createContentDigest(shStr) const z = { id: createNodeId(`${node.id} [${idx}] >>> ${node.extension}`), From 2320a4876ffd35f5abac21904eb7cc214961004e Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 17 Oct 2018 11:15:09 -0700 Subject: [PATCH 11/63] add createContentDigest helper for gatsby-source-mongodb --- .../gatsby-source-mongodb/src/gatsby-node.js | 11 +++++------ packages/gatsby-source-mongodb/src/mapping.js | 17 ++++++++++------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/gatsby-source-mongodb/src/gatsby-node.js b/packages/gatsby-source-mongodb/src/gatsby-node.js index f6cd45d56d74f..df7ed54ff1fd3 100644 --- a/packages/gatsby-source-mongodb/src/gatsby-node.js +++ b/packages/gatsby-source-mongodb/src/gatsby-node.js @@ -1,9 +1,10 @@ const MongoClient = require(`mongodb`).MongoClient -const crypto = require(`crypto`) const prepareMappingChildNode = require(`./mapping`) const _ = require(`lodash`) const queryString = require(`query-string`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) + exports.sourceNodes = ( { actions, getNode, createNodeId, hasNodeChanged }, pluginOptions @@ -85,10 +86,7 @@ function createNodes( internal: { type: `mongodb${caps(dbName)}${caps(collectionName)}`, content: JSON.stringify(item), - contentDigest: crypto - .createHash(`md5`) - .update(JSON.stringify(item)) - .digest(`hex`), + contentDigest: createContentDigest(item), }, } const childrenNodes = [] @@ -109,7 +107,8 @@ function createNodes( mediaItemFieldKey, node[mediaItemFieldKey], mapObj[mediaItemFieldKey], - createNode + createNode, + createContentDigest ) node[`${mediaItemFieldKey}___NODE`] = mappingChildNode.id diff --git a/packages/gatsby-source-mongodb/src/mapping.js b/packages/gatsby-source-mongodb/src/mapping.js index 61b027ebe61e3..29c5c1c586498 100644 --- a/packages/gatsby-source-mongodb/src/mapping.js +++ b/packages/gatsby-source-mongodb/src/mapping.js @@ -1,7 +1,13 @@ -const _ = require(`lodash`), - crypto = require(`crypto`) +const _ = require(`lodash`) -module.exports = function(node, key, text, mediaType, createNode) { +module.exports = function( + node, + key, + text, + mediaType, + createNode, + createContentDigest +) { const str = _.isString(text) ? text : ` ` const id = `${node.id}${key}MappingNode` const mappingNode = { @@ -13,10 +19,7 @@ module.exports = function(node, key, text, mediaType, createNode) { type: _.camelCase(`${node.internal.type} ${key} MappingNode`), mediaType: mediaType, content: str, - contentDigest: crypto - .createHash(`md5`) - .update(JSON.stringify(str)) - .digest(`hex`), + contentDigest: createContentDigest(str), }, } From 0a1e9c24a8fd30753ea409ffe250c6b11172f4da Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 17 Oct 2018 11:19:26 -0700 Subject: [PATCH 12/63] add createContentDigest helper for gatsby-source-lever --- packages/gatsby-source-lever/src/normalize.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/gatsby-source-lever/src/normalize.js b/packages/gatsby-source-lever/src/normalize.js index b0750c4cfb5c1..2dd1a7540c2cc 100644 --- a/packages/gatsby-source-lever/src/normalize.js +++ b/packages/gatsby-source-lever/src/normalize.js @@ -1,4 +1,3 @@ -const crypto = require(`crypto`) const deepMapKeys = require(`deep-map-keys`) const stringify = require(`json-stringify-safe`) @@ -6,16 +5,14 @@ const conflictFieldPrefix = `lever_` // restrictedNodeFields from here https://www.gatsbyjs.org/docs/node-interface/ const restrictedNodeFields = [`id`, `children`, `parent`, `fields`, `internal`] +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) + /** * Encrypts a String using md5 hash of hexadecimal digest. * * @param {any} str */ -const digest = str => - crypto - .createHash(`md5`) - .update(str) - .digest(`hex`) +const digest = str => createContentDigest(str) /** * Create the Graph QL Node From a5e2cd4bb9ec91f3f5265fcae0f200e296ddfd2c Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 17 Oct 2018 11:23:21 -0700 Subject: [PATCH 13/63] add createContentDigest helper for internal-plugins/query-runner --- .../src/internal-plugins/query-runner/redirects-writer.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js b/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js index da2452107af2d..5262da91176c7 100644 --- a/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js +++ b/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js @@ -1,8 +1,8 @@ import _ from "lodash" -import crypto from "crypto" import fs from "fs-extra" import { store, emitter } from "../../redux/" import { joinPath } from "../../utils/path" +import createContentDigest from "../../utils/create-content-digest" let lastHash = null @@ -14,10 +14,7 @@ const writeRedirects = async () => { // Filter for redirects that are meant for the browser. const browserRedirects = redirects.filter(r => r.redirectInBrowser) - const newHash = crypto - .createHash(`md5`) - .update(JSON.stringify(browserRedirects)) - .digest(`hex`) + const newHash = createContentDigest(browserRedirects) if (newHash === lastHash) { return Promise.resolve() From 917b140510b6cfdd3dbabe5daf60beca3d34aa37 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 17 Oct 2018 12:35:54 -0700 Subject: [PATCH 14/63] Updated gatsby dep gatsby-source-hacker-news gatsby-source-lever gatsby-source-mongodb gatsby-transformer-excel --- packages/gatsby-source-hacker-news/package.json | 2 +- packages/gatsby-source-lever/package.json | 2 +- packages/gatsby-source-mongodb/package.json | 2 +- packages/gatsby-transformer-excel/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/gatsby-source-hacker-news/package.json b/packages/gatsby-source-hacker-news/package.json index 98c1d0ede92ab..2377d193f2a27 100644 --- a/packages/gatsby-source-hacker-news/package.json +++ b/packages/gatsby-source-hacker-news/package.json @@ -24,7 +24,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-hacker-news", "scripts": { diff --git a/packages/gatsby-source-lever/package.json b/packages/gatsby-source-lever/package.json index bc861598488f7..08c2aa76bd57c 100644 --- a/packages/gatsby-source-lever/package.json +++ b/packages/gatsby-source-lever/package.json @@ -30,7 +30,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-lever", "scripts": { diff --git a/packages/gatsby-source-mongodb/package.json b/packages/gatsby-source-mongodb/package.json index 60466ea5fb3c3..c5fd9538a9919 100644 --- a/packages/gatsby-source-mongodb/package.json +++ b/packages/gatsby-source-mongodb/package.json @@ -25,7 +25,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-mongodb", "scripts": { diff --git a/packages/gatsby-transformer-excel/package.json b/packages/gatsby-transformer-excel/package.json index ca266d90a0649..68d8740b95b85 100644 --- a/packages/gatsby-transformer-excel/package.json +++ b/packages/gatsby-transformer-excel/package.json @@ -24,7 +24,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-excel", "scripts": { From e9366370835c9ce3a4359d6e93707f4800deb3c2 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 17 Oct 2018 12:52:46 -0700 Subject: [PATCH 15/63] add createContentDigest helper for gatsby-source-contentful --- packages/gatsby-source-contentful/package.json | 2 +- packages/gatsby-source-contentful/src/cache-image.js | 8 +++----- packages/gatsby-source-contentful/src/normalize.js | 9 +++------ 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/packages/gatsby-source-contentful/package.json b/packages/gatsby-source-contentful/package.json index 6a5a61c0aaa0b..3ce4165b8cb09 100644 --- a/packages/gatsby-source-contentful/package.json +++ b/packages/gatsby-source-contentful/package.json @@ -33,7 +33,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-contentful", "scripts": { diff --git a/packages/gatsby-source-contentful/src/cache-image.js b/packages/gatsby-source-contentful/src/cache-image.js index a0a9bb30b04d5..16077053dd955 100644 --- a/packages/gatsby-source-contentful/src/cache-image.js +++ b/packages/gatsby-source-contentful/src/cache-image.js @@ -1,9 +1,10 @@ -const crypto = require(`crypto`) const { resolve, parse } = require(`path`) const axios = require(`axios`) const { pathExists, createWriteStream } = require(`fs-extra`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) + module.exports = async function cacheImage(store, image, options) { const program = store.getState().program const CACHE_DIR = resolve(`${program.directory}/.cache/contentful/assets/`) @@ -37,10 +38,7 @@ module.exports = async function cacheImage(store, image, options) { params.push(`bg=${background}`) } - const optionsHash = crypto - .createHash(`md5`) - .update(JSON.stringify([url, ...params])) - .digest(`hex`) + const optionsHash = createContentDigest([url, ...params]) const { name, ext } = parse(fileName) const absolutePath = resolve(CACHE_DIR, `${name}-${optionsHash}${ext}`) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index e820a887b37d1..e35a6de695929 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -1,13 +1,10 @@ const _ = require(`lodash`) -const crypto = require(`crypto`) const stringify = require(`json-stringify-safe`) const deepMap = require(`deep-map`) -const digest = str => - crypto - .createHash(`md5`) - .update(str) - .digest(`hex`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) + +const digest = str => createContentDigest(str) const typePrefix = `Contentful` const makeTypeName = type => _.upperFirst(_.camelCase(`${typePrefix} ${type}`)) From 524ca7bd267fc0805cf161e89c2fd8165cc55ec7 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 17 Oct 2018 13:01:06 -0700 Subject: [PATCH 16/63] add createContentDigest helper for gatsby-transformer-hjson --- packages/gatsby-transformer-hjson/package.json | 2 +- .../src/__tests__/gatsby-node.js | 4 ++++ .../gatsby-transformer-hjson/src/gatsby-node.js | 14 ++++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/gatsby-transformer-hjson/package.json b/packages/gatsby-transformer-hjson/package.json index cb29e478beaaf..ca83b93d56628 100644 --- a/packages/gatsby-transformer-hjson/package.json +++ b/packages/gatsby-transformer-hjson/package.json @@ -24,7 +24,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-hjson", "scripts": { diff --git a/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js index f631ab236f4fd..754304e5fa4b1 100644 --- a/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js @@ -3,6 +3,8 @@ const HJSON = require(`hjson`) const { onCreateNode } = require(`../gatsby-node`) +const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) + describe(`Process HJSON nodes correctly`, () => { const node = { name: `nodeName`, @@ -37,6 +39,7 @@ describe(`Process HJSON nodes correctly`, () => { loadNodeContent, actions, createNodeId, + createContentDigest, }).then(() => { expect(createNode.mock.calls).toMatchSnapshot() expect(createParentChildLink.mock.calls).toMatchSnapshot() @@ -61,6 +64,7 @@ describe(`Process HJSON nodes correctly`, () => { loadNodeContent, actions, createNodeId, + createContentDigest, }).then(() => { expect(createNode.mock.calls).toMatchSnapshot() expect(createParentChildLink.mock.calls).toMatchSnapshot() diff --git a/packages/gatsby-transformer-hjson/src/gatsby-node.js b/packages/gatsby-transformer-hjson/src/gatsby-node.js index b300604449e04..66c629b493d63 100644 --- a/packages/gatsby-transformer-hjson/src/gatsby-node.js +++ b/packages/gatsby-transformer-hjson/src/gatsby-node.js @@ -1,17 +1,19 @@ const _ = require(`lodash`) -const crypto = require(`crypto`) const path = require(`path`) const HJSON = require(`hjson`) -async function onCreateNode({ node, actions, loadNodeContent, createNodeId }) { +async function onCreateNode({ + node, + actions, + loadNodeContent, + createNodeId, + createContentDigest, +}) { const { createNode, createParentChildLink } = actions function transformObject(obj, id, type) { const objStr = JSON.stringify(obj) - const contentDigest = crypto - .createHash(`md5`) - .update(objStr) - .digest(`hex`) + const contentDigest = createContentDigest(objStr) const jsonNode = { ...obj, id, From 9db133e884f6cca464614f38f5cb8b8be2b70129 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 17 Oct 2018 15:23:30 -0700 Subject: [PATCH 17/63] add createContentDigest helper for gatsby-transformer-sqip --- packages/gatsby-transformer-sqip/package.json | 2 +- packages/gatsby-transformer-sqip/src/generate-sqip.js | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/gatsby-transformer-sqip/package.json b/packages/gatsby-transformer-sqip/package.json index 9e7d656303fb8..8c857feb4f98f 100644 --- a/packages/gatsby-transformer-sqip/package.json +++ b/packages/gatsby-transformer-sqip/package.json @@ -22,7 +22,7 @@ "debug": "^3.1.0" }, "peerDependencies": { - "gatsby": ">2.0.0-alpha", + "gatsby": ">2.0.15", "gatsby-source-contentful": ">2.0.0-alpha", "gatsby-transformer-sharp": ">2.0.0-alpha" }, diff --git a/packages/gatsby-transformer-sqip/src/generate-sqip.js b/packages/gatsby-transformer-sqip/src/generate-sqip.js index 2149cd39ccc47..478ad1a4095d4 100644 --- a/packages/gatsby-transformer-sqip/src/generate-sqip.js +++ b/packages/gatsby-transformer-sqip/src/generate-sqip.js @@ -1,4 +1,3 @@ -const crypto = require(`crypto`) const { resolve, parse } = require(`path`) const Debug = require(`debug`) @@ -10,6 +9,8 @@ const sqip = require(`sqip`) const queue = new PQueue({ concurrency: 1 }) const debug = Debug(`gatsby-transformer-sqip`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) + module.exports = async function generateSqip(options) { const { cache, @@ -30,10 +31,7 @@ module.exports = async function generateSqip(options) { mode, } - const optionsHash = crypto - .createHash(`md5`) - .update(JSON.stringify(sqipOptions)) - .digest(`hex`) + const optionsHash = createContentDigest(sqipOptions) const cacheKey = `sqip-${name}-${optionsHash}` const cachePath = resolve(cacheDir, `${name}-${optionsHash}.svg`) From 90e40f72fe2b11f9117e2b512de0dc22587a390a Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 17 Oct 2018 15:36:56 -0700 Subject: [PATCH 18/63] add createContentDigest helper for gatsby-source-npm-package-search --- packages/gatsby-source-npm-package-search/package.json | 2 +- .../gatsby-source-npm-package-search/src/gatsby-node.js | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/gatsby-source-npm-package-search/package.json b/packages/gatsby-source-npm-package-search/package.json index db4ead25b9dba..5337b69c8ccce 100644 --- a/packages/gatsby-source-npm-package-search/package.json +++ b/packages/gatsby-source-npm-package-search/package.json @@ -19,7 +19,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "scripts": { "build": "babel src --out-dir . --ignore **/__tests__", diff --git a/packages/gatsby-source-npm-package-search/src/gatsby-node.js b/packages/gatsby-source-npm-package-search/src/gatsby-node.js index 13606aef3a41b..116a42c4a1fc9 100644 --- a/packages/gatsby-source-npm-package-search/src/gatsby-node.js +++ b/packages/gatsby-source-npm-package-search/src/gatsby-node.js @@ -1,15 +1,10 @@ const algoliasearch = require(`algoliasearch`) -const crypto = require(`crypto`) + +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const client = algoliasearch(`OFCNCOG2CU`, `6fbcaeafced8913bf0e4d39f0b541957`) var index = client.initIndex(`npm-search`) -const createContentDigest = obj => - crypto - .createHash(`md5`) - .update(JSON.stringify(obj)) - .digest(`hex`) - function browse({ index, ...params }) { let hits = [] const browser = index.browseAll(params) From 66674f1405a96c20cdf1611c638a88c1a472cdb5 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 17 Oct 2018 15:46:46 -0700 Subject: [PATCH 19/63] add createContentDigest helper for gatsby-transformer-toml --- packages/gatsby-transformer-toml/package.json | 2 +- .../src/__tests__/gatsby-node.js | 3 +++ .../gatsby-transformer-toml/src/gatsby-node.js | 14 ++++++++------ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/gatsby-transformer-toml/package.json b/packages/gatsby-transformer-toml/package.json index b72207da6f7b2..111c98d6d3aff 100644 --- a/packages/gatsby-transformer-toml/package.json +++ b/packages/gatsby-transformer-toml/package.json @@ -24,7 +24,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.0-alpha" + "gatsby": ">2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-toml", "scripts": { diff --git a/packages/gatsby-transformer-toml/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-toml/src/__tests__/gatsby-node.js index e827bfed37104..5e94579b18615 100644 --- a/packages/gatsby-transformer-toml/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-toml/src/__tests__/gatsby-node.js @@ -1,5 +1,7 @@ const { onCreateNode } = require(`../gatsby-node`) +const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) + describe(`Process TOML nodes correctly`, () => { const node = { id: `whatever`, @@ -48,6 +50,7 @@ describe(`Process TOML nodes correctly`, () => { loadNodeContent, actions, createNodeId, + createContentDigest, }).then(() => { expect(createNode.mock.calls).toMatchSnapshot() expect(createParentChildLink.mock.calls).toMatchSnapshot() diff --git a/packages/gatsby-transformer-toml/src/gatsby-node.js b/packages/gatsby-transformer-toml/src/gatsby-node.js index f4caa0f627769..8b4f8a3b56299 100644 --- a/packages/gatsby-transformer-toml/src/gatsby-node.js +++ b/packages/gatsby-transformer-toml/src/gatsby-node.js @@ -1,8 +1,13 @@ const toml = require(`toml`) const _ = require(`lodash`) -const crypto = require(`crypto`) -async function onCreateNode({ node, actions, loadNodeContent, createNodeId }) { +async function onCreateNode({ + node, + actions, + loadNodeContent, + createNodeId, + createContentDigest, +}) { const { createNode, createParentChildLink } = actions // Filter out non-toml content // Currently TOML files are considered null in 'mime-db' @@ -19,10 +24,7 @@ async function onCreateNode({ node, actions, loadNodeContent, createNodeId }) { // 1) More TOML files -> more types // 2) Different files with the same name creating conflicts const parsedContentStr = JSON.stringify(parsedContent) - const contentDigest = crypto - .createHash(`md5`) - .update(parsedContentStr) - .digest(`hex`) + const contentDigest = createContentDigest(parsedContentStr) const newNode = { ...parsedContent, From 002c8c3035c4355017747142701aea21f3ce12a4 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 18 Oct 2018 09:23:31 -0700 Subject: [PATCH 20/63] update createContentDigest helper for gatsby-source-mongodb remove direct import --- packages/gatsby-source-mongodb/src/gatsby-node.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/gatsby-source-mongodb/src/gatsby-node.js b/packages/gatsby-source-mongodb/src/gatsby-node.js index df7ed54ff1fd3..915204a321478 100644 --- a/packages/gatsby-source-mongodb/src/gatsby-node.js +++ b/packages/gatsby-source-mongodb/src/gatsby-node.js @@ -3,10 +3,8 @@ const prepareMappingChildNode = require(`./mapping`) const _ = require(`lodash`) const queryString = require(`query-string`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) - exports.sourceNodes = ( - { actions, getNode, createNodeId, hasNodeChanged }, + { actions, getNode, createNodeId, createContentDigest, hasNodeChanged }, pluginOptions ) => { const { createNode } = actions @@ -36,7 +34,15 @@ exports.sourceNodes = ( return Promise.all( collection.map(col => - createNodes(db, pluginOptions, dbName, createNode, createNodeId, col) + createNodes( + db, + pluginOptions, + dbName, + createNode, + createNodeId, + createContentDigest, + col + ) ) ) .then(() => { @@ -60,6 +66,7 @@ function createNodes( dbName, createNode, createNodeId, + createContentDigest, collectionName ) { return new Promise((resolve, reject) => { From 4b79ae906eee6c69e9d73a5b3bf9bea0a4bad64d Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 18 Oct 2018 09:34:45 -0700 Subject: [PATCH 21/63] update createContentDigest helper for gatsby-source-lever could not find import for createGraphQLNode, added helper anyways --- .../src/__tests__/normalize.js | 8 +++++- .../gatsby-source-lever/src/gatsby-node.js | 8 ++++-- packages/gatsby-source-lever/src/normalize.js | 28 ++++++++++--------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/packages/gatsby-source-lever/src/__tests__/normalize.js b/packages/gatsby-source-lever/src/__tests__/normalize.js index aa976b2abade5..8b97d01412d8a 100644 --- a/packages/gatsby-source-lever/src/__tests__/normalize.js +++ b/packages/gatsby-source-lever/src/__tests__/normalize.js @@ -2,6 +2,8 @@ const normalize = require(`../normalize`) let entities = require(`./data.json`) +const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) + describe(`Process Lever data`, () => { it(`Creates entities from object collections of entities`, () => { entities = normalize.normalizeEntities(entities) @@ -22,7 +24,11 @@ describe(`Process Lever data`, () => { }) it(`creates nodes for each entry`, () => { const createNode = jest.fn() - normalize.createNodesFromEntities({ entities, createNode }) + normalize.createNodesFromEntities({ + entities, + createNode, + createContentDigest, + }) expect(createNode.mock.calls).toMatchSnapshot() }) }) diff --git a/packages/gatsby-source-lever/src/gatsby-node.js b/packages/gatsby-source-lever/src/gatsby-node.js index afac68c621408..ce6a70b8d2add 100644 --- a/packages/gatsby-source-lever/src/gatsby-node.js +++ b/packages/gatsby-source-lever/src/gatsby-node.js @@ -4,7 +4,7 @@ const normalize = require(`./normalize`) const typePrefix = `lever__` exports.sourceNodes = async ( - { actions, getNode, store, cache, createNodeId }, + { actions, getNode, store, cache, createNodeId, createContentDigest }, { site, verboseOutput } ) => { const { createNode } = actions @@ -30,7 +30,11 @@ exports.sourceNodes = async ( entities = normalize.createGatsbyIds(createNodeId, entities) // creates nodes for each entry - normalize.createNodesFromEntities({ entities, createNode }) + normalize.createNodesFromEntities({ + entities, + createNode, + createContentDigest, + }) return } diff --git a/packages/gatsby-source-lever/src/normalize.js b/packages/gatsby-source-lever/src/normalize.js index 2dd1a7540c2cc..3f90f9321cd33 100644 --- a/packages/gatsby-source-lever/src/normalize.js +++ b/packages/gatsby-source-lever/src/normalize.js @@ -5,15 +5,6 @@ const conflictFieldPrefix = `lever_` // restrictedNodeFields from here https://www.gatsbyjs.org/docs/node-interface/ const restrictedNodeFields = [`id`, `children`, `parent`, `fields`, `internal`] -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) - -/** - * Encrypts a String using md5 hash of hexadecimal digest. - * - * @param {any} str - */ -const digest = str => createContentDigest(str) - /** * Create the Graph QL Node * @@ -21,7 +12,14 @@ const digest = str => createContentDigest(str) * @param {any} type * @param {any} createNode */ -async function createGraphQLNode(ent, type, createNode, store, cache) { +async function createGraphQLNode( + ent, + type, + createNode, + createContentDigest, + store, + cache +) { let id = !ent.id ? (!ent.ID ? 0 : ent.ID) : ent.id let node = { id: `${type}_${id.toString()}`, @@ -33,7 +31,7 @@ async function createGraphQLNode(ent, type, createNode, store, cache) { } node = recursiveAddFields(ent, node, createNode) node.internal.content = JSON.stringify(node) - node.internal.contentDigest = digest(stringify(node)) + node.internal.contentDigest = createContentDigest(stringify(node)) createNode(node) } exports.createGraphQLNode = createGraphQLNode @@ -131,7 +129,11 @@ exports.createGatsbyIds = (createNodeId, entities) => return e }) -exports.createNodesFromEntities = ({ entities, createNode }) => { +exports.createNodesFromEntities = ({ + entities, + createNode, + createContentDigest, +}) => { entities.forEach(e => { let { ...entity } = e let node = { @@ -140,7 +142,7 @@ exports.createNodesFromEntities = ({ entities, createNode }) => { children: [], internal: { type: `lever`, - contentDigest: digest(JSON.stringify(entity)), + contentDigest: createContentDigest(JSON.stringify(entity)), }, } createNode(node) From aa9c1ff5a2bdeeecc62f34cefa77ccaf3bc2c30b Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 18 Oct 2018 09:42:08 -0700 Subject: [PATCH 22/63] update createContentDigest helper for internal-plugins/query-runner modify src/bootstrap (imports writeRedirects) --- packages/gatsby/src/bootstrap/index.js | 3 ++- .../src/internal-plugins/query-runner/redirects-writer.js | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/gatsby/src/bootstrap/index.js b/packages/gatsby/src/bootstrap/index.js index 8adc768cdd327..b9bad4138f7b0 100644 --- a/packages/gatsby/src/bootstrap/index.js +++ b/packages/gatsby/src/bootstrap/index.js @@ -11,6 +11,7 @@ const convertHrtime = require(`convert-hrtime`) const Promise = require(`bluebird`) const apiRunnerNode = require(`../utils/api-runner-node`) +const createContentDigest = require(`../utils/create-content-digest`) const mergeGatsbyConfig = require(`../utils/merge-gatsby-config`) const { graphql } = require(`graphql`) const { store, emitter } = require(`../redux`) @@ -460,7 +461,7 @@ module.exports = async (args: BootstrapArgs) => { parentSpan: bootstrapSpan, }) activity.start() - await writeRedirects() + await writeRedirects(createContentDigest) activity.end() const checkJobsDone = _.debounce(resolve => { diff --git a/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js b/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js index 5262da91176c7..425d43ace9e58 100644 --- a/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js +++ b/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js @@ -2,11 +2,10 @@ import _ from "lodash" import fs from "fs-extra" import { store, emitter } from "../../redux/" import { joinPath } from "../../utils/path" -import createContentDigest from "../../utils/create-content-digest" let lastHash = null -const writeRedirects = async () => { +const writeRedirects = async createContentDigest => { bootstrapFinished = true let { program, redirects } = store.getState() From 4a9c1526ff3431f9ff002be171fb7eb5963fbe29 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 18 Oct 2018 09:58:54 -0700 Subject: [PATCH 23/63] update createContentDigest helper for gatsby-source-npm-package-search --- packages/gatsby-source-npm-package-search/src/gatsby-node.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/gatsby-source-npm-package-search/src/gatsby-node.js b/packages/gatsby-source-npm-package-search/src/gatsby-node.js index 116a42c4a1fc9..590a55fc154c1 100644 --- a/packages/gatsby-source-npm-package-search/src/gatsby-node.js +++ b/packages/gatsby-source-npm-package-search/src/gatsby-node.js @@ -1,7 +1,5 @@ const algoliasearch = require(`algoliasearch`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) - const client = algoliasearch(`OFCNCOG2CU`, `6fbcaeafced8913bf0e4d39f0b541957`) var index = client.initIndex(`npm-search`) @@ -17,7 +15,7 @@ function browse({ index, ...params }) { } exports.sourceNodes = async ( - { boundActionCreators, createNodeId }, + { boundActionCreators, createNodeId, createContentDigest }, { keywords } ) => { const { createNode } = boundActionCreators From 58ecb370430c53eb1dc8c642e9458fdcf43edd85 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 18 Oct 2018 11:05:27 -0700 Subject: [PATCH 24/63] update createContentDigest helper for gatsby-source-contentful* TODO add ccd (createContentDigest) in cache-image --- .../src/__tests__/normalize.js | 4 ++ .../src/cache-image.js | 9 +++-- .../src/gatsby-node.js | 12 +++++- .../gatsby-source-contentful/src/normalize.js | 40 +++++++++++++------ 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/packages/gatsby-source-contentful/src/__tests__/normalize.js b/packages/gatsby-source-contentful/src/__tests__/normalize.js index 97067311f02b0..feb339af2d092 100644 --- a/packages/gatsby-source-contentful/src/__tests__/normalize.js +++ b/packages/gatsby-source-contentful/src/__tests__/normalize.js @@ -6,6 +6,8 @@ const { locales, } = require(`./data.json`) +const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) + let entryList let resolvable let foreignReferenceMap @@ -62,6 +64,7 @@ describe(`Process contentful data`, () => { entries: entryList[i].map(normalize.fixIds), createNode, createNodeId, + createContentDigest, resolvable, foreignReferenceMap, defaultLocale, @@ -81,6 +84,7 @@ describe(`Process contentful data`, () => { assetItem, createNode, createNodeId, + createContentDigest, defaultLocale, locales, }) diff --git a/packages/gatsby-source-contentful/src/cache-image.js b/packages/gatsby-source-contentful/src/cache-image.js index 16077053dd955..7affe01c3380e 100644 --- a/packages/gatsby-source-contentful/src/cache-image.js +++ b/packages/gatsby-source-contentful/src/cache-image.js @@ -1,10 +1,9 @@ +const crypto = require(`crypto`) const { resolve, parse } = require(`path`) const axios = require(`axios`) const { pathExists, createWriteStream } = require(`fs-extra`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) - module.exports = async function cacheImage(store, image, options) { const program = store.getState().program const CACHE_DIR = resolve(`${program.directory}/.cache/contentful/assets/`) @@ -38,7 +37,11 @@ module.exports = async function cacheImage(store, image, options) { params.push(`bg=${background}`) } - const optionsHash = createContentDigest([url, ...params]) + // TODO convert to `createContentDigest` using dependency import + const optionsHash = crypto + .createHash(`md5`) + .update(JSON.stringify([url, ...params])) + .digest(`hex`) const { name, ext } = parse(fileName) const absolutePath = resolve(CACHE_DIR, `${name}-${optionsHash}${ext}`) diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 9f922498dc0c4..94f207db2437d 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -32,7 +32,15 @@ exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeTyp */ exports.sourceNodes = async ( - { actions, getNode, getNodes, createNodeId, hasNodeChanged, store }, + { + actions, + getNode, + getNodes, + createNodeId, + createContentDigest, + hasNodeChanged, + store, + }, options ) => { const { createNode, deleteNode, touchNode, setPluginStatus } = actions @@ -193,6 +201,7 @@ exports.sourceNodes = async ( entries: entryList[i], createNode, createNodeId, + createContentDigest, resolvable, foreignReferenceMap, defaultLocale, @@ -205,6 +214,7 @@ exports.sourceNodes = async ( assetItem, createNode, createNodeId, + createContentDigest, defaultLocale, locales, }) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index e35a6de695929..46ba694981073 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -2,9 +2,6 @@ const _ = require(`lodash`) const stringify = require(`json-stringify-safe`) const deepMap = require(`deep-map`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) - -const digest = str => createContentDigest(str) const typePrefix = `Contentful` const makeTypeName = type => _.upperFirst(_.camelCase(`${typePrefix} ${type}`)) @@ -161,7 +158,14 @@ exports.buildForeignReferenceMap = ({ return foreignReferenceMap } -function prepareTextNode(node, key, text, createNode, createNodeId) { +function prepareTextNode( + node, + key, + text, + createNode, + createNodeId, + createContentDigest +) { const str = _.isString(text) ? text : ` ` const textNode = { id: createNodeId(`${node.id}${key}TextNode`), @@ -172,7 +176,7 @@ function prepareTextNode(node, key, text, createNode, createNodeId) { type: _.camelCase(`${node.internal.type} ${key} TextNode`), mediaType: `text/markdown`, content: str, - contentDigest: digest(str), + contentDigest: createContentDigest(str), }, } @@ -181,7 +185,14 @@ function prepareTextNode(node, key, text, createNode, createNodeId) { return textNode } -function prepareJSONNode(node, key, content, createNodeId, i = ``) { +function prepareJSONNode( + node, + key, + content, + createNodeId, + createContentDigest, + i = `` +) { const str = JSON.stringify(content) const JSONNode = { ...(_.isPlainObject(content) ? { ...content } : { content: content }), @@ -192,7 +203,7 @@ function prepareJSONNode(node, key, content, createNodeId, i = ``) { type: _.camelCase(`${node.internal.type} ${key} JSONNode`), mediaType: `application/json`, content: str, - contentDigest: digest(str), + contentDigest: createContentDigest(str), }, } @@ -208,6 +219,7 @@ exports.createContentTypeNodes = ({ entries, createNode, createNodeId, + createContentDigest, resolvable, foreignReferenceMap, defaultLocale, @@ -364,7 +376,8 @@ exports.createContentTypeNodes = ({ entryItemFieldKey, entryItemFields[entryItemFieldKey], createNode, - createNodeId + createNodeId, + createContentDigest ) childrenNodes.push(textNode) @@ -379,7 +392,8 @@ exports.createContentTypeNodes = ({ entryNode, entryItemFieldKey, entryItemFields[entryItemFieldKey], - createNodeId + createNodeId, + createContentDigest ) childrenNodes.push(jsonNode) @@ -398,6 +412,7 @@ exports.createContentTypeNodes = ({ entryItemFieldKey, obj, createNodeId, + createContentDigest, i ) @@ -412,7 +427,7 @@ exports.createContentTypeNodes = ({ entryNode = { ...entryItemFields, ...entryNode, node_locale: locale.code } // Get content digest of node. - const contentDigest = digest(stringify(entryNode)) + const contentDigest = createContentDigest(stringify(entryNode)) entryNode.internal.contentDigest = contentDigest @@ -433,7 +448,7 @@ exports.createContentTypeNodes = ({ } // Get content digest of node. - const contentDigest = digest(stringify(contentTypeNode)) + const contentDigest = createContentDigest(stringify(contentTypeNode)) contentTypeNode.internal.contentDigest = contentDigest @@ -451,6 +466,7 @@ exports.createAssetNodes = ({ assetItem, createNode, createNodeId, + createContentDigest, defaultLocale, locales, }) => { @@ -494,7 +510,7 @@ exports.createAssetNodes = ({ } // Get content digest of node. - const contentDigest = digest(stringify(assetNode)) + const contentDigest = createContentDigest(stringify(assetNode)) assetNode.internal.contentDigest = contentDigest From cdb0a8a2602387996c88d53c5ba25605f6c372fa Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 18 Oct 2018 11:08:50 -0700 Subject: [PATCH 25/63] update createContentDigest helper for gatsby-transformer-sqip* TODO add ccd (createContentDigest) in generate-sqip --- .../src/__tests__/generate-sqip.js | 4 ++++ packages/gatsby-transformer-sqip/src/generate-sqip.js | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-transformer-sqip/src/__tests__/generate-sqip.js b/packages/gatsby-transformer-sqip/src/__tests__/generate-sqip.js index 179e011fa843f..4e6ad70a4b995 100644 --- a/packages/gatsby-transformer-sqip/src/__tests__/generate-sqip.js +++ b/packages/gatsby-transformer-sqip/src/__tests__/generate-sqip.js @@ -5,6 +5,8 @@ const sqip = require(`sqip`) const generateSqip = require(`../generate-sqip.js`) +const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) + jest.mock(`sqip`, () => jest.fn(() => { return { @@ -52,6 +54,7 @@ describe(`gatsby-transformer-sqip`, async () => { numberOfPrimitives, blur, mode, + createContentDigest, }) expect(result).toMatchSnapshot() @@ -81,6 +84,7 @@ describe(`gatsby-transformer-sqip`, async () => { numberOfPrimitives, blur, mode, + createContentDigest, }) expect(result).toMatchSnapshot() diff --git a/packages/gatsby-transformer-sqip/src/generate-sqip.js b/packages/gatsby-transformer-sqip/src/generate-sqip.js index 478ad1a4095d4..2149cd39ccc47 100644 --- a/packages/gatsby-transformer-sqip/src/generate-sqip.js +++ b/packages/gatsby-transformer-sqip/src/generate-sqip.js @@ -1,3 +1,4 @@ +const crypto = require(`crypto`) const { resolve, parse } = require(`path`) const Debug = require(`debug`) @@ -9,8 +10,6 @@ const sqip = require(`sqip`) const queue = new PQueue({ concurrency: 1 }) const debug = Debug(`gatsby-transformer-sqip`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) - module.exports = async function generateSqip(options) { const { cache, @@ -31,7 +30,10 @@ module.exports = async function generateSqip(options) { mode, } - const optionsHash = createContentDigest(sqipOptions) + const optionsHash = crypto + .createHash(`md5`) + .update(JSON.stringify(sqipOptions)) + .digest(`hex`) const cacheKey = `sqip-${name}-${optionsHash}` const cachePath = resolve(cacheDir, `${name}-${optionsHash}.svg`) From c0f681c37ce5c29f57693d14ea6ace2098b299df Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 18 Oct 2018 11:39:08 -0700 Subject: [PATCH 26/63] update createContentDigest helper for gatsby-transformer-sqip removed ccd from tests --- .../gatsby-transformer-sqip/src/__tests__/generate-sqip.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/gatsby-transformer-sqip/src/__tests__/generate-sqip.js b/packages/gatsby-transformer-sqip/src/__tests__/generate-sqip.js index 4e6ad70a4b995..179e011fa843f 100644 --- a/packages/gatsby-transformer-sqip/src/__tests__/generate-sqip.js +++ b/packages/gatsby-transformer-sqip/src/__tests__/generate-sqip.js @@ -5,8 +5,6 @@ const sqip = require(`sqip`) const generateSqip = require(`../generate-sqip.js`) -const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) - jest.mock(`sqip`, () => jest.fn(() => { return { @@ -54,7 +52,6 @@ describe(`gatsby-transformer-sqip`, async () => { numberOfPrimitives, blur, mode, - createContentDigest, }) expect(result).toMatchSnapshot() @@ -84,7 +81,6 @@ describe(`gatsby-transformer-sqip`, async () => { numberOfPrimitives, blur, mode, - createContentDigest, }) expect(result).toMatchSnapshot() From 26fa18abc6cd66bf89b73cbc8b54df80767cb7b1 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 31 Dec 2018 16:01:17 +0100 Subject: [PATCH 27/63] use caret version selector --- packages/gatsby-source-filesystem/package.json | 2 +- packages/gatsby-source-hacker-news/package.json | 2 +- packages/gatsby-source-lever/package.json | 2 +- packages/gatsby-source-mongodb/package.json | 2 +- packages/gatsby-source-npm-package-search/package.json | 2 +- packages/gatsby-transformer-excel/package.json | 2 +- packages/gatsby-transformer-hjson/package.json | 2 +- packages/gatsby-transformer-sqip/package.json | 2 +- packages/gatsby-transformer-toml/package.json | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json index 10342e8c6b88f..ea256fa614a10 100644 --- a/packages/gatsby-source-filesystem/package.json +++ b/packages/gatsby-source-filesystem/package.json @@ -35,7 +35,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.15" + "gatsby": "^2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem", "scripts": { diff --git a/packages/gatsby-source-hacker-news/package.json b/packages/gatsby-source-hacker-news/package.json index 44c07a2320a12..91312c35c0a40 100644 --- a/packages/gatsby-source-hacker-news/package.json +++ b/packages/gatsby-source-hacker-news/package.json @@ -25,7 +25,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.15" + "gatsby": "^2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-hacker-news", "scripts": { diff --git a/packages/gatsby-source-lever/package.json b/packages/gatsby-source-lever/package.json index 86f9ff4f3d2ce..511ed7e60eef4 100644 --- a/packages/gatsby-source-lever/package.json +++ b/packages/gatsby-source-lever/package.json @@ -31,7 +31,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.15" + "gatsby": "^2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-lever", "scripts": { diff --git a/packages/gatsby-source-mongodb/package.json b/packages/gatsby-source-mongodb/package.json index 7e4723aa1fa79..5baee38ca39b4 100644 --- a/packages/gatsby-source-mongodb/package.json +++ b/packages/gatsby-source-mongodb/package.json @@ -26,7 +26,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { - "gatsby": ">2.0.15" + "gatsby": "^2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-mongodb", "scripts": { diff --git a/packages/gatsby-source-npm-package-search/package.json b/packages/gatsby-source-npm-package-search/package.json index 254c30db0ac47..d84cd69dc51bf 100644 --- a/packages/gatsby-source-npm-package-search/package.json +++ b/packages/gatsby-source-npm-package-search/package.json @@ -20,7 +20,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { - "gatsby": ">2.0.15" + "gatsby": "^2.0.15" }, "scripts": { "build": "babel src --out-dir . --ignore **/__tests__", diff --git a/packages/gatsby-transformer-excel/package.json b/packages/gatsby-transformer-excel/package.json index cc868334ea8b6..0c00b7a3f9626 100644 --- a/packages/gatsby-transformer-excel/package.json +++ b/packages/gatsby-transformer-excel/package.json @@ -25,7 +25,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { - "gatsby": ">2.0.15" + "gatsby": "^2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-excel", "scripts": { diff --git a/packages/gatsby-transformer-hjson/package.json b/packages/gatsby-transformer-hjson/package.json index 58b954ce1092c..2054fb5be55a5 100644 --- a/packages/gatsby-transformer-hjson/package.json +++ b/packages/gatsby-transformer-hjson/package.json @@ -25,7 +25,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.15" + "gatsby": "^2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-hjson", "scripts": { diff --git a/packages/gatsby-transformer-sqip/package.json b/packages/gatsby-transformer-sqip/package.json index e5f2a42d8a5ec..92709d0c0f2a6 100644 --- a/packages/gatsby-transformer-sqip/package.json +++ b/packages/gatsby-transformer-sqip/package.json @@ -23,7 +23,7 @@ "debug": "^3.1.0" }, "peerDependencies": { - "gatsby": ">2.0.15", + "gatsby": "^2.0.15", "gatsby-source-contentful": ">2.0.0-alpha", "gatsby-transformer-sharp": ">2.0.0-alpha" }, diff --git a/packages/gatsby-transformer-toml/package.json b/packages/gatsby-transformer-toml/package.json index fc2f8b0291a96..5a28478e0e7b0 100644 --- a/packages/gatsby-transformer-toml/package.json +++ b/packages/gatsby-transformer-toml/package.json @@ -25,7 +25,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": ">2.0.15" + "gatsby": "^2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-toml", "scripts": { From 10d995a3ff8e8f63e7310b6ff91d0fba5163f05c Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 31 Dec 2018 16:10:39 +0100 Subject: [PATCH 28/63] no need to import it - it's passed to `sourceNodes` --- .../src/internal-plugins/internal-data-bridge/gatsby-node.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js index 3ab73ca4c6d7d..e7369142def92 100644 --- a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js +++ b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js @@ -6,7 +6,6 @@ const _ = require(`lodash`) const { emitter } = require(`../../redux`) const { boundActionCreators } = require(`../../redux/actions`) const { getNode } = require(`../../db/nodes`) -const createContentDigest = require(`../../utils/create-content-digest`) function transformPackageJson(json) { const transformDeps = deps => From 595e5ceb12f0c7329eb024465ad707715ff35900 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Fri, 1 Mar 2019 21:41:02 -0600 Subject: [PATCH 29/63] merge again --- packages/gatsby-source-mongodb/package.json | 4 ---- packages/gatsby-source-mongodb/src/mapping.js | 14 -------------- .../gatsby-source-npm-package-search/package.json | 4 ---- packages/gatsby-transformer-excel/package.json | 4 ---- packages/gatsby-transformer-hjson/package.json | 4 ---- .../src/on-node-create.js | 12 ------------ packages/gatsby-transformer-sqip/package.json | 6 ------ packages/gatsby-transformer-toml/package.json | 4 ---- 8 files changed, 52 deletions(-) diff --git a/packages/gatsby-source-mongodb/package.json b/packages/gatsby-source-mongodb/package.json index d05efbc0fecb2..768218f93af29 100644 --- a/packages/gatsby-source-mongodb/package.json +++ b/packages/gatsby-source-mongodb/package.json @@ -27,11 +27,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { -<<<<<<< HEAD - "gatsby": "^2.0.15" -======= "gatsby": "^2.0.0" ->>>>>>> 888c124d24aea36a57086d096154ecacf44f0980 }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-mongodb", "scripts": { diff --git a/packages/gatsby-source-mongodb/src/mapping.js b/packages/gatsby-source-mongodb/src/mapping.js index 97b744a83936c..6643615258d68 100644 --- a/packages/gatsby-source-mongodb/src/mapping.js +++ b/packages/gatsby-source-mongodb/src/mapping.js @@ -1,23 +1,9 @@ -<<<<<<< HEAD -const _ = require(`lodash`) - -module.exports = function( - node, - key, - text, - mediaType, - createNode, - createContentDigest -) { - const str = _.isString(text) ? text : ` ` -======= const camelCase = require(`lodash.camelcase`) const isString = require(`lodash.isstring`) const crypto = require(`crypto`) module.exports = function(node, key, text, mediaType, createNode) { const str = isString(text) ? text : ` ` ->>>>>>> 888c124d24aea36a57086d096154ecacf44f0980 const id = `${node.id}${key}MappingNode` const mappingNode = { id: id, diff --git a/packages/gatsby-source-npm-package-search/package.json b/packages/gatsby-source-npm-package-search/package.json index 3eb807b28c843..085f03c0c5a83 100644 --- a/packages/gatsby-source-npm-package-search/package.json +++ b/packages/gatsby-source-npm-package-search/package.json @@ -20,11 +20,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { -<<<<<<< HEAD - "gatsby": "^2.0.15" -======= "gatsby": "^2.0.0" ->>>>>>> 888c124d24aea36a57086d096154ecacf44f0980 }, "scripts": { "build": "babel src --out-dir . --ignore **/__tests__", diff --git a/packages/gatsby-transformer-excel/package.json b/packages/gatsby-transformer-excel/package.json index ea18e13245393..bdd04365bf523 100644 --- a/packages/gatsby-transformer-excel/package.json +++ b/packages/gatsby-transformer-excel/package.json @@ -25,11 +25,7 @@ "license": "MIT", "main": "index.js", "peerDependencies": { -<<<<<<< HEAD - "gatsby": "^2.0.15" -======= "gatsby": "^2.0.0" ->>>>>>> 888c124d24aea36a57086d096154ecacf44f0980 }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-excel", "scripts": { diff --git a/packages/gatsby-transformer-hjson/package.json b/packages/gatsby-transformer-hjson/package.json index 6cb3887520dce..c3acfe29cb801 100644 --- a/packages/gatsby-transformer-hjson/package.json +++ b/packages/gatsby-transformer-hjson/package.json @@ -25,11 +25,7 @@ ], "license": "MIT", "peerDependencies": { -<<<<<<< HEAD - "gatsby": "^2.0.15" -======= "gatsby": "^2.0.0" ->>>>>>> 888c124d24aea36a57086d096154ecacf44f0980 }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-hjson", "scripts": { diff --git a/packages/gatsby-transformer-remark/src/on-node-create.js b/packages/gatsby-transformer-remark/src/on-node-create.js index 1842009e156e0..57a2ba7eda587 100644 --- a/packages/gatsby-transformer-remark/src/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/on-node-create.js @@ -2,19 +2,7 @@ const grayMatter = require(`gray-matter`) const _ = require(`lodash`) module.exports = async function onCreateNode( -<<<<<<< HEAD - { - node, - getNode, - loadNodeContent, - actions, - createNodeId, - createContentDigest, - reporter, - }, -======= { node, loadNodeContent, actions, createNodeId, reporter }, ->>>>>>> 888c124d24aea36a57086d096154ecacf44f0980 pluginOptions ) { const { createNode, createParentChildLink } = actions diff --git a/packages/gatsby-transformer-sqip/package.json b/packages/gatsby-transformer-sqip/package.json index 97d5dc27bac81..c1501dfcf267e 100644 --- a/packages/gatsby-transformer-sqip/package.json +++ b/packages/gatsby-transformer-sqip/package.json @@ -23,15 +23,9 @@ "debug": "^3.1.0" }, "peerDependencies": { -<<<<<<< HEAD - "gatsby": "^2.0.15", - "gatsby-source-contentful": ">2.0.0-alpha", - "gatsby-transformer-sharp": ">2.0.0-alpha" -======= "gatsby": "^2.0.0", "gatsby-source-contentful": "^2.0.0", "gatsby-transformer-sharp": "^2.0.0" ->>>>>>> 888c124d24aea36a57086d096154ecacf44f0980 }, "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-sqip#readme", "keywords": [ diff --git a/packages/gatsby-transformer-toml/package.json b/packages/gatsby-transformer-toml/package.json index fbf51d44bcea6..0d3cb681a02cd 100644 --- a/packages/gatsby-transformer-toml/package.json +++ b/packages/gatsby-transformer-toml/package.json @@ -25,11 +25,7 @@ ], "license": "MIT", "peerDependencies": { -<<<<<<< HEAD - "gatsby": "^2.0.15" -======= "gatsby": "^2.0.0" ->>>>>>> 888c124d24aea36a57086d096154ecacf44f0980 }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-transformer-toml", "scripts": { From 93ee035c0f83fa6b9245af69527578d099bb01ee Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Fri, 1 Mar 2019 22:13:01 -0600 Subject: [PATCH 30/63] update to pass tests --- packages/gatsby-source-contentful/src/normalize.js | 1 + .../src/create-remote-file-node.js | 3 ++- packages/gatsby-source-mongodb/src/mapping.js | 10 ++++++++-- .../gatsby-transformer-remark/src/on-node-create.js | 1 + packages/gatsby/src/bootstrap/index.js | 1 + 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 8e108dbfa7cff..7e67e37cc6b3f 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -1,6 +1,7 @@ const _ = require(`lodash`) const stringify = require(`json-stringify-safe`) const deepMap = require(`deep-map`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const typePrefix = `Contentful` const makeTypeName = type => _.upperFirst(_.camelCase(`${typePrefix} ${type}`)) diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index 0c6c3e182e93d..4b7addfa13510 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -173,6 +173,7 @@ async function processRemoteNode({ createNodeId, ext, name, + createContentDigest, }) { // Ensure our cache directory exists. const pluginCacheDir = path.join( @@ -198,7 +199,7 @@ async function processRemoteNode({ } // Create the temp and permanent file names for the url. - const digest = createHash(url) + const digest = createContentDigest(url) if (!name) { name = getRemoteFileName(url) } diff --git a/packages/gatsby-source-mongodb/src/mapping.js b/packages/gatsby-source-mongodb/src/mapping.js index 6643615258d68..d7f05b3310e3d 100644 --- a/packages/gatsby-source-mongodb/src/mapping.js +++ b/packages/gatsby-source-mongodb/src/mapping.js @@ -1,8 +1,14 @@ const camelCase = require(`lodash.camelcase`) const isString = require(`lodash.isstring`) -const crypto = require(`crypto`) -module.exports = function(node, key, text, mediaType, createNode) { +module.exports = function( + node, + key, + text, + mediaType, + createNode, + createContentDigest +) { const str = isString(text) ? text : ` ` const id = `${node.id}${key}MappingNode` const mappingNode = { diff --git a/packages/gatsby-transformer-remark/src/on-node-create.js b/packages/gatsby-transformer-remark/src/on-node-create.js index 57a2ba7eda587..959b670276313 100644 --- a/packages/gatsby-transformer-remark/src/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/on-node-create.js @@ -1,3 +1,4 @@ +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const grayMatter = require(`gray-matter`) const _ = require(`lodash`) diff --git a/packages/gatsby/src/bootstrap/index.js b/packages/gatsby/src/bootstrap/index.js index b651835aa8f7f..cac0443f2735a 100644 --- a/packages/gatsby/src/bootstrap/index.js +++ b/packages/gatsby/src/bootstrap/index.js @@ -21,6 +21,7 @@ const getConfigFile = require(`./get-config-file`) const tracer = require(`opentracing`).globalTracer() const preferDefault = require(`./prefer-default`) const nodeTracking = require(`../db/node-tracking`) +const createContentDigest = require(`../../src/utils/create-content-digest`) require(`../db`).startAutosave() // Show stack trace on unhandled promises. From 6fd385098ac6395cc9af7f0e81bcebcc5406972b Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Wed, 6 Mar 2019 21:10:51 -0600 Subject: [PATCH 31/63] update hash to use createContentDigest --- packages/gatsby-plugin-sharp/src/index.js | 12 +++--------- .../gatsby-remark-images-contentful/src/index.js | 7 ++----- packages/gatsby-source-contentful/src/cache-image.js | 8 ++------ packages/gatsby-source-drupal/src/gatsby-node.js | 9 +-------- packages/gatsby-source-graphql/src/gatsby-node.js | 7 ++----- packages/gatsby-source-medium/src/gatsby-node.js | 7 ++----- packages/gatsby-source-wikipedia/src/gatsby-node.js | 7 +++---- packages/gatsby-source-wordpress/src/normalize.js | 8 ++------ .../src/gatsby-node.js | 8 ++------ .../src/gatsby-node.js | 7 ++----- .../src/gatsby-node.js | 7 ++----- .../src/on-node-create.js | 8 ++------ .../gatsby-transformer-screenshot/lambda/index.js | 7 ++----- .../gatsby-transformer-sqip/src/generate-sqip.js | 7 ++----- packages/gatsby/src/bootstrap/index.js | 6 +----- .../internal-plugins/query-runner/pages-writer.js | 7 ++----- .../internal-plugins/query-runner/query-runner.js | 5 +---- 17 files changed, 33 insertions(+), 94 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index 48f801677b433..70201af099216 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -1,5 +1,5 @@ const sharp = require(`sharp`) -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const imageSize = require(`probe-image-size`) const _ = require(`lodash`) const Promise = require(`bluebird`) @@ -403,10 +403,7 @@ function queueImageResizing({ file, args = {}, reporter }) { const sortedArgs = _.sortBy(filteredArgs, arg => arg[0] === `width`) const fileExtension = options.toFormat ? options.toFormat : file.extension - const argsDigest = crypto - .createHash(`md5`) - .update(JSON.stringify(sortedArgs)) - .digest(`hex`) + const argsDigest = createContentDigest(JSON.stringify(sortedArgs)) const argsDigestShort = argsDigest.substr(argsDigest.length - 5) @@ -928,10 +925,7 @@ async function notMemoizedtraceSVG({ file, args, fileArgs, reporter }) { const tmpDir = require(`os`).tmpdir() const tmpFilePath = `${tmpDir}/${file.internal.contentDigest}-${ file.name - }-${crypto - .createHash(`md5`) - .update(JSON.stringify(fileArgs)) - .digest(`hex`)}.${file.extension}` + }-${createContentDigest(JSON.stringify(fileArgs))}.${file.extension}` await new Promise(resolve => pipeline.toFile(tmpFilePath, (err, info) => { diff --git a/packages/gatsby-remark-images-contentful/src/index.js b/packages/gatsby-remark-images-contentful/src/index.js index 3eb9374c73cb8..8a0eace211b34 100644 --- a/packages/gatsby-remark-images-contentful/src/index.js +++ b/packages/gatsby-remark-images-contentful/src/index.js @@ -1,4 +1,3 @@ -const crypto = require(`crypto`) const select = require(`unist-util-select`) const sharp = require(`sharp`) const axios = require(`axios`) @@ -6,6 +5,7 @@ const _ = require(`lodash`) const Promise = require(`bluebird`) const cheerio = require(`cheerio`) const { buildResponsiveSizes } = require(`./utils`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) // If the image is hosted on contentful // 1. Find the image file @@ -49,10 +49,7 @@ module.exports = async ( const fileName = srcSplit[srcSplit.length - 1] const options = _.defaults(pluginOptions, defaults) - const optionsHash = crypto - .createHash(`md5`) - .update(JSON.stringify(options)) - .digest(`hex`) + const optionsHash = createContentDigest(options) const cacheKey = `remark-images-ctf-${fileName}-${optionsHash}` let cahedRawHTML = await cache.get(cacheKey) diff --git a/packages/gatsby-source-contentful/src/cache-image.js b/packages/gatsby-source-contentful/src/cache-image.js index 7affe01c3380e..5275ab2090989 100644 --- a/packages/gatsby-source-contentful/src/cache-image.js +++ b/packages/gatsby-source-contentful/src/cache-image.js @@ -1,4 +1,4 @@ -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const { resolve, parse } = require(`path`) const axios = require(`axios`) @@ -37,11 +37,7 @@ module.exports = async function cacheImage(store, image, options) { params.push(`bg=${background}`) } - // TODO convert to `createContentDigest` using dependency import - const optionsHash = crypto - .createHash(`md5`) - .update(JSON.stringify([url, ...params])) - .digest(`hex`) + const optionsHash = createContentDigest(JSON.stringify([url, ...params])) const { name, ext } = parse(fileName) const absolutePath = resolve(CACHE_DIR, `${name}-${optionsHash}${ext}`) diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index c8971b84f7d09..e9a228a95f367 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -1,17 +1,10 @@ const axios = require(`axios`) -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const _ = require(`lodash`) const { createRemoteFileNode } = require(`gatsby-source-filesystem`) const { URL } = require(`url`) const { nodeFromData } = require(`./normalize`) -// Get content digest of node. -const createContentDigest = obj => - crypto - .createHash(`md5`) - .update(JSON.stringify(obj)) - .digest(`hex`) - exports.sourceNodes = async ( { actions, getNode, hasNodeChanged, store, cache, createNodeId }, { baseUrl, apiBase, basicAuth } diff --git a/packages/gatsby-source-graphql/src/gatsby-node.js b/packages/gatsby-source-graphql/src/gatsby-node.js index 904f672be7577..27bf397d9d1dd 100644 --- a/packages/gatsby-source-graphql/src/gatsby-node.js +++ b/packages/gatsby-source-graphql/src/gatsby-node.js @@ -1,4 +1,4 @@ -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const uuidv4 = require(`uuid/v4`) const { buildSchema, printSchema } = require(`graphql`) const { @@ -114,10 +114,7 @@ exports.sourceNodes = async ( function createSchemaNode({ id, typeName, fieldName }) { const nodeContent = uuidv4() - const nodeContentDigest = crypto - .createHash(`md5`) - .update(nodeContent) - .digest(`hex`) + const nodeContentDigest = createContentDigest(nodeContent) return { id, typeName: typeName, diff --git a/packages/gatsby-source-medium/src/gatsby-node.js b/packages/gatsby-source-medium/src/gatsby-node.js index 397437ad07b04..38b893754e4b0 100644 --- a/packages/gatsby-source-medium/src/gatsby-node.js +++ b/packages/gatsby-source-medium/src/gatsby-node.js @@ -1,5 +1,5 @@ const axios = require(`axios`) -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const fetch = (username, limit = 100) => { const url = `https://medium.com/${username}/latest?format=json&limit=${limit}` @@ -77,10 +77,7 @@ exports.sourceNodes = async ( resources.map(resource => { convertTimestamps(resource) - const digest = crypto - .createHash(`md5`) - .update(JSON.stringify(resource)) - .digest(`hex`) + const digest = createContentDigest(resource) const links = resource.type === `Post` diff --git a/packages/gatsby-source-wikipedia/src/gatsby-node.js b/packages/gatsby-source-wikipedia/src/gatsby-node.js index bfb311ceaef65..6da957720d7d0 100644 --- a/packages/gatsby-source-wikipedia/src/gatsby-node.js +++ b/packages/gatsby-source-wikipedia/src/gatsby-node.js @@ -17,10 +17,9 @@ exports.sourceNodes = ({ actions, createNodeId }, pluginOptions) => { type: `WikipediaArticle`, }, } - node.internal.contentDigest = require(`crypto`) - .createHash(`md5`) - .update(JSON.stringify(node)) - .digest(`hex`) + node.internal.contentDigest = require(`../../gatsby/src/utils/create-content-digest`)( + node + ) createNode(node) }) ) diff --git a/packages/gatsby-source-wordpress/src/normalize.js b/packages/gatsby-source-wordpress/src/normalize.js index 15210eaae50d1..3e9f124dfae28 100644 --- a/packages/gatsby-source-wordpress/src/normalize.js +++ b/packages/gatsby-source-wordpress/src/normalize.js @@ -1,4 +1,4 @@ -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const deepMapKeys = require(`deep-map-keys`) const _ = require(`lodash`) const { createRemoteFileNode } = require(`gatsby-source-filesystem`) @@ -14,11 +14,7 @@ const restrictedNodeFields = [`id`, `children`, `parent`, `fields`, `internal`] * * @param {any} str */ -const digest = str => - crypto - .createHash(`md5`) - .update(str) - .digest(`hex`) +const digest = str => createContentDigest(str) /** * Validate the GraphQL naming convetions & protect specific fields. diff --git a/packages/gatsby-transformer-documentationjs/src/gatsby-node.js b/packages/gatsby-transformer-documentationjs/src/gatsby-node.js index 18bb7f8c01f4b..c93a57e17d3b4 100644 --- a/packages/gatsby-transformer-documentationjs/src/gatsby-node.js +++ b/packages/gatsby-transformer-documentationjs/src/gatsby-node.js @@ -1,10 +1,6 @@ const documentation = require(`documentation`) -const crypto = require(`crypto`) -const digest = str => - crypto - .createHash(`md5`) - .update(str) - .digest(`hex`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const digest = str => createContentDigest(str) const remark = require(`remark`) const _ = require(`lodash`) const Prism = require(`prismjs`) diff --git a/packages/gatsby-transformer-javascript-frontmatter/src/gatsby-node.js b/packages/gatsby-transformer-javascript-frontmatter/src/gatsby-node.js index cb762e81aaa4a..9921ab96c9d65 100644 --- a/packages/gatsby-transformer-javascript-frontmatter/src/gatsby-node.js +++ b/packages/gatsby-transformer-javascript-frontmatter/src/gatsby-node.js @@ -1,5 +1,5 @@ const _ = require(`lodash`) -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const babylon = require(`@babel/parser`) const traverse = require(`@babel/traverse`).default @@ -108,10 +108,7 @@ async function onCreateNode({ node, getNode, actions, loadNodeContent }) { } const objStr = JSON.stringify(node) - const contentDigest = crypto - .createHash(`md5`) - .update(objStr) - .digest(`hex`) + const contentDigest = createContentDigest(objStr) const nodeData = { id: `${node.id} >>> JavascriptFrontmatter`, diff --git a/packages/gatsby-transformer-javascript-static-exports/src/gatsby-node.js b/packages/gatsby-transformer-javascript-static-exports/src/gatsby-node.js index 41532df6b4a7d..f890c4a74340a 100644 --- a/packages/gatsby-transformer-javascript-static-exports/src/gatsby-node.js +++ b/packages/gatsby-transformer-javascript-static-exports/src/gatsby-node.js @@ -1,5 +1,5 @@ const _ = require(`lodash`) -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const babylon = require(`@babel/parser`) const traverse = require(`@babel/traverse`).default @@ -127,10 +127,7 @@ async function onCreateNode({ } } finally { const objStr = JSON.stringify(node) - const contentDigest = crypto - .createHash(`md5`) - .update(objStr) - .digest(`hex`) + const contentDigest = createContentDigest(objStr) const nodeData = { id: createNodeId(`${node.id} >>> JSFrontmatter`), diff --git a/packages/gatsby-transformer-react-docgen/src/on-node-create.js b/packages/gatsby-transformer-react-docgen/src/on-node-create.js index 782b5f581bc74..3236ba2d40087 100644 --- a/packages/gatsby-transformer-react-docgen/src/on-node-create.js +++ b/packages/gatsby-transformer-react-docgen/src/on-node-create.js @@ -1,11 +1,7 @@ -import crypto from "crypto" +import createContentDigest from "../../gatsby/src/utils/create-content-digest" import parseMetadata from "./parse" -const digest = str => - crypto - .createHash(`md5`) - .update(str) - .digest(`hex`) +const digest = str => createContentDigest(str) const propsId = (parentId, name) => `${parentId}--ComponentProp-${name}` const descId = parentId => `${parentId}--ComponentDescription` diff --git a/packages/gatsby-transformer-screenshot/lambda/index.js b/packages/gatsby-transformer-screenshot/lambda/index.js index 3bb3a76d1b092..15c74c93ec822 100644 --- a/packages/gatsby-transformer-screenshot/lambda/index.js +++ b/packages/gatsby-transformer-screenshot/lambda/index.js @@ -1,6 +1,6 @@ const setup = require(`./starter-kit/setup`) -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const AWS = require(`aws-sdk`) const s3 = new AWS.S3({ @@ -56,10 +56,7 @@ exports.run = async (browser, url, width, height, fullPage) => { } const keyBase = `${url}-(${width},${height})` - const digest = crypto - .createHash(`md5`) - .update(keyBase) - .digest(`hex`) + const digest = createContentDigest(keyBase) const key = `${digest}.png` const screenshotUrl = `https://s3-${region}.amazonaws.com/${ diff --git a/packages/gatsby-transformer-sqip/src/generate-sqip.js b/packages/gatsby-transformer-sqip/src/generate-sqip.js index 2149cd39ccc47..aef073fbb67e5 100644 --- a/packages/gatsby-transformer-sqip/src/generate-sqip.js +++ b/packages/gatsby-transformer-sqip/src/generate-sqip.js @@ -1,4 +1,4 @@ -const crypto = require(`crypto`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const { resolve, parse } = require(`path`) const Debug = require(`debug`) @@ -30,10 +30,7 @@ module.exports = async function generateSqip(options) { mode, } - const optionsHash = crypto - .createHash(`md5`) - .update(JSON.stringify(sqipOptions)) - .digest(`hex`) + const optionsHash = createContentDigest(JSON.stringify(sqipOptions)) const cacheKey = `sqip-${name}-${optionsHash}` const cachePath = resolve(cacheDir, `${name}-${optionsHash}.svg`) diff --git a/packages/gatsby/src/bootstrap/index.js b/packages/gatsby/src/bootstrap/index.js index 52a5e739bcd8f..d63fbc00ad17b 100644 --- a/packages/gatsby/src/bootstrap/index.js +++ b/packages/gatsby/src/bootstrap/index.js @@ -4,7 +4,6 @@ const _ = require(`lodash`) const slash = require(`slash`) const fs = require(`fs-extra`) const md5File = require(`md5-file/promise`) -const crypto = require(`crypto`) const del = require(`del`) const path = require(`path`) const convertHrtime = require(`convert-hrtime`) @@ -159,10 +158,7 @@ module.exports = async (args: BootstrapArgs) => { md5File(`${program.directory}/gatsby-node.js`).catch(() => {}) ), // ignore as this file isn't required), ]) - const pluginsHash = crypto - .createHash(`md5`) - .update(JSON.stringify(pluginVersions.concat(hashes))) - .digest(`hex`) + const pluginsHash = createContentDigest(pluginVersions.concat(hashes)) let state = store.getState() const oldPluginsHash = state && state.status ? state.status.PLUGINS_HASH : `` diff --git a/packages/gatsby/src/internal-plugins/query-runner/pages-writer.js b/packages/gatsby/src/internal-plugins/query-runner/pages-writer.js index 10e8c21e76bd9..3412ef84b3bb6 100644 --- a/packages/gatsby/src/internal-plugins/query-runner/pages-writer.js +++ b/packages/gatsby/src/internal-plugins/query-runner/pages-writer.js @@ -1,6 +1,6 @@ const _ = require(`lodash`) const fs = require(`fs-extra`) -const crypto = require(`crypto`) +const createContentDigest = require(`../../utils/create-content-digest`) const { store, emitter } = require(`../../redux/`) @@ -48,10 +48,7 @@ const writePages = async () => { }` ) .value() - const newHash = crypto - .createHash(`md5`) - .update(JSON.stringify(pagesComponentDependencies)) - .digest(`hex`) + const newHash = createContentDigest(pagesComponentDependencies) if (newHash === lastHash) { // components didn't change - no need to rewrite pages.json diff --git a/packages/gatsby/src/internal-plugins/query-runner/query-runner.js b/packages/gatsby/src/internal-plugins/query-runner/query-runner.js index 1e3197efc43f4..d86bfdb2a3aba 100644 --- a/packages/gatsby/src/internal-plugins/query-runner/query-runner.js +++ b/packages/gatsby/src/internal-plugins/query-runner/query-runner.js @@ -80,10 +80,7 @@ ${formatErrorDetails(errorDetails)}`) } const resultJSON = JSON.stringify(result) - const resultHash = require(`crypto`) - .createHash(`sha1`) - .update(resultJSON) - .digest(`base64`) + const resultHash = require(`../../utils/create-content-digest`)(resultJSON) // Remove potentially unsafe characters. This increases chances of collisions // slightly but it should still be very safe + we get a shorter // url vs hex. From 4360e587fa4cd2b4bcc2d51fa12e3432515963dd Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 20:09:02 -0600 Subject: [PATCH 32/63] move createContentDigest to import --- .../gatsby-source-filesystem/src/create-file-node.js | 2 +- .../src/create-remote-file-node.js | 2 +- packages/gatsby-source-filesystem/src/gatsby-node.js | 11 +---------- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/gatsby-source-filesystem/src/create-file-node.js b/packages/gatsby-source-filesystem/src/create-file-node.js index 7ed975472d8e3..702e45e08bacb 100644 --- a/packages/gatsby-source-filesystem/src/create-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-file-node.js @@ -4,12 +4,12 @@ const fs = require(`fs-extra`) const mime = require(`mime`) const prettyBytes = require(`pretty-bytes`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const md5File = require(`bluebird`).promisify(require(`md5-file`)) exports.createFileNode = async ( pathToFile, createNodeId, - createContentDigest, pluginOptions = {} ) => { const slashed = slash(pathToFile) diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index 4b7addfa13510..624bca82ad261 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -1,5 +1,6 @@ const fs = require(`fs-extra`) const got = require(`got`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const path = require(`path`) const { isWebUri } = require(`valid-url`) const Queue = require(`better-queue`) @@ -173,7 +174,6 @@ async function processRemoteNode({ createNodeId, ext, name, - createContentDigest, }) { // Ensure our cache directory exists. const pluginCacheDir = path.join( diff --git a/packages/gatsby-source-filesystem/src/gatsby-node.js b/packages/gatsby-source-filesystem/src/gatsby-node.js index 6eb8d237d29ea..b74d3a4cab6c8 100644 --- a/packages/gatsby-source-filesystem/src/gatsby-node.js +++ b/packages/gatsby-source-filesystem/src/gatsby-node.js @@ -40,15 +40,7 @@ const createFSMachine = () => }) exports.sourceNodes = ( - { - actions, - getNode, - createNodeId, - createContentDigest, - hasNodeChanged, - reporter, - emitter, - }, + { actions, getNode, createNodeId, hasNodeChanged, reporter, emitter }, pluginOptions ) => { const { createNode, deleteNode } = actions @@ -103,7 +95,6 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/ const fileNodePromise = createFileNode( path, createNodeId, - createContentDigest, pluginOptions ).then(fileNode => { createNode(fileNode) From f5a5583707888d27693aab967e129be2ead69b5a Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 20:20:26 -0600 Subject: [PATCH 33/63] move createContentDigest to import for gatsby-source-contentful --- .../src/__tests__/normalize.js | 4 ---- .../src/cache-image.js | 2 +- .../src/gatsby-node.js | 12 +--------- .../gatsby-source-contentful/src/normalize.js | 23 ++++--------------- 4 files changed, 7 insertions(+), 34 deletions(-) diff --git a/packages/gatsby-source-contentful/src/__tests__/normalize.js b/packages/gatsby-source-contentful/src/__tests__/normalize.js index feb339af2d092..97067311f02b0 100644 --- a/packages/gatsby-source-contentful/src/__tests__/normalize.js +++ b/packages/gatsby-source-contentful/src/__tests__/normalize.js @@ -6,8 +6,6 @@ const { locales, } = require(`./data.json`) -const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) - let entryList let resolvable let foreignReferenceMap @@ -64,7 +62,6 @@ describe(`Process contentful data`, () => { entries: entryList[i].map(normalize.fixIds), createNode, createNodeId, - createContentDigest, resolvable, foreignReferenceMap, defaultLocale, @@ -84,7 +81,6 @@ describe(`Process contentful data`, () => { assetItem, createNode, createNodeId, - createContentDigest, defaultLocale, locales, }) diff --git a/packages/gatsby-source-contentful/src/cache-image.js b/packages/gatsby-source-contentful/src/cache-image.js index 5275ab2090989..d415c6931273a 100644 --- a/packages/gatsby-source-contentful/src/cache-image.js +++ b/packages/gatsby-source-contentful/src/cache-image.js @@ -37,7 +37,7 @@ module.exports = async function cacheImage(store, image, options) { params.push(`bg=${background}`) } - const optionsHash = createContentDigest(JSON.stringify([url, ...params])) + const optionsHash = createContentDigest([url, ...params]) const { name, ext } = parse(fileName) const absolutePath = resolve(CACHE_DIR, `${name}-${optionsHash}${ext}`) diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index f281722374bae..ee68091e34669 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -32,15 +32,7 @@ exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeTyp */ exports.sourceNodes = async ( - { - actions, - getNode, - getNodes, - createNodeId, - createContentDigest, - hasNodeChanged, - store, - }, + { actions, getNode, getNodes, createNodeId, hasNodeChanged, store }, options ) => { const { createNode, deleteNode, touchNode, setPluginStatus } = actions @@ -200,7 +192,6 @@ exports.sourceNodes = async ( entries: entryList[i], createNode, createNodeId, - createContentDigest, resolvable, foreignReferenceMap, defaultLocale, @@ -213,7 +204,6 @@ exports.sourceNodes = async ( assetItem, createNode, createNodeId, - createContentDigest, defaultLocale, locales, }) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 7e67e37cc6b3f..8aabe3f49a1cc 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -168,7 +168,7 @@ exports.buildForeignReferenceMap = ({ return foreignReferenceMap } -function prepareTextNode(node, key, text, createNodeId, createContentDigest) { +function prepareTextNode(node, key, text, createNodeId) { const str = _.isString(text) ? text : ` ` const textNode = { id: createNodeId(`${node.id}${key}TextNode`), @@ -209,14 +209,7 @@ function prepareRichTextNode(node, key, content, createNodeId) { return richTextNode } -function prepareJSONNode( - node, - key, - content, - createNodeId, - createContentDigest, - i = `` -) { +function prepareJSONNode(node, key, content, createNodeId, i = ``) { const str = JSON.stringify(content) const JSONNode = { ...(_.isPlainObject(content) ? { ...content } : { content: content }), @@ -243,7 +236,6 @@ exports.createContentTypeNodes = ({ entries, createNode, createNodeId, - createContentDigest, resolvable, foreignReferenceMap, defaultLocale, @@ -398,8 +390,7 @@ exports.createContentTypeNodes = ({ entryNode, entryItemFieldKey, entryItemFields[entryItemFieldKey], - createNodeId, - createContentDigest + createNodeId ) childrenNodes.push(textNode) @@ -414,8 +405,7 @@ exports.createContentTypeNodes = ({ entryNode, entryItemFieldKey, entryItemFields[entryItemFieldKey], - createNodeId, - createContentDigest + createNodeId ) childrenNodes.push(richTextNode) @@ -430,8 +420,7 @@ exports.createContentTypeNodes = ({ entryNode, entryItemFieldKey, entryItemFields[entryItemFieldKey], - createNodeId, - createContentDigest + createNodeId ) childrenNodes.push(jsonNode) @@ -450,7 +439,6 @@ exports.createContentTypeNodes = ({ entryItemFieldKey, obj, createNodeId, - createContentDigest, i ) @@ -504,7 +492,6 @@ exports.createAssetNodes = ({ assetItem, createNode, createNodeId, - createContentDigest, defaultLocale, locales, }) => { From 942a1741a98faa1a393470133d1d31bf3642998b Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 20:39:53 -0600 Subject: [PATCH 34/63] remove createContentDigest import in gatsby-source-filesystem test --- .../gatsby-source-filesystem/src/__tests__/create-file-node.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js b/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js index 701df26ebf8f8..3224b1ad833c4 100644 --- a/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js +++ b/packages/gatsby-source-filesystem/src/__tests__/create-file-node.js @@ -1,7 +1,6 @@ const path = require(`path`) const { createFileNode } = require(`../create-file-node`) -const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) // FIXME: This test needs to not use snapshots because of file differences // and locations across users and CI systems @@ -12,7 +11,6 @@ describe(`create-file-node`, () => { return createFileNode( path.resolve(`${__dirname}/fixtures/file.json`), createNodeId, - createContentDigest, {} ) }) From 58a5ef1eebe615606c7b5d29db8b882ab44bf3d8 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 20:57:35 -0600 Subject: [PATCH 35/63] rereplace createContentDigest with crypto in gatsby-source-filesystem --- .../src/create-remote-file-node.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index 624bca82ad261..7f6d1cf09ae96 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -1,6 +1,6 @@ const fs = require(`fs-extra`) const got = require(`got`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const path = require(`path`) const { isWebUri } = require(`valid-url`) const Queue = require(`better-queue`) @@ -57,6 +57,20 @@ const bar = new ProgressBar( * utils * *********/ +/** + * createHash + * -- + * + * Create an md5 hash of the given str + * @param {Stringq} str + * @return {String} + */ +const createHash = str => + crypto + .createHash(`md5`) + .update(str) + .digest(`hex`) + const CACHE_DIR = `.cache` const FS_PLUGIN_DIR = `gatsby-source-filesystem` @@ -199,7 +213,7 @@ async function processRemoteNode({ } // Create the temp and permanent file names for the url. - const digest = createContentDigest(url) + const digest = createHash(url) if (!name) { name = getRemoteFileName(url) } From 42f7d4ebcfcc358944d35b77f02af73bec61f1f3 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 21:30:05 -0600 Subject: [PATCH 36/63] revert createContentDigest to crypto for create-file-node in gatsby-source-filesystem --- .../gatsby-source-filesystem/src/create-file-node.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/gatsby-source-filesystem/src/create-file-node.js b/packages/gatsby-source-filesystem/src/create-file-node.js index 702e45e08bacb..3db6fcadd8483 100644 --- a/packages/gatsby-source-filesystem/src/create-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-file-node.js @@ -4,8 +4,8 @@ const fs = require(`fs-extra`) const mime = require(`mime`) const prettyBytes = require(`pretty-bytes`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const md5File = require(`bluebird`).promisify(require(`md5-file`)) +const crypto = require(`crypto`) exports.createFileNode = async ( pathToFile, @@ -27,10 +27,12 @@ exports.createFileNode = async ( const stats = await fs.stat(slashedFile.absolutePath) let internal if (stats.isDirectory()) { - const contentDigest = createContentDigest({ - stats: stats, - absolutePath: slashedFile.absolutePath, - }) + const contentDigest = crypto + .createHash(`md5`) + .update( + JSON.stringify({ stats: stats, absolutePath: slashedFile.absolutePath }) + ) + .digest(`hex`) internal = { contentDigest, type: `Directory`, From 2320c790477ddd6da3798012714a7554b1b8e673 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 21:58:17 -0600 Subject: [PATCH 37/63] revert createContentDigest for gatsby-plugin-sharp --- packages/gatsby-plugin-sharp/src/index.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index 70201af099216..7c5d8538cca22 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -1,5 +1,5 @@ const sharp = require(`sharp`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const imageSize = require(`probe-image-size`) const _ = require(`lodash`) const Promise = require(`bluebird`) @@ -403,7 +403,10 @@ function queueImageResizing({ file, args = {}, reporter }) { const sortedArgs = _.sortBy(filteredArgs, arg => arg[0] === `width`) const fileExtension = options.toFormat ? options.toFormat : file.extension - const argsDigest = createContentDigest(JSON.stringify(sortedArgs)) + const argsDigest = cyrpto + .createHash(`md5`) + .update(JSON.stringify(sortedArgs)) + .digest(`hex`) const argsDigestShort = argsDigest.substr(argsDigest.length - 5) @@ -925,7 +928,10 @@ async function notMemoizedtraceSVG({ file, args, fileArgs, reporter }) { const tmpDir = require(`os`).tmpdir() const tmpFilePath = `${tmpDir}/${file.internal.contentDigest}-${ file.name - }-${createContentDigest(JSON.stringify(fileArgs))}.${file.extension}` + }-${crypto + .createHash(`md5`) + .update(JSON.stringify(fileArgs)) + .digest(`hex`)}.${file.extension}` await new Promise(resolve => pipeline.toFile(tmpFilePath, (err, info) => { From 91531097e08fdb13749cda188afacfd51a1e4d05 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 22:03:01 -0600 Subject: [PATCH 38/63] update crypto misspell --- packages/gatsby-plugin-sharp/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index 7c5d8538cca22..48f801677b433 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -403,7 +403,7 @@ function queueImageResizing({ file, args = {}, reporter }) { const sortedArgs = _.sortBy(filteredArgs, arg => arg[0] === `width`) const fileExtension = options.toFormat ? options.toFormat : file.extension - const argsDigest = cyrpto + const argsDigest = crypto .createHash(`md5`) .update(JSON.stringify(sortedArgs)) .digest(`hex`) From 2e7b3058fb333225b802f37fcec808150b55b7bc Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 22:33:38 -0600 Subject: [PATCH 39/63] revert gatsby-remark-contentful and gatsby-source-contentful --- .../src/index.js | 7 +++++-- .../src/cache-image.js | 7 +++++-- .../gatsby-source-contentful/src/normalize.js | 19 ++++++++++++------- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/packages/gatsby-remark-images-contentful/src/index.js b/packages/gatsby-remark-images-contentful/src/index.js index 8a0eace211b34..3eb9374c73cb8 100644 --- a/packages/gatsby-remark-images-contentful/src/index.js +++ b/packages/gatsby-remark-images-contentful/src/index.js @@ -1,3 +1,4 @@ +const crypto = require(`crypto`) const select = require(`unist-util-select`) const sharp = require(`sharp`) const axios = require(`axios`) @@ -5,7 +6,6 @@ const _ = require(`lodash`) const Promise = require(`bluebird`) const cheerio = require(`cheerio`) const { buildResponsiveSizes } = require(`./utils`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) // If the image is hosted on contentful // 1. Find the image file @@ -49,7 +49,10 @@ module.exports = async ( const fileName = srcSplit[srcSplit.length - 1] const options = _.defaults(pluginOptions, defaults) - const optionsHash = createContentDigest(options) + const optionsHash = crypto + .createHash(`md5`) + .update(JSON.stringify(options)) + .digest(`hex`) const cacheKey = `remark-images-ctf-${fileName}-${optionsHash}` let cahedRawHTML = await cache.get(cacheKey) diff --git a/packages/gatsby-source-contentful/src/cache-image.js b/packages/gatsby-source-contentful/src/cache-image.js index d415c6931273a..a0a9bb30b04d5 100644 --- a/packages/gatsby-source-contentful/src/cache-image.js +++ b/packages/gatsby-source-contentful/src/cache-image.js @@ -1,4 +1,4 @@ -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const { resolve, parse } = require(`path`) const axios = require(`axios`) @@ -37,7 +37,10 @@ module.exports = async function cacheImage(store, image, options) { params.push(`bg=${background}`) } - const optionsHash = createContentDigest([url, ...params]) + const optionsHash = crypto + .createHash(`md5`) + .update(JSON.stringify([url, ...params])) + .digest(`hex`) const { name, ext } = parse(fileName) const absolutePath = resolve(CACHE_DIR, `${name}-${optionsHash}${ext}`) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 8aabe3f49a1cc..d7744168af49e 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -1,8 +1,13 @@ const _ = require(`lodash`) +const crypto = require(`crypto`) const stringify = require(`json-stringify-safe`) const deepMap = require(`deep-map`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const digest = str => + crypto + .createHash(`md5`) + .update(str) + .digest(`hex`) const typePrefix = `Contentful` const makeTypeName = type => _.upperFirst(_.camelCase(`${typePrefix} ${type}`)) @@ -179,7 +184,7 @@ function prepareTextNode(node, key, text, createNodeId) { type: _.camelCase(`${node.internal.type} ${key} TextNode`), mediaType: `text/markdown`, content: str, - contentDigest: createContentDigest(str), + contentDigest: digest(str), }, } @@ -200,7 +205,7 @@ function prepareRichTextNode(node, key, content, createNodeId) { type: _.camelCase(`${node.internal.type} ${key} RichTextNode`), mediaType: `text/richtext`, content: str, - contentDigest: createContentDigest(str), + contentDigest: digest(str), }, } @@ -220,7 +225,7 @@ function prepareJSONNode(node, key, content, createNodeId, i = ``) { type: _.camelCase(`${node.internal.type} ${key} JSONNode`), mediaType: `application/json`, content: str, - contentDigest: createContentDigest(str), + contentDigest: digest(str), }, } @@ -453,7 +458,7 @@ exports.createContentTypeNodes = ({ entryNode = { ...entryItemFields, ...entryNode, node_locale: locale.code } // Get content digest of node. - const contentDigest = createContentDigest(stringify(entryNode)) + const contentDigest = digest(stringify(entryNode)) entryNode.internal.contentDigest = contentDigest @@ -474,7 +479,7 @@ exports.createContentTypeNodes = ({ } // Get content digest of node. - const contentDigest = createContentDigest(stringify(contentTypeNode)) + const contentDigest = digest(stringify(contentTypeNode)) contentTypeNode.internal.contentDigest = contentDigest @@ -535,7 +540,7 @@ exports.createAssetNodes = ({ } // Get content digest of node. - const contentDigest = createContentDigest(stringify(assetNode)) + const contentDigest = digest(stringify(assetNode)) assetNode.internal.contentDigest = contentDigest From bd9e193d19711550ad64f2a56c29f46df980dfde Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 23:02:51 -0600 Subject: [PATCH 40/63] fix old code --- .../gatsby-source-contentful/src/normalize.js | 1 - .../src/gatsby-node.js | 2 +- .../src/__tests__/normalize.js | 8 +------- .../gatsby-source-lever/src/gatsby-node.js | 8 ++------ packages/gatsby-source-lever/src/normalize.js | 18 ++++-------------- .../gatsby-source-mongodb/src/gatsby-node.js | 17 ++++------------- packages/gatsby-source-mongodb/src/mapping.js | 10 ++-------- .../src/gatsby-node.js | 3 ++- .../src/__tests__/gatsby-node.js | 4 ---- .../src/gatsby-node.js | 9 ++------- .../src/__tests__/extend-node.js | 2 -- .../src/__tests__/on-node-create.js | 5 ----- .../src/on-node-create.js | 2 +- .../src/__tests__/gatsby-node.js | 3 --- .../gatsby-transformer-toml/src/gatsby-node.js | 9 ++------- packages/gatsby/src/bootstrap/index.js | 2 +- 16 files changed, 22 insertions(+), 81 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index d7744168af49e..58160e3201d24 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -213,7 +213,6 @@ function prepareRichTextNode(node, key, content, createNodeId) { return richTextNode } - function prepareJSONNode(node, key, content, createNodeId, i = ``) { const str = JSON.stringify(content) const JSONNode = { diff --git a/packages/gatsby-source-hacker-news/src/gatsby-node.js b/packages/gatsby-source-hacker-news/src/gatsby-node.js index efb96ce4db52d..4e04935e9f2d6 100644 --- a/packages/gatsby-source-hacker-news/src/gatsby-node.js +++ b/packages/gatsby-source-hacker-news/src/gatsby-node.js @@ -1,4 +1,5 @@ const axios = require(`axios`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const url = require(`url`) const _ = require(`lodash`) @@ -11,7 +12,6 @@ exports.sourceNodes = async ({ actions, getNode, createNodeId, - createContentDigest, hasNodeChanged, }) => { const { createNode } = actions diff --git a/packages/gatsby-source-lever/src/__tests__/normalize.js b/packages/gatsby-source-lever/src/__tests__/normalize.js index 8b97d01412d8a..aa976b2abade5 100644 --- a/packages/gatsby-source-lever/src/__tests__/normalize.js +++ b/packages/gatsby-source-lever/src/__tests__/normalize.js @@ -2,8 +2,6 @@ const normalize = require(`../normalize`) let entities = require(`./data.json`) -const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) - describe(`Process Lever data`, () => { it(`Creates entities from object collections of entities`, () => { entities = normalize.normalizeEntities(entities) @@ -24,11 +22,7 @@ describe(`Process Lever data`, () => { }) it(`creates nodes for each entry`, () => { const createNode = jest.fn() - normalize.createNodesFromEntities({ - entities, - createNode, - createContentDigest, - }) + normalize.createNodesFromEntities({ entities, createNode }) expect(createNode.mock.calls).toMatchSnapshot() }) }) diff --git a/packages/gatsby-source-lever/src/gatsby-node.js b/packages/gatsby-source-lever/src/gatsby-node.js index ce6a70b8d2add..afac68c621408 100644 --- a/packages/gatsby-source-lever/src/gatsby-node.js +++ b/packages/gatsby-source-lever/src/gatsby-node.js @@ -4,7 +4,7 @@ const normalize = require(`./normalize`) const typePrefix = `lever__` exports.sourceNodes = async ( - { actions, getNode, store, cache, createNodeId, createContentDigest }, + { actions, getNode, store, cache, createNodeId }, { site, verboseOutput } ) => { const { createNode } = actions @@ -30,11 +30,7 @@ exports.sourceNodes = async ( entities = normalize.createGatsbyIds(createNodeId, entities) // creates nodes for each entry - normalize.createNodesFromEntities({ - entities, - createNode, - createContentDigest, - }) + normalize.createNodesFromEntities({ entities, createNode }) return } diff --git a/packages/gatsby-source-lever/src/normalize.js b/packages/gatsby-source-lever/src/normalize.js index b0833a75acefd..39316e0739068 100644 --- a/packages/gatsby-source-lever/src/normalize.js +++ b/packages/gatsby-source-lever/src/normalize.js @@ -1,3 +1,4 @@ +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const deepMapKeys = require(`deep-map-keys`) const stringify = require(`json-stringify-safe`) @@ -12,14 +13,7 @@ const restrictedNodeFields = [`id`, `children`, `parent`, `fields`, `internal`] * @param {any} type * @param {any} createNode */ -async function createGraphQLNode( - ent, - type, - createNode, - createContentDigest, - store, - cache -) { +async function createGraphQLNode(ent, type, createNode, store, cache) { let id = !ent.id ? (!ent.ID ? 0 : ent.ID) : ent.id let node = { id: `${type}_${id.toString()}`, @@ -128,11 +122,7 @@ exports.createGatsbyIds = (createNodeId, entities) => return e }) -exports.createNodesFromEntities = ({ - entities, - createNode, - createContentDigest, -}) => { +exports.createNodesFromEntities = ({ entities, createNode }) => { entities.forEach(e => { let { ...entity } = e let node = { @@ -141,7 +131,7 @@ exports.createNodesFromEntities = ({ children: [], internal: { type: `lever`, - contentDigest: createContentDigest(JSON.stringify(entity)), + contentDigest: createContentDigest(entity), }, } createNode(node) diff --git a/packages/gatsby-source-mongodb/src/gatsby-node.js b/packages/gatsby-source-mongodb/src/gatsby-node.js index aef88cc2fef29..f07db58f59478 100644 --- a/packages/gatsby-source-mongodb/src/gatsby-node.js +++ b/packages/gatsby-source-mongodb/src/gatsby-node.js @@ -1,9 +1,10 @@ const MongoClient = require(`mongodb`).MongoClient +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const prepareMappingChildNode = require(`./mapping`) const queryString = require(`query-string`) exports.sourceNodes = ( - { actions, getNode, createNodeId, createContentDigest, hasNodeChanged }, + { actions, getNode, createNodeId, hasNodeChanged }, pluginOptions ) => { const { createNode } = actions @@ -33,15 +34,7 @@ exports.sourceNodes = ( return Promise.all( collection.map(col => - createNodes( - db, - pluginOptions, - dbName, - createNode, - createNodeId, - createContentDigest, - col - ) + createNodes(db, pluginOptions, dbName, createNode, createNodeId, col) ) ) .then(() => { @@ -65,7 +58,6 @@ function createNodes( dbName, createNode, createNodeId, - createContentDigest, collectionName ) { return new Promise((resolve, reject) => { @@ -115,8 +107,7 @@ function createNodes( mediaItemFieldKey, node[mediaItemFieldKey], mapObj[mediaItemFieldKey], - createNode, - createContentDigest + createNode ) node[`${mediaItemFieldKey}___NODE`] = mappingChildNode.id diff --git a/packages/gatsby-source-mongodb/src/mapping.js b/packages/gatsby-source-mongodb/src/mapping.js index d7f05b3310e3d..e322640bd45f7 100644 --- a/packages/gatsby-source-mongodb/src/mapping.js +++ b/packages/gatsby-source-mongodb/src/mapping.js @@ -1,14 +1,8 @@ const camelCase = require(`lodash.camelcase`) const isString = require(`lodash.isstring`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) -module.exports = function( - node, - key, - text, - mediaType, - createNode, - createContentDigest -) { +module.exports = function(node, key, text, mediaType, createNode) { const str = isString(text) ? text : ` ` const id = `${node.id}${key}MappingNode` const mappingNode = { diff --git a/packages/gatsby-source-npm-package-search/src/gatsby-node.js b/packages/gatsby-source-npm-package-search/src/gatsby-node.js index 590a55fc154c1..59bec29168341 100644 --- a/packages/gatsby-source-npm-package-search/src/gatsby-node.js +++ b/packages/gatsby-source-npm-package-search/src/gatsby-node.js @@ -1,4 +1,5 @@ const algoliasearch = require(`algoliasearch`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const client = algoliasearch(`OFCNCOG2CU`, `6fbcaeafced8913bf0e4d39f0b541957`) var index = client.initIndex(`npm-search`) @@ -15,7 +16,7 @@ function browse({ index, ...params }) { } exports.sourceNodes = async ( - { boundActionCreators, createNodeId, createContentDigest }, + { boundActionCreators, createNodeId }, { keywords } ) => { const { createNode } = boundActionCreators diff --git a/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js index b7804aaf2ca63..eccbcb21b434f 100644 --- a/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-hjson/src/__tests__/gatsby-node.js @@ -3,8 +3,6 @@ const HJSON = require(`hjson`) const { onCreateNode } = require(`../gatsby-node`) -const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) - describe(`Process HJSON nodes correctly`, () => { const node = { name: `nodeName`, @@ -39,7 +37,6 @@ describe(`Process HJSON nodes correctly`, () => { loadNodeContent, actions, createNodeId, - createContentDigest, }).then(() => { expect(createNode.mock.calls).toMatchSnapshot() expect(createParentChildLink.mock.calls).toMatchSnapshot() @@ -64,7 +61,6 @@ describe(`Process HJSON nodes correctly`, () => { loadNodeContent, actions, createNodeId, - createContentDigest, }).then(() => { expect(createNode.mock.calls).toMatchSnapshot() expect(createParentChildLink.mock.calls).toMatchSnapshot() diff --git a/packages/gatsby-transformer-hjson/src/gatsby-node.js b/packages/gatsby-transformer-hjson/src/gatsby-node.js index 66c629b493d63..1e08091cbcee2 100644 --- a/packages/gatsby-transformer-hjson/src/gatsby-node.js +++ b/packages/gatsby-transformer-hjson/src/gatsby-node.js @@ -1,14 +1,9 @@ const _ = require(`lodash`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const path = require(`path`) const HJSON = require(`hjson`) -async function onCreateNode({ - node, - actions, - loadNodeContent, - createNodeId, - createContentDigest, -}) { +async function onCreateNode({ node, actions, loadNodeContent, createNodeId }) { const { createNode, createParentChildLink } = actions function transformObject(obj, id, type) { diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 095ae5249a4ea..435fc8b8d1862 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -117,14 +117,12 @@ const bootstrapTest = ( const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) - const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode( { node, loadNodeContent, actions, createNodeId, - createContentDigest, }, { ...additionalParameters, ...pluginOptions } ) diff --git a/packages/gatsby-transformer-remark/src/__tests__/on-node-create.js b/packages/gatsby-transformer-remark/src/__tests__/on-node-create.js index 191668e8c9536..62564aedcfa9e 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/__tests__/on-node-create.js @@ -12,7 +12,6 @@ const { const { inferObjectStructureFromNodes, } = require(`../../../gatsby/src/schema/infer-graphql-type`) -const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) let node let actions @@ -49,7 +48,6 @@ Where oh where is my little pony? loadNodeContent, actions, createNodeId, - createContentDigest, }).then(() => { expect(actions.createNode.mock.calls).toMatchSnapshot() expect( @@ -87,7 +85,6 @@ Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat ma loadNodeContent, actions, createNodeId, - createContentDigest, }, { excerpt_separator: `` } ).then(() => { @@ -223,7 +220,6 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid loadNodeContent, actions, createNodeId, - createContentDigest, }, { excerpt_separator: `` } ) @@ -279,7 +275,6 @@ Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat ma loadNodeContent, actions, createNodeId, - createContentDigest, }) }) }) diff --git a/packages/gatsby-transformer-remark/src/on-node-create.js b/packages/gatsby-transformer-remark/src/on-node-create.js index 959b670276313..e8a720579a9eb 100644 --- a/packages/gatsby-transformer-remark/src/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/on-node-create.js @@ -1,5 +1,5 @@ -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const grayMatter = require(`gray-matter`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) const _ = require(`lodash`) module.exports = async function onCreateNode( diff --git a/packages/gatsby-transformer-toml/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-toml/src/__tests__/gatsby-node.js index a003d85232067..973f3c6eda227 100644 --- a/packages/gatsby-transformer-toml/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-toml/src/__tests__/gatsby-node.js @@ -1,7 +1,5 @@ const { onCreateNode } = require(`../gatsby-node`) -const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) - describe(`Process TOML nodes correctly`, () => { const node = { id: `whatever`, @@ -50,7 +48,6 @@ describe(`Process TOML nodes correctly`, () => { loadNodeContent, actions, createNodeId, - createContentDigest, }).then(() => { expect(createNode.mock.calls).toMatchSnapshot() expect(createParentChildLink.mock.calls).toMatchSnapshot() diff --git a/packages/gatsby-transformer-toml/src/gatsby-node.js b/packages/gatsby-transformer-toml/src/gatsby-node.js index 8b4f8a3b56299..0e9f1bc38149e 100644 --- a/packages/gatsby-transformer-toml/src/gatsby-node.js +++ b/packages/gatsby-transformer-toml/src/gatsby-node.js @@ -1,13 +1,8 @@ const toml = require(`toml`) const _ = require(`lodash`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) -async function onCreateNode({ - node, - actions, - loadNodeContent, - createNodeId, - createContentDigest, -}) { +async function onCreateNode({ node, actions, loadNodeContent, createNodeId }) { const { createNode, createParentChildLink } = actions // Filter out non-toml content // Currently TOML files are considered null in 'mime-db' diff --git a/packages/gatsby/src/bootstrap/index.js b/packages/gatsby/src/bootstrap/index.js index d63fbc00ad17b..e7a5cc91ed051 100644 --- a/packages/gatsby/src/bootstrap/index.js +++ b/packages/gatsby/src/bootstrap/index.js @@ -4,6 +4,7 @@ const _ = require(`lodash`) const slash = require(`slash`) const fs = require(`fs-extra`) const md5File = require(`md5-file/promise`) +const createContentDigest = require(`../../src/utils/create-content-digest`) const del = require(`del`) const path = require(`path`) const convertHrtime = require(`convert-hrtime`) @@ -20,7 +21,6 @@ const getConfigFile = require(`./get-config-file`) const tracer = require(`opentracing`).globalTracer() const preferDefault = require(`./prefer-default`) const nodeTracking = require(`../db/node-tracking`) -const createContentDigest = require(`../../src/utils/create-content-digest`) require(`../db`).startAutosave() // Show stack trace on unhandled promises. From e7e262519c7210eb9fa51e9948cb6980197fd6fd Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 23:14:29 -0600 Subject: [PATCH 41/63] fix old code v2 --- .../gatsby-transformer-excel/src/__tests__/gatsby-node.js | 4 ---- packages/gatsby-transformer-excel/src/gatsby-node.js | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/gatsby-transformer-excel/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-excel/src/__tests__/gatsby-node.js index 695427c1b719b..00ccf2b0e50ce 100644 --- a/packages/gatsby-transformer-excel/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-excel/src/__tests__/gatsby-node.js @@ -2,7 +2,6 @@ const Promise = require(`bluebird`) const XLSX = require(`xlsx`) const { onCreateNode } = require(`../gatsby-node`) -const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) describe(`Process nodes correctly`, () => { const node = { @@ -36,7 +35,6 @@ describe(`Process nodes correctly`, () => { loadNodeContent, actions, createNodeId, - createContentDigest, }).then(() => { expect(createNode.mock.calls).toMatchSnapshot() expect(createParentChildLink.mock.calls).toMatchSnapshot() @@ -62,7 +60,6 @@ describe(`Process nodes correctly`, () => { loadNodeContent, actions, createNodeId, - createContentDigest, }, { raw: false } ).then(() => { @@ -90,7 +87,6 @@ describe(`Process nodes correctly`, () => { loadNodeContent, actions, createNodeId, - createContentDigest, }, { rawOutput: false } ).then(() => { diff --git a/packages/gatsby-transformer-excel/src/gatsby-node.js b/packages/gatsby-transformer-excel/src/gatsby-node.js index d37c390e38123..26f89dab54ca5 100644 --- a/packages/gatsby-transformer-excel/src/gatsby-node.js +++ b/packages/gatsby-transformer-excel/src/gatsby-node.js @@ -1,6 +1,7 @@ const XLSX = require(`xlsx`) const fs = require(`fs-extra`) const _ = require(`lodash`) +const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) // read files as `binary` from file system function _loadNodeContent(fileNode, fallback) { @@ -10,7 +11,7 @@ function _loadNodeContent(fileNode, fallback) { } async function onCreateNode( - { node, actions, loadNodeContent, createNodeId, createContentDigest }, + { node, actions, loadNodeContent, createNodeId }, options = {} ) { const { createNode, createParentChildLink } = actions From 07527778f470d3bc36109c4694ed720e82a921d0 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 23:28:55 -0600 Subject: [PATCH 42/63] update gatsby pkg --- packages/gatsby/src/bootstrap/index.js | 4 ++-- .../src/internal-plugins/query-runner/pages-writer.js | 7 +++++-- .../src/internal-plugins/query-runner/query-runner.js | 5 ++++- .../src/internal-plugins/query-runner/redirects-writer.js | 8 ++++++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/gatsby/src/bootstrap/index.js b/packages/gatsby/src/bootstrap/index.js index e7a5cc91ed051..dfe71c44c6581 100644 --- a/packages/gatsby/src/bootstrap/index.js +++ b/packages/gatsby/src/bootstrap/index.js @@ -4,7 +4,7 @@ const _ = require(`lodash`) const slash = require(`slash`) const fs = require(`fs-extra`) const md5File = require(`md5-file/promise`) -const createContentDigest = require(`../../src/utils/create-content-digest`) +const createContentDigest = require(`../utils/create-content-digest`) const del = require(`del`) const path = require(`path`) const convertHrtime = require(`convert-hrtime`) @@ -480,7 +480,7 @@ module.exports = async (args: BootstrapArgs) => { parentSpan: bootstrapSpan, }) activity.start() - await writeRedirects(createContentDigest) + await writeRedirects() activity.end() let onEndJob diff --git a/packages/gatsby/src/internal-plugins/query-runner/pages-writer.js b/packages/gatsby/src/internal-plugins/query-runner/pages-writer.js index 3412ef84b3bb6..10e8c21e76bd9 100644 --- a/packages/gatsby/src/internal-plugins/query-runner/pages-writer.js +++ b/packages/gatsby/src/internal-plugins/query-runner/pages-writer.js @@ -1,6 +1,6 @@ const _ = require(`lodash`) const fs = require(`fs-extra`) -const createContentDigest = require(`../../utils/create-content-digest`) +const crypto = require(`crypto`) const { store, emitter } = require(`../../redux/`) @@ -48,7 +48,10 @@ const writePages = async () => { }` ) .value() - const newHash = createContentDigest(pagesComponentDependencies) + const newHash = crypto + .createHash(`md5`) + .update(JSON.stringify(pagesComponentDependencies)) + .digest(`hex`) if (newHash === lastHash) { // components didn't change - no need to rewrite pages.json diff --git a/packages/gatsby/src/internal-plugins/query-runner/query-runner.js b/packages/gatsby/src/internal-plugins/query-runner/query-runner.js index d86bfdb2a3aba..1e3197efc43f4 100644 --- a/packages/gatsby/src/internal-plugins/query-runner/query-runner.js +++ b/packages/gatsby/src/internal-plugins/query-runner/query-runner.js @@ -80,7 +80,10 @@ ${formatErrorDetails(errorDetails)}`) } const resultJSON = JSON.stringify(result) - const resultHash = require(`../../utils/create-content-digest`)(resultJSON) + const resultHash = require(`crypto`) + .createHash(`sha1`) + .update(resultJSON) + .digest(`base64`) // Remove potentially unsafe characters. This increases chances of collisions // slightly but it should still be very safe + we get a shorter // url vs hex. diff --git a/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js b/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js index 425d43ace9e58..da2452107af2d 100644 --- a/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js +++ b/packages/gatsby/src/internal-plugins/query-runner/redirects-writer.js @@ -1,11 +1,12 @@ import _ from "lodash" +import crypto from "crypto" import fs from "fs-extra" import { store, emitter } from "../../redux/" import { joinPath } from "../../utils/path" let lastHash = null -const writeRedirects = async createContentDigest => { +const writeRedirects = async () => { bootstrapFinished = true let { program, redirects } = store.getState() @@ -13,7 +14,10 @@ const writeRedirects = async createContentDigest => { // Filter for redirects that are meant for the browser. const browserRedirects = redirects.filter(r => r.redirectInBrowser) - const newHash = createContentDigest(browserRedirects) + const newHash = crypto + .createHash(`md5`) + .update(JSON.stringify(browserRedirects)) + .digest(`hex`) if (newHash === lastHash) { return Promise.resolve() From c3d6853958741937f8d1a0a80e18b9690fdff967 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Thu, 7 Mar 2019 23:42:06 -0600 Subject: [PATCH 43/63] revert gatsby-transformer-remark --- packages/gatsby-transformer-remark/src/on-node-create.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-transformer-remark/src/on-node-create.js b/packages/gatsby-transformer-remark/src/on-node-create.js index e8a720579a9eb..c2937b08595da 100644 --- a/packages/gatsby-transformer-remark/src/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/on-node-create.js @@ -1,5 +1,5 @@ const grayMatter = require(`gray-matter`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const _ = require(`lodash`) module.exports = async function onCreateNode( @@ -53,7 +53,10 @@ module.exports = async function onCreateNode( markdownNode.fileAbsolutePath = node.absolutePath } - markdownNode.internal.contentDigest = createContentDigest(markdownNode) + markdownNode.internal.contentDigest = crypto + .createHash(`md5`) + .update(JSON.stringify(markdownNode)) + .digest(`hex`) createNode(markdownNode) createParentChildLink({ parent: node, child: markdownNode }) From d54c26100dc5b9a8a53b787729ae789613022145 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Fri, 8 Mar 2019 06:57:06 -0600 Subject: [PATCH 44/63] add export for gatsby utils --- packages/gatsby-transformer-remark/package.json | 1 + .../src/__tests__/extend-node.js | 2 ++ .../src/__tests__/on-node-create.js | 7 +++++++ .../src/on-node-create.js | 15 +++++++++------ packages/gatsby/utils.js | 1 + 5 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 packages/gatsby/utils.js diff --git a/packages/gatsby-transformer-remark/package.json b/packages/gatsby-transformer-remark/package.json index 9348d6a18e5a8..56a3d749cb94c 100644 --- a/packages/gatsby-transformer-remark/package.json +++ b/packages/gatsby-transformer-remark/package.json @@ -9,6 +9,7 @@ "dependencies": { "@babel/runtime": "^7.0.0", "bluebird": "^3.5.0", + "gatsby": "^2.1.24", "gray-matter": "^4.0.0", "hast-util-raw": "^4.0.0", "hast-util-to-html": "^4.0.0", diff --git a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js index 435fc8b8d1862..f3b53d2eae32d 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/extend-node.js +++ b/packages/gatsby-transformer-remark/src/__tests__/extend-node.js @@ -9,6 +9,7 @@ const { inferObjectStructureFromNodes, } = require(`../../../gatsby/src/schema/infer-graphql-type`) const extendNodeType = require(`../extend-node-type`) +const { createContentDigest } = require(`gatsby/utils`) // given a set of nodes and a query, return the result of the query async function queryResult( @@ -123,6 +124,7 @@ const bootstrapTest = ( loadNodeContent, actions, createNodeId, + createContentDigest, }, { ...additionalParameters, ...pluginOptions } ) diff --git a/packages/gatsby-transformer-remark/src/__tests__/on-node-create.js b/packages/gatsby-transformer-remark/src/__tests__/on-node-create.js index 62564aedcfa9e..7a72b148b22cf 100644 --- a/packages/gatsby-transformer-remark/src/__tests__/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/__tests__/on-node-create.js @@ -13,6 +13,8 @@ const { inferObjectStructureFromNodes, } = require(`../../../gatsby/src/schema/infer-graphql-type`) +const { createContentDigest } = require(`gatsby/utils`) + let node let actions let createNodeId @@ -48,6 +50,7 @@ Where oh where is my little pony? loadNodeContent, actions, createNodeId, + createContentDigest, }).then(() => { expect(actions.createNode.mock.calls).toMatchSnapshot() expect( @@ -85,6 +88,7 @@ Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat ma loadNodeContent, actions, createNodeId, + createContentDigest, }, { excerpt_separator: `` } ).then(() => { @@ -116,6 +120,7 @@ yadda yadda actions, createNodeId, loadNodeContent, + createContentDigest, }) expect(parsed.frontmatter.date).toEqual(new Date(date).toJSON()) @@ -220,6 +225,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid loadNodeContent, actions, createNodeId, + createContentDigest, }, { excerpt_separator: `` } ) @@ -275,6 +281,7 @@ Sed bibendum sem iaculis, pellentesque leo sed, imperdiet ante. Sed consequat ma loadNodeContent, actions, createNodeId, + createContentDigest, }) }) }) diff --git a/packages/gatsby-transformer-remark/src/on-node-create.js b/packages/gatsby-transformer-remark/src/on-node-create.js index c2937b08595da..115b9daef9999 100644 --- a/packages/gatsby-transformer-remark/src/on-node-create.js +++ b/packages/gatsby-transformer-remark/src/on-node-create.js @@ -1,9 +1,15 @@ const grayMatter = require(`gray-matter`) -const crypto = require(`crypto`) const _ = require(`lodash`) module.exports = async function onCreateNode( - { node, loadNodeContent, actions, createNodeId, reporter }, + { + node, + loadNodeContent, + actions, + createNodeId, + reporter, + createContentDigest, + }, pluginOptions ) { const { createNode, createParentChildLink } = actions @@ -53,10 +59,7 @@ module.exports = async function onCreateNode( markdownNode.fileAbsolutePath = node.absolutePath } - markdownNode.internal.contentDigest = crypto - .createHash(`md5`) - .update(JSON.stringify(markdownNode)) - .digest(`hex`) + markdownNode.internal.contentDigest = createContentDigest(markdownNode) createNode(markdownNode) createParentChildLink({ parent: node, child: markdownNode }) diff --git a/packages/gatsby/utils.js b/packages/gatsby/utils.js new file mode 100644 index 0000000000000..cb2c70625a460 --- /dev/null +++ b/packages/gatsby/utils.js @@ -0,0 +1 @@ +exports.createContentDigest = require(`./dist/utils/create-content-digest`) From e90f38120925561230424b2024fd781d7dc9be62 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 12:05:17 -0600 Subject: [PATCH 45/63] update gatsby-source-filesystem --- .../gatsby-source-filesystem/package.json | 2 +- .../src/create-file-node.js | 12 +++++----- .../src/create-remote-file-node.js | 22 ++----------------- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json index 15dfb7ae43a44..7168686b0c358 100644 --- a/packages/gatsby-source-filesystem/package.json +++ b/packages/gatsby-source-filesystem/package.json @@ -36,7 +36,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": "^2.0.0" + "gatsby": "^2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem", "scripts": { diff --git a/packages/gatsby-source-filesystem/src/create-file-node.js b/packages/gatsby-source-filesystem/src/create-file-node.js index 3db6fcadd8483..a11add1b72cf4 100644 --- a/packages/gatsby-source-filesystem/src/create-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-file-node.js @@ -5,7 +5,7 @@ const mime = require(`mime`) const prettyBytes = require(`pretty-bytes`) const md5File = require(`bluebird`).promisify(require(`md5-file`)) -const crypto = require(`crypto`) +const { createContentDigest } = require(`gatsby/utils`) exports.createFileNode = async ( pathToFile, @@ -27,12 +27,10 @@ exports.createFileNode = async ( const stats = await fs.stat(slashedFile.absolutePath) let internal if (stats.isDirectory()) { - const contentDigest = crypto - .createHash(`md5`) - .update( - JSON.stringify({ stats: stats, absolutePath: slashedFile.absolutePath }) - ) - .digest(`hex`) + const contentDigest = createContentDigest({ + stats: stats, + absolutePath: slashedFile.absolutePath, + }) internal = { contentDigest, type: `Directory`, diff --git a/packages/gatsby-source-filesystem/src/create-remote-file-node.js b/packages/gatsby-source-filesystem/src/create-remote-file-node.js index 7f6d1cf09ae96..36f4ed9d95833 100644 --- a/packages/gatsby-source-filesystem/src/create-remote-file-node.js +++ b/packages/gatsby-source-filesystem/src/create-remote-file-node.js @@ -1,6 +1,6 @@ const fs = require(`fs-extra`) const got = require(`got`) -const crypto = require(`crypto`) +const { createContentDigest } = require(`gatsby/utils`) const path = require(`path`) const { isWebUri } = require(`valid-url`) const Queue = require(`better-queue`) @@ -53,24 +53,6 @@ const bar = new ProgressBar( * @param {Auth} [options.auth] */ -/********* - * utils * - *********/ - -/** - * createHash - * -- - * - * Create an md5 hash of the given str - * @param {Stringq} str - * @return {String} - */ -const createHash = str => - crypto - .createHash(`md5`) - .update(str) - .digest(`hex`) - const CACHE_DIR = `.cache` const FS_PLUGIN_DIR = `gatsby-source-filesystem` @@ -213,7 +195,7 @@ async function processRemoteNode({ } // Create the temp and permanent file names for the url. - const digest = createHash(url) + const digest = createContentDigest(url) if (!name) { name = getRemoteFileName(url) } From 1fe7b893a627ee23f3f98068f097b6c0744a3aa6 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:35:33 -0600 Subject: [PATCH 46/63] revert gatsby-source-graphql --- packages/gatsby-source-drupal/src/gatsby-node.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index e9a228a95f367..c8971b84f7d09 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -1,10 +1,17 @@ const axios = require(`axios`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const _ = require(`lodash`) const { createRemoteFileNode } = require(`gatsby-source-filesystem`) const { URL } = require(`url`) const { nodeFromData } = require(`./normalize`) +// Get content digest of node. +const createContentDigest = obj => + crypto + .createHash(`md5`) + .update(JSON.stringify(obj)) + .digest(`hex`) + exports.sourceNodes = async ( { actions, getNode, hasNodeChanged, store, cache, createNodeId }, { baseUrl, apiBase, basicAuth } From 84558cd859c2870e08b973ed03f240f663a121a4 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:36:09 -0600 Subject: [PATCH 47/63] revert gatsby-source-graphql --- packages/gatsby-source-graphql/src/gatsby-node.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-graphql/src/gatsby-node.js b/packages/gatsby-source-graphql/src/gatsby-node.js index 27bf397d9d1dd..904f672be7577 100644 --- a/packages/gatsby-source-graphql/src/gatsby-node.js +++ b/packages/gatsby-source-graphql/src/gatsby-node.js @@ -1,4 +1,4 @@ -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const uuidv4 = require(`uuid/v4`) const { buildSchema, printSchema } = require(`graphql`) const { @@ -114,7 +114,10 @@ exports.sourceNodes = async ( function createSchemaNode({ id, typeName, fieldName }) { const nodeContent = uuidv4() - const nodeContentDigest = createContentDigest(nodeContent) + const nodeContentDigest = crypto + .createHash(`md5`) + .update(nodeContent) + .digest(`hex`) return { id, typeName: typeName, From 335c0c6e3253f4c8bf43ace38ae1cde33f297b77 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:37:53 -0600 Subject: [PATCH 48/63] revert gatsby-source-hacker-news --- .../gatsby-source-hacker-news/src/gatsby-node.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/gatsby-source-hacker-news/src/gatsby-node.js b/packages/gatsby-source-hacker-news/src/gatsby-node.js index 4e04935e9f2d6..b8c47f8eccaf5 100644 --- a/packages/gatsby-source-hacker-news/src/gatsby-node.js +++ b/packages/gatsby-source-hacker-news/src/gatsby-node.js @@ -1,5 +1,5 @@ const axios = require(`axios`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const url = require(`url`) const _ = require(`lodash`) @@ -62,7 +62,6 @@ exports.sourceNodes = async ({ } } } - fragment commentsFragment on HackerNewsItem { id text @@ -117,7 +116,10 @@ fragment commentsFragment on HackerNewsItem { storyNode.by = storyNode.by.id // Get content digest of node. - const contentDigest = createContentDigest(storyNode) + const contentDigest = crypto + .createHash(`md5`) + .update(JSON.stringify(storyNode)) + .digest(`hex`) storyNode.internal.contentDigest = contentDigest createNode(storyNode) @@ -144,7 +146,10 @@ fragment commentsFragment on HackerNewsItem { const nodeStr = JSON.stringify(commentNode) // Get content digest of comment node. - const contentDigest = createContentDigest(nodeStr) + const contentDigest = crypto + .createHash(`md5`) + .update(nodeStr) + .digest(`hex`) commentNode.internal.contentDigest = contentDigest commentNode.internal.content = nodeStr From 9f709e0de0532d89d2206ed8fa261b954a766bea Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:39:54 -0600 Subject: [PATCH 49/63] revert gatsby-source-lever --- packages/gatsby-source-lever/src/normalize.js | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-source-lever/src/normalize.js b/packages/gatsby-source-lever/src/normalize.js index 39316e0739068..b9aaa4cb58b9a 100644 --- a/packages/gatsby-source-lever/src/normalize.js +++ b/packages/gatsby-source-lever/src/normalize.js @@ -1,4 +1,4 @@ -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const deepMapKeys = require(`deep-map-keys`) const stringify = require(`json-stringify-safe`) @@ -6,6 +6,17 @@ const conflictFieldPrefix = `lever_` // restrictedNodeFields from here https://www.gatsbyjs.org/docs/node-interface/ const restrictedNodeFields = [`id`, `children`, `parent`, `fields`, `internal`] +/** + * Encrypts a String using md5 hash of hexadecimal digest. + * + * @param {any} str + */ +const digest = str => + crypto + .createHash(`md5`) + .update(str) + .digest(`hex`) + /** * Create the Graph QL Node * @@ -25,7 +36,7 @@ async function createGraphQLNode(ent, type, createNode, store, cache) { } node = recursiveAddFields(ent, node) node.internal.content = JSON.stringify(node) - node.internal.contentDigest = createContentDigest(stringify(node)) + node.internal.contentDigest = digest(stringify(node)) createNode(node) } exports.createGraphQLNode = createGraphQLNode @@ -131,7 +142,7 @@ exports.createNodesFromEntities = ({ entities, createNode }) => { children: [], internal: { type: `lever`, - contentDigest: createContentDigest(entity), + contentDigest: digest(JSON.stringify(entity)), }, } createNode(node) From c2b4c1a0e115efc8361704ced32ad2d8010298ee Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:40:11 -0600 Subject: [PATCH 50/63] revert gatsby-source-medium --- packages/gatsby-source-medium/src/gatsby-node.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-medium/src/gatsby-node.js b/packages/gatsby-source-medium/src/gatsby-node.js index 38b893754e4b0..397437ad07b04 100644 --- a/packages/gatsby-source-medium/src/gatsby-node.js +++ b/packages/gatsby-source-medium/src/gatsby-node.js @@ -1,5 +1,5 @@ const axios = require(`axios`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const fetch = (username, limit = 100) => { const url = `https://medium.com/${username}/latest?format=json&limit=${limit}` @@ -77,7 +77,10 @@ exports.sourceNodes = async ( resources.map(resource => { convertTimestamps(resource) - const digest = createContentDigest(resource) + const digest = crypto + .createHash(`md5`) + .update(JSON.stringify(resource)) + .digest(`hex`) const links = resource.type === `Post` From 5537c7709123a833bb5a1f2bf42edec4552c4f33 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:40:58 -0600 Subject: [PATCH 51/63] revert gatsby-source-mongodb --- packages/gatsby-source-mongodb/src/gatsby-node.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-mongodb/src/gatsby-node.js b/packages/gatsby-source-mongodb/src/gatsby-node.js index f07db58f59478..0a80143ff187a 100644 --- a/packages/gatsby-source-mongodb/src/gatsby-node.js +++ b/packages/gatsby-source-mongodb/src/gatsby-node.js @@ -1,5 +1,5 @@ const MongoClient = require(`mongodb`).MongoClient -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const prepareMappingChildNode = require(`./mapping`) const queryString = require(`query-string`) @@ -86,7 +86,10 @@ function createNodes( collectionName )}`, content: JSON.stringify(item), - contentDigest: createContentDigest(item), + contentDigest: crypto + .createHash(`md5`) + .update(JSON.stringify(item)) + .digest(`hex`), }, } const childrenNodes = [] From da57567126a66a8602f4e3d8d2b02d1eb62c2843 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:42:30 -0600 Subject: [PATCH 52/63] revert gatsby-source-mongodb mapping --- packages/gatsby-source-mongodb/src/mapping.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-mongodb/src/mapping.js b/packages/gatsby-source-mongodb/src/mapping.js index e322640bd45f7..1a8155fb37ff2 100644 --- a/packages/gatsby-source-mongodb/src/mapping.js +++ b/packages/gatsby-source-mongodb/src/mapping.js @@ -1,6 +1,6 @@ const camelCase = require(`lodash.camelcase`) const isString = require(`lodash.isstring`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) module.exports = function(node, key, text, mediaType, createNode) { const str = isString(text) ? text : ` ` @@ -14,7 +14,10 @@ module.exports = function(node, key, text, mediaType, createNode) { type: camelCase(`${node.internal.type} ${key} MappingNode`), mediaType: mediaType, content: str, - contentDigest: createContentDigest(str), + contentDigest: crypto + .createHash(`md5`) + .update(JSON.stringify(str)) + .digest(`hex`), }, } From 74222a574e038add3671f726ea69da7e7a3be012 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:43:57 -0600 Subject: [PATCH 53/63] revert gatsby-source-npm-package-search --- .../gatsby-source-npm-package-search/src/gatsby-node.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-npm-package-search/src/gatsby-node.js b/packages/gatsby-source-npm-package-search/src/gatsby-node.js index 59bec29168341..13606aef3a41b 100644 --- a/packages/gatsby-source-npm-package-search/src/gatsby-node.js +++ b/packages/gatsby-source-npm-package-search/src/gatsby-node.js @@ -1,9 +1,15 @@ const algoliasearch = require(`algoliasearch`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const client = algoliasearch(`OFCNCOG2CU`, `6fbcaeafced8913bf0e4d39f0b541957`) var index = client.initIndex(`npm-search`) +const createContentDigest = obj => + crypto + .createHash(`md5`) + .update(JSON.stringify(obj)) + .digest(`hex`) + function browse({ index, ...params }) { let hits = [] const browser = index.browseAll(params) From 41b2deb71a96e3ebadede0ff2f5b3ec71c34a10d Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:45:03 -0600 Subject: [PATCH 54/63] revert gatsby-source-wikipedia --- packages/gatsby-source-wikipedia/src/gatsby-node.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-source-wikipedia/src/gatsby-node.js b/packages/gatsby-source-wikipedia/src/gatsby-node.js index 6da957720d7d0..bfb311ceaef65 100644 --- a/packages/gatsby-source-wikipedia/src/gatsby-node.js +++ b/packages/gatsby-source-wikipedia/src/gatsby-node.js @@ -17,9 +17,10 @@ exports.sourceNodes = ({ actions, createNodeId }, pluginOptions) => { type: `WikipediaArticle`, }, } - node.internal.contentDigest = require(`../../gatsby/src/utils/create-content-digest`)( - node - ) + node.internal.contentDigest = require(`crypto`) + .createHash(`md5`) + .update(JSON.stringify(node)) + .digest(`hex`) createNode(node) }) ) From c105a3feadf5d6193ad17c3a60688e15a3ad4b2b Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:46:15 -0600 Subject: [PATCH 55/63] revert gatsby-source-wordpress --- packages/gatsby-source-wordpress/src/normalize.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-wordpress/src/normalize.js b/packages/gatsby-source-wordpress/src/normalize.js index 3e9f124dfae28..15210eaae50d1 100644 --- a/packages/gatsby-source-wordpress/src/normalize.js +++ b/packages/gatsby-source-wordpress/src/normalize.js @@ -1,4 +1,4 @@ -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const deepMapKeys = require(`deep-map-keys`) const _ = require(`lodash`) const { createRemoteFileNode } = require(`gatsby-source-filesystem`) @@ -14,7 +14,11 @@ const restrictedNodeFields = [`id`, `children`, `parent`, `fields`, `internal`] * * @param {any} str */ -const digest = str => createContentDigest(str) +const digest = str => + crypto + .createHash(`md5`) + .update(str) + .digest(`hex`) /** * Validate the GraphQL naming convetions & protect specific fields. From 90b54c5dab3643f5b2d27d5114e81813add24824 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:48:57 -0600 Subject: [PATCH 56/63] revert gatsby-transformer-csv --- .../__snapshots__/gatsby-node.js.snap | 22 +++++++++---------- .../src/__tests__/gatsby-node.js | 3 ++- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap index 97189dfe5d17c..b11a568f3ea57 100644 --- a/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap +++ b/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -9,7 +9,7 @@ Array [ "funny": "yup", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "5005aee6b2557974cae0aabc712e125a", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -22,7 +22,7 @@ Array [ "funny": "nope", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "0af87da7e572c2d6d8493bcd642c2c78", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -41,7 +41,7 @@ Array [ "funny": "yup", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "5005aee6b2557974cae0aabc712e125a", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -70,7 +70,7 @@ false,\\"nope\\"", "funny": "nope", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "0af87da7e572c2d6d8493bcd642c2c78", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -103,7 +103,7 @@ Array [ "field2": "funny", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "49605f5e32a2250d7ca740c5d2e79784", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -116,7 +116,7 @@ Array [ "field2": "yup", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "2bf0bc0e8539ef5f6bbcf67c8b733233", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -129,7 +129,7 @@ Array [ "field2": "nope", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "f5f617ce4243d639de924543dec5600e", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -148,7 +148,7 @@ Array [ "field2": "funny", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "49605f5e32a2250d7ca740c5d2e79784", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -177,7 +177,7 @@ false,nope", "field2": "yup", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "2bf0bc0e8539ef5f6bbcf67c8b733233", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -206,7 +206,7 @@ false,nope", "field2": "nope", "id": "uuid-from-gatsby", "internal": Object { - "contentDigest": "f5f617ce4243d639de924543dec5600e", + "contentDigest": "contentDigest", "type": "TestCsv", }, "parent": "whatever", @@ -228,4 +228,4 @@ false,nope", }, ], ] -`; +`; \ No newline at end of file diff --git a/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js b/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js index 9b41bd4ee74ad..0dadd5ac6b9f3 100644 --- a/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js +++ b/packages/gatsby-transformer-csv/src/__tests__/gatsby-node.js @@ -2,7 +2,6 @@ const Promise = require(`bluebird`) const json2csv = require(`json2csv`) const { onCreateNode } = require(`../gatsby-node`) -const createContentDigest = require(`../../../gatsby/src/utils/create-content-digest`) describe(`Process nodes correctly`, () => { const node = { @@ -31,6 +30,7 @@ describe(`Process nodes correctly`, () => { const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) + const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode({ node, @@ -54,6 +54,7 @@ describe(`Process nodes correctly`, () => { const actions = { createNode, createParentChildLink } const createNodeId = jest.fn() createNodeId.mockReturnValue(`uuid-from-gatsby`) + const createContentDigest = jest.fn().mockReturnValue(`contentDigest`) await onCreateNode( { From cf92d0e213161c4609e0e14c0d2417078da17c39 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:50:16 -0600 Subject: [PATCH 57/63] revert gatsby-transformer-documentationjs --- .../gatsby-transformer-documentationjs/src/gatsby-node.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-transformer-documentationjs/src/gatsby-node.js b/packages/gatsby-transformer-documentationjs/src/gatsby-node.js index c93a57e17d3b4..18bb7f8c01f4b 100644 --- a/packages/gatsby-transformer-documentationjs/src/gatsby-node.js +++ b/packages/gatsby-transformer-documentationjs/src/gatsby-node.js @@ -1,6 +1,10 @@ const documentation = require(`documentation`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) -const digest = str => createContentDigest(str) +const crypto = require(`crypto`) +const digest = str => + crypto + .createHash(`md5`) + .update(str) + .digest(`hex`) const remark = require(`remark`) const _ = require(`lodash`) const Prism = require(`prismjs`) From 18c86b1e616b6419624a4e27ebb89d9e6011ab81 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:51:30 -0600 Subject: [PATCH 58/63] revert gatsby-transformer-excel --- packages/gatsby-transformer-excel/src/gatsby-node.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-transformer-excel/src/gatsby-node.js b/packages/gatsby-transformer-excel/src/gatsby-node.js index 26f89dab54ca5..eb537e6e24719 100644 --- a/packages/gatsby-transformer-excel/src/gatsby-node.js +++ b/packages/gatsby-transformer-excel/src/gatsby-node.js @@ -1,7 +1,7 @@ const XLSX = require(`xlsx`) const fs = require(`fs-extra`) const _ = require(`lodash`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) // read files as `binary` from file system function _loadNodeContent(fileNode, fallback) { @@ -46,7 +46,10 @@ async function onCreateNode( if (_.isArray(parsedContent)) { const csvArray = parsedContent.map((obj, i) => { const objStr = JSON.stringify(obj) - const contentDigest = createContentDigest(objStr) + const contentDigest = crypto + .createHash(`md5`) + .update(objStr) + .digest(`hex`) return { ...obj, @@ -72,7 +75,10 @@ async function onCreateNode( const shObj = { name: n, idx: idx } const shStr = JSON.stringify(shObj) - const contentDigest = createContentDigest(shStr) + const contentDigest = crypto + .createHash(`md5`) + .update(shStr) + .digest(`hex`) const z = { id: createNodeId(`${node.id} [${idx}] >>> ${node.extension}`), From da908599ac134812d81d3064c6de4bd41a6d068e Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:52:33 -0600 Subject: [PATCH 59/63] revert gatsby-transformer-hjson --- packages/gatsby-transformer-hjson/src/gatsby-node.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-transformer-hjson/src/gatsby-node.js b/packages/gatsby-transformer-hjson/src/gatsby-node.js index 1e08091cbcee2..b300604449e04 100644 --- a/packages/gatsby-transformer-hjson/src/gatsby-node.js +++ b/packages/gatsby-transformer-hjson/src/gatsby-node.js @@ -1,5 +1,5 @@ const _ = require(`lodash`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const path = require(`path`) const HJSON = require(`hjson`) @@ -8,7 +8,10 @@ async function onCreateNode({ node, actions, loadNodeContent, createNodeId }) { function transformObject(obj, id, type) { const objStr = JSON.stringify(obj) - const contentDigest = createContentDigest(objStr) + const contentDigest = crypto + .createHash(`md5`) + .update(objStr) + .digest(`hex`) const jsonNode = { ...obj, id, From e005647cee94ef09e465d77dbb46c39c1f3c5fe9 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 16:59:09 -0600 Subject: [PATCH 60/63] revert other packages --- .../src/gatsby-node.js | 7 +++++-- .../src/gatsby-node.js | 7 +++++-- .../gatsby-transformer-react-docgen/src/on-node-create.js | 8 ++++++-- packages/gatsby-transformer-screenshot/lambda/index.js | 7 +++++-- packages/gatsby-transformer-sqip/src/generate-sqip.js | 7 +++++-- packages/gatsby-transformer-toml/src/gatsby-node.js | 7 +++++-- 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/packages/gatsby-transformer-javascript-frontmatter/src/gatsby-node.js b/packages/gatsby-transformer-javascript-frontmatter/src/gatsby-node.js index 9921ab96c9d65..cb762e81aaa4a 100644 --- a/packages/gatsby-transformer-javascript-frontmatter/src/gatsby-node.js +++ b/packages/gatsby-transformer-javascript-frontmatter/src/gatsby-node.js @@ -1,5 +1,5 @@ const _ = require(`lodash`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const babylon = require(`@babel/parser`) const traverse = require(`@babel/traverse`).default @@ -108,7 +108,10 @@ async function onCreateNode({ node, getNode, actions, loadNodeContent }) { } const objStr = JSON.stringify(node) - const contentDigest = createContentDigest(objStr) + const contentDigest = crypto + .createHash(`md5`) + .update(objStr) + .digest(`hex`) const nodeData = { id: `${node.id} >>> JavascriptFrontmatter`, diff --git a/packages/gatsby-transformer-javascript-static-exports/src/gatsby-node.js b/packages/gatsby-transformer-javascript-static-exports/src/gatsby-node.js index f890c4a74340a..41532df6b4a7d 100644 --- a/packages/gatsby-transformer-javascript-static-exports/src/gatsby-node.js +++ b/packages/gatsby-transformer-javascript-static-exports/src/gatsby-node.js @@ -1,5 +1,5 @@ const _ = require(`lodash`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const babylon = require(`@babel/parser`) const traverse = require(`@babel/traverse`).default @@ -127,7 +127,10 @@ async function onCreateNode({ } } finally { const objStr = JSON.stringify(node) - const contentDigest = createContentDigest(objStr) + const contentDigest = crypto + .createHash(`md5`) + .update(objStr) + .digest(`hex`) const nodeData = { id: createNodeId(`${node.id} >>> JSFrontmatter`), diff --git a/packages/gatsby-transformer-react-docgen/src/on-node-create.js b/packages/gatsby-transformer-react-docgen/src/on-node-create.js index 3236ba2d40087..782b5f581bc74 100644 --- a/packages/gatsby-transformer-react-docgen/src/on-node-create.js +++ b/packages/gatsby-transformer-react-docgen/src/on-node-create.js @@ -1,7 +1,11 @@ -import createContentDigest from "../../gatsby/src/utils/create-content-digest" +import crypto from "crypto" import parseMetadata from "./parse" -const digest = str => createContentDigest(str) +const digest = str => + crypto + .createHash(`md5`) + .update(str) + .digest(`hex`) const propsId = (parentId, name) => `${parentId}--ComponentProp-${name}` const descId = parentId => `${parentId}--ComponentDescription` diff --git a/packages/gatsby-transformer-screenshot/lambda/index.js b/packages/gatsby-transformer-screenshot/lambda/index.js index 15c74c93ec822..3bb3a76d1b092 100644 --- a/packages/gatsby-transformer-screenshot/lambda/index.js +++ b/packages/gatsby-transformer-screenshot/lambda/index.js @@ -1,6 +1,6 @@ const setup = require(`./starter-kit/setup`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const AWS = require(`aws-sdk`) const s3 = new AWS.S3({ @@ -56,7 +56,10 @@ exports.run = async (browser, url, width, height, fullPage) => { } const keyBase = `${url}-(${width},${height})` - const digest = createContentDigest(keyBase) + const digest = crypto + .createHash(`md5`) + .update(keyBase) + .digest(`hex`) const key = `${digest}.png` const screenshotUrl = `https://s3-${region}.amazonaws.com/${ diff --git a/packages/gatsby-transformer-sqip/src/generate-sqip.js b/packages/gatsby-transformer-sqip/src/generate-sqip.js index aef073fbb67e5..2149cd39ccc47 100644 --- a/packages/gatsby-transformer-sqip/src/generate-sqip.js +++ b/packages/gatsby-transformer-sqip/src/generate-sqip.js @@ -1,4 +1,4 @@ -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) const { resolve, parse } = require(`path`) const Debug = require(`debug`) @@ -30,7 +30,10 @@ module.exports = async function generateSqip(options) { mode, } - const optionsHash = createContentDigest(JSON.stringify(sqipOptions)) + const optionsHash = crypto + .createHash(`md5`) + .update(JSON.stringify(sqipOptions)) + .digest(`hex`) const cacheKey = `sqip-${name}-${optionsHash}` const cachePath = resolve(cacheDir, `${name}-${optionsHash}.svg`) diff --git a/packages/gatsby-transformer-toml/src/gatsby-node.js b/packages/gatsby-transformer-toml/src/gatsby-node.js index 0e9f1bc38149e..f4caa0f627769 100644 --- a/packages/gatsby-transformer-toml/src/gatsby-node.js +++ b/packages/gatsby-transformer-toml/src/gatsby-node.js @@ -1,6 +1,6 @@ const toml = require(`toml`) const _ = require(`lodash`) -const createContentDigest = require(`../../gatsby/src/utils/create-content-digest`) +const crypto = require(`crypto`) async function onCreateNode({ node, actions, loadNodeContent, createNodeId }) { const { createNode, createParentChildLink } = actions @@ -19,7 +19,10 @@ async function onCreateNode({ node, actions, loadNodeContent, createNodeId }) { // 1) More TOML files -> more types // 2) Different files with the same name creating conflicts const parsedContentStr = JSON.stringify(parsedContent) - const contentDigest = createContentDigest(parsedContentStr) + const contentDigest = crypto + .createHash(`md5`) + .update(parsedContentStr) + .digest(`hex`) const newNode = { ...parsedContent, From 2e5846c4275eaa8a5ca9ae5220885d82e2120045 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 21:20:34 -0600 Subject: [PATCH 61/63] revert some pkgs --- packages/gatsby-source-hacker-news/src/gatsby-node.js | 1 + .../src/__tests__/__snapshots__/gatsby-node.js.snap | 2 +- packages/gatsby/src/bootstrap/index.js | 7 +++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-source-hacker-news/src/gatsby-node.js b/packages/gatsby-source-hacker-news/src/gatsby-node.js index b8c47f8eccaf5..e53b2e2b06a9e 100644 --- a/packages/gatsby-source-hacker-news/src/gatsby-node.js +++ b/packages/gatsby-source-hacker-news/src/gatsby-node.js @@ -62,6 +62,7 @@ exports.sourceNodes = async ({ } } } + fragment commentsFragment on HackerNewsItem { id text diff --git a/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap b/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap index b11a568f3ea57..b42b670c9e4be 100644 --- a/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap +++ b/packages/gatsby-transformer-csv/src/__tests__/__snapshots__/gatsby-node.js.snap @@ -228,4 +228,4 @@ false,nope", }, ], ] -`; \ No newline at end of file +`; diff --git a/packages/gatsby/src/bootstrap/index.js b/packages/gatsby/src/bootstrap/index.js index dfe71c44c6581..6a8bbf075d514 100644 --- a/packages/gatsby/src/bootstrap/index.js +++ b/packages/gatsby/src/bootstrap/index.js @@ -4,7 +4,7 @@ const _ = require(`lodash`) const slash = require(`slash`) const fs = require(`fs-extra`) const md5File = require(`md5-file/promise`) -const createContentDigest = require(`../utils/create-content-digest`) +const crypto = require(`crypto`) const del = require(`del`) const path = require(`path`) const convertHrtime = require(`convert-hrtime`) @@ -158,7 +158,10 @@ module.exports = async (args: BootstrapArgs) => { md5File(`${program.directory}/gatsby-node.js`).catch(() => {}) ), // ignore as this file isn't required), ]) - const pluginsHash = createContentDigest(pluginVersions.concat(hashes)) + const pluginsHash = crypto + .createHash(`md5`) + .update(JSON.stringify(pluginVersions.concat(hashes))) + .digest(`hex`) let state = store.getState() const oldPluginsHash = state && state.status ? state.status.PLUGINS_HASH : `` From 66136a0218980d893176b25e08db431ca9311618 Mon Sep 17 00:00:00 2001 From: Samuel Cha Date: Sat, 9 Mar 2019 22:45:24 -0600 Subject: [PATCH 62/63] add gatsby --- packages/gatsby-source-filesystem/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json index 7168686b0c358..5ce83c449ddf8 100644 --- a/packages/gatsby-source-filesystem/package.json +++ b/packages/gatsby-source-filesystem/package.json @@ -13,6 +13,7 @@ "chokidar": "^2.1.2", "file-type": "^10.2.0", "fs-extra": "^5.0.0", + "gatsby": "^2.1.27", "got": "^7.1.0", "md5-file": "^3.1.1", "mime": "^2.2.0", @@ -36,7 +37,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": "^2.0.15" + "gatsby": "^2.0.0" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem", "scripts": { From fd1217d4a9fdd54274402ab8f688a0d19206fb16 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 23 Apr 2019 12:30:18 +0200 Subject: [PATCH 63/63] move gatsby to peerdeps --- packages/gatsby-source-filesystem/package.json | 3 +-- packages/gatsby-transformer-remark/package.json | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/gatsby-source-filesystem/package.json b/packages/gatsby-source-filesystem/package.json index 9ced229db570e..317dc597b5406 100644 --- a/packages/gatsby-source-filesystem/package.json +++ b/packages/gatsby-source-filesystem/package.json @@ -13,7 +13,6 @@ "chokidar": "2.1.2", "file-type": "^10.2.0", "fs-extra": "^5.0.0", - "gatsby": "^2.1.27", "got": "^7.1.0", "md5-file": "^3.1.1", "mime": "^2.2.0", @@ -37,7 +36,7 @@ ], "license": "MIT", "peerDependencies": { - "gatsby": "^2.0.0" + "gatsby": "^2.0.15" }, "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-filesystem", "scripts": { diff --git a/packages/gatsby-transformer-remark/package.json b/packages/gatsby-transformer-remark/package.json index 94bc8fcc69bff..9a6c76b576f87 100644 --- a/packages/gatsby-transformer-remark/package.json +++ b/packages/gatsby-transformer-remark/package.json @@ -9,7 +9,6 @@ "dependencies": { "@babel/runtime": "^7.0.0", "bluebird": "^3.5.0", - "gatsby": "^2.1.24", "gray-matter": "^4.0.0", "hast-util-raw": "^4.0.0", "hast-util-to-html": "^4.0.0",