From 4bef9d1ec44351032729f5b98ef22fc83d8f41e3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 17 Jun 2017 23:17:44 -0700 Subject: [PATCH] For Contentful, filter out unresolvable entries and create markdown text nodes (#1202) --- docs/docs/plugins.md | 1 + .../gatsby-plugin-offline/src/app-shell.js | 2 + .../gatsby-source-contentful/package.json | 1 + .../src/gatsby-node.js | 98 ++++++++++++++----- packages/gatsby/package.json | 1 + .../dev-404-page/raw_dev-404-page.js | 2 +- packages/gatsby/src/redux/index.js | 3 +- .../gatsby/src/schema/infer-graphql-type.js | 6 +- 8 files changed, 86 insertions(+), 28 deletions(-) diff --git a/docs/docs/plugins.md b/docs/docs/plugins.md index 8d6a606024a02..067ffd61cbd98 100644 --- a/docs/docs/plugins.md +++ b/docs/docs/plugins.md @@ -94,6 +94,7 @@ you can place the files in a `src` subfolder and build them to the plugin folder * [gatsby-remark-responsive-image](/docs/packages/gatsby-remark-responsive-image/) * [gatsby-remark-smartypants](/docs/packages/gatsby-remark-smartypants/) * [gatsby-sharp](/docs/packages/gatsby-sharp/) +* [gatsby-source-contentful](/docs/packages/gatsby-source-contentful/) * [gatsby-source-drupal](/docs/packages/gatsby-source-drupal/) * [gatsby-source-filesystem](/docs/packages/gatsby-source-filesystem/) * [gatsby-source-hacker-news](/docs/packages/gatsby-source-hacker-news/) diff --git a/packages/gatsby-plugin-offline/src/app-shell.js b/packages/gatsby-plugin-offline/src/app-shell.js index 3cfaf0a532868..abf79d38160a4 100644 --- a/packages/gatsby-plugin-offline/src/app-shell.js +++ b/packages/gatsby-plugin-offline/src/app-shell.js @@ -2,6 +2,8 @@ import React from "react" class AppShell extends React.Component { componentDidMount() { + // TODO check if page exists and if not, + // force a hard reload. window.___navigateTo(this.props.location.pathname) } diff --git a/packages/gatsby-source-contentful/package.json b/packages/gatsby-source-contentful/package.json index fc5c9f761a841..ca6667f25e848 100644 --- a/packages/gatsby-source-contentful/package.json +++ b/packages/gatsby-source-contentful/package.json @@ -17,6 +17,7 @@ "axios": "^0.16.1", "bluebird": "^3.4.6", "contentful": "^4.3.0", + "json-stringify-safe": "^5.0.1", "lodash": "^4.17.2" } } diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 12ba27e7cbcab..4d0812d10ce4f 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -1,5 +1,8 @@ const contentful = require(`contentful`) const crypto = require(`crypto`) +const stringify = require("json-stringify-safe") + +const digest = str => crypto.createHash(`md5`).update(str).digest(`hex`) const typePrefix = `contentful__` const conflictFieldPrefix = `contentful` @@ -58,6 +61,19 @@ exports.sourceNodes = async ( console.log(`assets fetched`, assets.items.length) console.timeEnd(`fetch Contentful data`) + // Create map of not resolvable ids so we can filter them out while creating + // links. + const notResolvable = new Map() + entryList.forEach(ents => { + if (ents.errors) { + ents.errors.forEach(error => { + if (error.sys.id === `notResolvable`) { + notResolvable.set(error.details.id, error.details) + } + }) + } + }) + const contentTypeItems = contentTypes.items // Build foreign reference map before starting to insert any nodes @@ -75,6 +91,11 @@ exports.sourceNodes = async ( entryItemFieldValue[0].sys.id ) { entryItemFieldValue.forEach(v => { + // Don't create link to an unresolvable field. + if (notResolvable.has(v.sys.id)) { + return + } + if (!foreignReferenceMap[v.sys.id]) { foreignReferenceMap[v.sys.id] = [] } @@ -87,7 +108,8 @@ exports.sourceNodes = async ( } else if ( entryItemFieldValue.sys && entryItemFieldValue.sys.type && - entryItemFieldValue.sys.id + entryItemFieldValue.sys.id && + !notResolvable.has(entryItemFieldValue.sys.id) ) { if (!foreignReferenceMap[entryItemFieldValue.sys.id]) { foreignReferenceMap[entryItemFieldValue.sys.id] = [] @@ -101,6 +123,26 @@ exports.sourceNodes = async ( }) }) + function createTextNode(node, text, createNode) { + const textNode = { + id: `${node.id}TextNode`, + parent: node.id, + children: [], + text, + internal: { + type: `ComponentDescription`, + mediaType: `text/x-markdown`, + content: text, + contentDigest: digest(text), + }, + } + + node.children = node.children.concat([textNode.id]) + createNode(textNode) + + return textNode.id + } + contentTypeItems.forEach((contentTypeItem, i) => { const contentTypeItemId = contentTypeItem.sys.id @@ -138,13 +180,17 @@ exports.sourceNodes = async ( ) { entryItemFields[ `${entryItemFieldKey}___NODE` - ] = entryItemFieldValue.map(v => v.sys.id) + ] = entryItemFieldValue + .filter(v => !notResolvable.has(v.sys.id)) + .map(v => v.sys.id) + delete entryItemFields[entryItemFieldKey] } } else if ( entryItemFieldValue.sys && entryItemFieldValue.sys.type && - entryItemFieldValue.sys.id + entryItemFieldValue.sys.id && + !notResolvable.has(entryItemFieldValue.sys.id) ) { entryItemFields[`${entryItemFieldKey}___NODE`] = entryItemFieldValue.sys.id @@ -167,30 +213,42 @@ exports.sourceNodes = async ( }) } - const entryNode = { + let entryNode = { id: entryItem.sys.id, parent: contentTypeItemId, children: [], - ...entryItemFields, internal: { type: `${makeTypeName(contentTypeItemId)}`, - content: JSON.stringify(entryItem), mediaType: `application/json`, }, } + // Replace text fields with text nodes so we can process their markdown + // into HTML. + Object.keys(entryItemFields).forEach(entryItemFieldKey => { + if (entryItemFieldKey === `text`) { + entryItemFields[`${entryItemFieldKey}___NODE`] = createTextNode( + entryNode, + entryItemFields[entryItemFieldKey], + createNode + ) + + delete entryItemFields[entryItemFieldKey] + } + }) + + entryNode = { ...entryItemFields, ...entryNode } + // Get content digest of node. - const contentDigest = crypto - .createHash(`md5`) - .update(JSON.stringify(entryNode)) - .digest(`hex`) + const contentDigest = digest(stringify(entryNode)) entryNode.internal.contentDigest = contentDigest return entryNode }) + // Create a node for each content type - const contentTypeItemStr = JSON.stringify(contentTypeItem) + const contentTypeItemStr = stringify(contentTypeItem) const contentTypeNode = { id: contentTypeItemId, @@ -201,16 +259,12 @@ exports.sourceNodes = async ( description: contentTypeItem.description, internal: { type: `${makeTypeName(`ContentType`)}`, - content: contentTypeItemStr, - mediaType: `application/json`, + mediaType: `text/x-contentful`, }, } // Get content digest of node. - const contentDigest = crypto - .createHash(`md5`) - .update(JSON.stringify(contentTypeNode)) - .digest(`hex`) + const contentDigest = digest(stringify(contentTypeNode)) contentTypeNode.internal.contentDigest = contentDigest @@ -222,7 +276,7 @@ exports.sourceNodes = async ( assets.items.forEach(assetItem => { // Create a node for each asset. They may be referenced by Entries - const assetItemStr = JSON.stringify(assetItem) + const assetItemStr = stringify(assetItem) const assetNode = { id: assetItem.sys.id, @@ -231,16 +285,12 @@ exports.sourceNodes = async ( ...assetItem.fields, internal: { type: `${makeTypeName(`Asset`)}`, - content: assetItemStr, - mediaType: `application/json`, + mediaType: `text/x-contentful`, }, } // Get content digest of node. - const contentDigest = crypto - .createHash(`md5`) - .update(JSON.stringify(assetNode)) - .digest(`hex`) + const contentDigest = digest(stringify(assetNode)) assetNode.internal.contentDigest = contentDigest diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json index 7903209439f25..de9da46d2939f 100644 --- a/packages/gatsby/package.json +++ b/packages/gatsby/package.json @@ -55,6 +55,7 @@ "is-relative-url": "^2.0.0", "joi": "^9.1.1", "json-loader": "^0.5.2", + "json-stringify-safe": "^5.0.1", "json5": "^0.5.0", "loader-utils": "^0.2.16", "lodash": "^4.17.4", diff --git a/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js b/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js index 4095d9de7163e..818ed58f22c8b 100644 --- a/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js +++ b/packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js @@ -35,7 +35,7 @@ class Dev404Page extends React.Component {

Pages ({this.props.data.allSitePage.totalCount})

} diff --git a/packages/gatsby/src/redux/index.js b/packages/gatsby/src/redux/index.js index c9759c5d956e0..398269eae9ea5 100644 --- a/packages/gatsby/src/redux/index.js +++ b/packages/gatsby/src/redux/index.js @@ -4,6 +4,7 @@ const _ = require(`lodash`) const { composeWithDevTools } = require(`remote-redux-devtools`) const fs = require(`fs`) const EventEmitter = require(`eventemitter2`) +const stringify = require(`json-stringify-safe`) // Create event emitter for actions const emitter = new EventEmitter() @@ -47,7 +48,7 @@ const saveState = _.debounce(state => { const pickedState = _.pick(state, [`nodes`, `status`, `pageDataDependencies`]) fs.writeFile( `${process.cwd()}/.cache/redux-state.json`, - JSON.stringify(pickedState, null, 2), + stringify(pickedState, null, 2), () => {} ) }, 1000) diff --git a/packages/gatsby/src/schema/infer-graphql-type.js b/packages/gatsby/src/schema/infer-graphql-type.js index 35bb526d5224b..2d0ca7c1776f0 100644 --- a/packages/gatsby/src/schema/infer-graphql-type.js +++ b/packages/gatsby/src/schema/infer-graphql-type.js @@ -253,6 +253,7 @@ function inferFromFieldName(value, selector, types): GraphQLFieldConfig<*, *> { const [, , linkedField] = key.split(`___`) const linkedNode = findLinkedNode(value, linkedField) + invariant( linkedNode, oneLine` @@ -464,9 +465,10 @@ export function inferObjectStructureFromNodes({ shouldInferFile(nodes, nextSelector, value) ) { inferredField = inferFromUri(key, types) + } - // Finally our automatic inference of field value type. - } else { + // Finally our automatic inference of field value type. + if (!inferredField) { inferredField = inferGraphQLType({ nodes, types,