feat(gatsby-source-drupal): port hot-fixes for Drupal/Gatsby customer…

… to package (#33419) (cherry picked from commit 1777f08)
gatsbyjs · Oct 5, 2021 · 13b0305 · 13b0305
1 parent e10b2b9
commit 13b0305
Show file tree

Hide file tree

Showing 6 changed files with 193 additions and 121 deletions.
diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/1593545806.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/1593545806.json
@@ -16,6 +16,25 @@
       "id": "does-not-exist",
       "type": "file--file"
     },
+    {
+      "data": {
+        "type": "paragraph--image_gallery_reusable",
+        "id": "paragraph-image-1",
+        "links": {
+            "self": {
+                "href": "https://sc-cms-dev.psu.edu/jsonapi/paragraph/image_gallery_reusable/6cf00353-86f9-4a9d-b85a-35f3b00d6937"
+            }
+        },
+        "relationships": {
+            "field_gallery": {
+                "data": {
+                    "type": "node--article",
+                    "id": "article-1"
+                }
+            }
+        }
+      }
+    },
     {
       "jsonapi": {
         "version": "1.0",

diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/image-gallary.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/image-gallary.json
@@ -0,0 +1,30 @@
+{
+  "jsonapi": {
+    "version": "1.0",
+    "meta": {
+      "links": {
+        "self": {
+          "href": "http:\/\/jsonapi.org\/format\/1.0\/"
+        }
+      }
+    }
+  },
+  "data": [
+  {
+      "type": "paragraph--image_gallery_reusable",
+      "id": "paragraph-image-1",
+      "links": {
+          "self": {
+              "href": "https://sc-cms-dev.psu.edu/jsonapi/paragraph/image_gallery_reusable/6cf00353-86f9-4a9d-b85a-35f3b00d6937"
+          }
+      },
+      "relationships": {
+          "field_gallery": {
+              "data": {
+                  "type": "node--article",
+                  "id": "article-2"
+              }
+          }
+    }
+  }
+  ]}
diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/jsonapi.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/jsonapi.json
@@ -5,6 +5,7 @@
     "file--file": "http://fixture/jsonapi/file/file",
     "node--article": "http://fixture/jsonapi/node/article",
     "paragraph--text": "http://fixture/jsonapi/paragraph/text",
+    "paragraph--image": "http://fixture/jsonapi/paragraph/image-gallary",
     "taxonomy_term--tags": {
       "href": "http://fixture/jsonapi/taxonomy_term/tags"
     },

diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js
@@ -48,6 +48,7 @@ describe(`gatsby-source-drupal`, () => {
   }
   const reporter = {
     info: jest.fn(),
+    warn: jest.fn(),
     verbose: jest.fn(),
     activityTimer: jest.fn(() => activity),
     log: jest.fn(),
@@ -125,6 +126,10 @@ describe(`gatsby-source-drupal`, () => {
     expect(
       nodes[createNodeId(`und.article-3`)].relationships.field_main_image___NODE
     ).toEqual(createNodeId(`und.file-1`))
+
+    expect(nodes[createNodeId(`und.paragraph-image-1`)].relationships).toEqual({
+      field_gallery___NODE: createNodeId(`und.article-2`),
+    })
   })
 
   it(`Handles 1:N relationship`, () => {
@@ -493,6 +498,11 @@ describe(`gatsby-source-drupal`, () => {
         expect(nodes[createNodeId(`und.article-3`)].relationships).toEqual({
           field_tags___NODE: [createNodeId(`und.tag-2`)],
         })
+        expect(
+          nodes[createNodeId(`und.paragraph-image-1`)].relationships
+        ).toEqual({
+          field_gallery___NODE: createNodeId(`und.article-1`),
+        })
         expect(
           nodes[createNodeId(`und.article-2`)].relationships
             .field_secondary_image___NODE
@@ -593,20 +603,17 @@ describe(`gatsby-source-drupal`, () => {
       it(`during refresh webhook handling`, async () => {
         expect.assertions(5)
 
-        try {
-          await sourceNodes(
-            {
-              ...args,
-              webhookBody: {
-                malformattedPayload: true,
-              },
+        await sourceNodes(
+          {
+            ...args,
+            webhookBody: {
+              malformattedPayload: true,
             },
-            { baseUrl }
-          )
-        } catch (e) {
-          expect(e).toBeTruthy()
-        }
+          },
+          { baseUrl }
+        )
 
+        expect(reporter.warn).toHaveBeenCalledTimes(1)
         expect(reporter.activityTimer).toHaveBeenCalledTimes(1)
         expect(reporter.activityTimer).toHaveBeenNthCalledWith(
           1,

diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js
@@ -67,12 +67,12 @@ async function worker([url, options]) {
     ...options,
   })
 
-  httpSpan.setTag(SemanticAttributes.HTTP_STATUS_CODE, response.statusCode)
+  httpSpan.setTag(SemanticAttributes.HTTP_STATUS_CODE, response?.statusCode)
   httpSpan.setTag(SemanticAttributes.HTTP_METHOD, `GET`)
-  httpSpan.setTag(SemanticAttributes.NET_PEER_IP, response.ip)
+  httpSpan.setTag(SemanticAttributes.NET_PEER_IP, response?.ip)
   httpSpan.setTag(
     SemanticAttributes.HTTP_RESPONSE_CONTENT_LENGTH,
-    response.rawBody?.length
+    response?.rawBody?.length
   )
 
   httpSpan.finish()
@@ -176,9 +176,7 @@ exports.sourceNodes = async (
         reporter.warn(
           `The webhook body was malformed
 
-${JSON.stringify(webhookBody, null, 4)}
-
-          `
+${JSON.stringify(webhookBody, null, 4)}`
         )
 
         changesActivity.end()
@@ -245,127 +243,137 @@ ${JSON.stringify(webhookBody, null, 4)}
     fastBuildsSpan.setTag(`sourceNodes.fetch.type`, `delta`)
 
     const lastFetched =
-      store.getState().status.plugins?.[`gatsby-source-drupal`]?.lastFetched ??
-      0
+      store.getState().status.plugins?.[`gatsby-source-drupal`]?.lastFetched
 
     reporter.verbose(
       `[gatsby-source-drupal]: value of lastFetched for fastbuilds "${lastFetched}"`
     )
 
-    const drupalFetchIncrementalActivity = reporter.activityTimer(
-      `Fetch incremental changes from Drupal`,
-      { parentSpan: fastBuildsSpan }
-    )
     let requireFullRebuild = false
 
-    drupalFetchIncrementalActivity.start()
+    // lastFetched isn't set so do a full rebuild.
+    if (!lastFetched) {
+      setPluginStatus({ lastFetched: new Date().getTime() })
+      requireFullRebuild = true
+    } else {
+      const drupalFetchIncrementalActivity = reporter.activityTimer(
+        `Fetch incremental changes from Drupal`,
+        { parentSpan: fastBuildsSpan }
+      )
 
-    try {
-      // Hit fastbuilds endpoint with the lastFetched date.
-      const res = await requestQueue.push([
-        urlJoin(baseUrl, `gatsby-fastbuilds/sync/`, lastFetched.toString()),
-        {
-          username: basicAuth.username,
-          password: basicAuth.password,
-          headers,
-          searchParams: params,
-          responseType: `json`,
-          parentSpan: fastBuildsSpan,
-        },
-      ])
-
-      // Fastbuilds returns a -1 if:
-      // - the timestamp has expired
-      // - if old fastbuild logs were purged
-      // - it's been a really long time since you synced so you just do a full fetch.
-      if (res.body.status === -1) {
-        // The incremental data is expired or this is the first fetch.
-        reporter.info(`Unable to pull incremental data changes from Drupal`)
-        setPluginStatus({ lastFetched: res.body.timestamp })
-        requireFullRebuild = true
-      } else {
-        const touchNodesSpan = tracer.startSpan(`sourceNodes.touchNodes`, {
-          childOf: fastBuildsSpan,
-        })
-        touchNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)
-
-        // Touch nodes so they are not garbage collected by Gatsby.
-        let touchCount = 0
-        getNodes().forEach(node => {
-          if (node.internal.owner === `gatsby-source-drupal`) {
-            touchCount += 1
-            touchNode(node)
+      drupalFetchIncrementalActivity.start()
+
+      try {
+        // Hit fastbuilds endpoint with the lastFetched date.
+        const res = await requestQueue.push([
+          urlJoin(baseUrl, `gatsby-fastbuilds/sync/`, lastFetched.toString()),
+          {
+            username: basicAuth.username,
+            password: basicAuth.password,
+            headers,
+            searchParams: params,
+            responseType: `json`,
+            parentSpan: fastBuildsSpan,
+          },
+        ])
+
+        // Fastbuilds returns a -1 if:
+        // - the timestamp has expired
+        // - if old fastbuild logs were purged
+        // - it's been a really long time since you synced so you just do a full fetch.
+        if (res.body.status === -1) {
+          // The incremental data is expired or this is the first fetch.
+          reporter.info(`Unable to pull incremental data changes from Drupal`)
+          setPluginStatus({ lastFetched: res.body.timestamp })
+          requireFullRebuild = true
+        } else {
+          // Touch nodes so they are not garbage collected by Gatsby.
+          if (initialSourcing) {
+            const touchNodesSpan = tracer.startSpan(`sourceNodes.touchNodes`, {
+              childOf: fastBuildsSpan,
+            })
+            touchNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)
+            let touchCount = 0
+            getNodes().forEach(node => {
+              if (node.internal.owner === `gatsby-source-drupal`) {
+                touchCount += 1
+                touchNode(node)
+              }
+            })
+            touchNodesSpan.setTag(`sourceNodes.touchNodes.count`, touchCount)
+            touchNodesSpan.finish()
           }
-        })
-        touchNodesSpan.setTag(`sourceNodes.touchNodes.count`, touchCount)
-        touchNodesSpan.finish()
 
-        const createNodesSpan = tracer.startSpan(`sourceNodes.createNodes`, {
-          childOf: parentSpan,
-        })
-        createNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)
-        createNodesSpan.setTag(`sourceNodes.fetch.type`, `delta`)
-        createNodesSpan.setTag(
-          `sourceNodes.createNodes.count`,
-          res.body.entities?.length
-        )
+          const createNodesSpan = tracer.startSpan(`sourceNodes.createNodes`, {
+            childOf: parentSpan,
+          })
+          createNodesSpan.setTag(`plugin`, `gatsby-source-drupal`)
+          createNodesSpan.setTag(`sourceNodes.fetch.type`, `delta`)
+          createNodesSpan.setTag(
+            `sourceNodes.createNodes.count`,
+            res.body.entities?.length
+          )
 
-        // Process sync data from Drupal.
-        const nodesToSync = res.body.entities
-        for (const nodeSyncData of nodesToSync) {
-          if (nodeSyncData.action === `delete`) {
-            handleDeletedNode({
-              actions,
-              getNode,
-              node: nodeSyncData,
-              createNodeId,
-              createContentDigest,
-              entityReferenceRevisions,
-            })
-          } else {
-            // The data could be a single Drupal entity or an array of Drupal
-            // entities to update.
-            let nodesToUpdate = nodeSyncData.data
-            if (!Array.isArray(nodeSyncData.data)) {
-              nodesToUpdate = [nodeSyncData.data]
-            }
+          // Process sync data from Drupal.
+          const nodesToSync = res.body.entities
+          for (const nodeSyncData of nodesToSync) {
+            if (nodeSyncData.action === `delete`) {
+              handleDeletedNode({
+                actions,
+                getNode,
+                node: nodeSyncData,
+                createNodeId,
+                createContentDigest,
+                entityReferenceRevisions,
+              })
+            } else {
+              // The data could be a single Drupal entity or an array of Drupal
+              // entities to update.
+              let nodesToUpdate = nodeSyncData.data
+              if (!Array.isArray(nodeSyncData.data)) {
+                nodesToUpdate = [nodeSyncData.data]
+              }
 
-            for (const nodeToUpdate of nodesToUpdate) {
-              await handleWebhookUpdate(
-                {
-                  nodeToUpdate,
-                  actions,
-                  cache,
-                  createNodeId,
-                  createContentDigest,
-                  getCache,
-                  getNode,
-                  reporter,
-                  store,
-                  languageConfig,
-                },
-                pluginOptions
-              )
+              for (const nodeToUpdate of nodesToUpdate) {
+                await handleWebhookUpdate(
+                  {
+                    nodeToUpdate,
+                    actions,
+                    cache,
+                    createNodeId,
+                    createContentDigest,
+                    getCache,
+                    getNode,
+                    reporter,
+                    store,
+                    languageConfig,
+                  },
+                  pluginOptions
+                )
+              }
             }
           }
+
+          createNodesSpan.finish()
+          setPluginStatus({ lastFetched: res.body.timestamp })
         }
+      } catch (e) {
+        gracefullyRethrow(drupalFetchIncrementalActivity, e)
 
-        createNodesSpan.finish()
-        setPluginStatus({ lastFetched: res.body.timestamp })
+        drupalFetchIncrementalActivity.end()
+        fastBuildsSpan.finish()
+        return
       }
-    } catch (e) {
-      gracefullyRethrow(drupalFetchIncrementalActivity, e)
 
       drupalFetchIncrementalActivity.end()
       fastBuildsSpan.finish()
-      return
-    }
 
-    drupalFetchIncrementalActivity.end()
-    fastBuildsSpan.finish()
+      // We're now done with the initial (fastbuilds flavored) sourcing.
+      initialSourcing = false
 
-    if (!requireFullRebuild) {
-      return
+      if (!requireFullRebuild) {
+        return
+      }
     }
   }