test(embed): add tests for data embeds

see #593

src/defaults/html.pipe.js

@@ -41,6 +41,7 @@ const addHeaders = require('../html/add-headers');
 const timing = require('../utils/timing');
 const sanitize = require('../html/sanitize');
 const removeHlxProps = require('../html/removeHlxProps');
+const dataEmbeds = require('../html/fetch-data');
 
 /* eslint newline-per-chained-call: off */
 
@@ -67,6 +68,7 @@ const htmlpipe = (cont, context, action) => {
     .use(parse).expose('parse')
     .use(parseFrontmatter)
     .use(embeds)
+    .use(dataEmbeds)
     .use(smartypants)
     .use(iconize)
     .use(sections)

src/html/fetch-data.js

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+const { selectAll } = require('unist-util-select');
+const {
+  pipe, map, uniq, list,
+} = require('ferrum');
+
+function fetch({ content: { mdast } }, { downloader, secrets: { DATA_EMBED_SERVICE } }) {
+  const fetches = pipe(
+    selectAll('dataEmbed', mdast),
+    map((node) => node.url),
+    uniq,
+    map((url) => {
+      console.log(`${DATA_EMBED_SERVICE}/${url}`);
+      return downloader.fetch({
+        uri: `${DATA_EMBED_SERVICE}/${url}`,
+        id: `dataEmbed:${url}`,
+      });
+    }),
+  );
+  list(fetches);
+}
+
+module.exports = fetch;

src/schemas/secrets.schema.json

@@ -38,12 +38,12 @@
     "EMBED_WHITELIST": {
       "type": "string",
       "description": "Comma-separated list of allowed hostnames for embeds. Supports `*.example.com` as a subdomain wildcard. Use `*` to allow all embeds (potentially insecure and conflicting with `DATA_EMBED_WHITELIST`)",
-      "default": "www.youtube.com, spark.adobe.com, unsplash.com/photos, soundcloud.com"
+      "default": "www.youtube.com, spark.adobe.com, unsplash.com, soundcloud.com"
     },
     "DATA_EMBED_WHITELIST": {
       "type": "string",
       "description": "Comma-separated list of allowed hostnames for data embeds. Supports `*.example.com` as a subdomain wildcard. Use `*` to allow all embeds (potentially insecure and conflicting with `EMBED_WHITELIST`)",
-      "default": "docs.google.com/spreadsheets"
+      "default": "docs.google.com"
     },
     "EMBED_SERVICE": {
       "type": "string",

test/testDataEmbeds.js

@@ -0,0 +1,134 @@
+/*
+ * Copyright 2018 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+/* eslint-env mocha */
+const assert = require('assert');
+const { dom: { assertEquivalentNode } } = require('@adobe/helix-shared');
+const { logging } = require('@adobe/helix-testutils');
+const nock = require('nock');
+const { JSDOM } = require('jsdom');
+const { pipe } = require('../src/defaults/html.pipe.js');
+const coerce = require('../src/utils/coerce-secrets');
+const Downloader = require('../src/utils/Downloader.js');
+
+const params = {
+  path: '/hello.md',
+  __ow_method: 'get',
+  owner: 'trieloff',
+  __ow_headers: {
+    'X-Forwarded-Port': '443',
+    'X-CDN-Request-Id': '2a208a89-e071-44cf-aee9-220880da4c1e',
+    'Fastly-Client': '1',
+    'X-Forwarded-Host': 'runtime.adobe.io',
+    'Upgrade-Insecure-Requests': '1',
+    Host: 'controller-a',
+    Connection: 'close',
+    'Fastly-SSL': '1',
+    'X-Request-Id': 'RUss5tPdgOfw74a68aNc24FeTipGpVfW',
+    'X-Branch': 'master',
+    'Accept-Language': 'en-US, en;q=0.9, de;q=0.8',
+    'X-Forwarded-Proto': 'https',
+    'Fastly-Orig-Accept-Encoding': 'gzip',
+    'X-Varnish': '267021320',
+    DNT: '1',
+    'X-Forwarded-For':
+      '192.147.117.11, 157.52.92.27, 23.235.46.33, 10.64.221.107',
+    'X-Host': 'www.primordialsoup.life',
+    Accept:
+      'text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, image/apng, */*;q=0.8',
+    'X-Real-IP': '10.64.221.107',
+    'X-Forwarded-Server': 'cache-lcy19249-LCY, cache-iad2127-IAD',
+    'Fastly-Client-IP': '192.147.117.11',
+    'Perf-Br-Req-In': '1529585370.116',
+    'X-Timer': 'S1529585370.068237,VS0,VS0',
+    'Fastly-FF':
+      'dc/x3e9z8KMmlHLQr8BEvVMmTcpl3y2YY5y6gjSJa3g=!LCY!cache-lcy19249-LCY, dc/x3e9z8KMmlHLQr8BEvVMmTcpl3y2YY5y6gjSJa3g=!LCY!cache-lcy19227-LCY, dc/x3e9z8KMmlHLQr8BEvVMmTcpl3y2YY5y6gjSJa3g=!IAD!cache-iad2127-IAD, dc/x3e9z8KMmlHLQr8BEvVMmTcpl3y2YY5y6gjSJa3g=!IAD!cache-iad2133-IAD',
+    'Accept-Encoding': 'gzip',
+    'User-Agent':
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36',
+  },
+  repo: 'soupdemo',
+  ref: 'master',
+  selector: 'md',
+};
+
+const secrets = {
+  REPO_RAW_ROOT: 'https://raw.githubusercontent.com/',
+  EMBED_WHITELIST: '*.youtube.com',
+  DATA_EMBED_WHITELIST: 'docs.google.com',
+};
+
+const logger = logging.createTestLogger({
+  // tune this for debugging
+  level: 'debug',
+});
+
+
+const crequest = {
+  extension: 'html',
+  url: '/test/test.html',
+};
+
+describe('Integration Test with Data Embeds', () => {
+  afterEach(() => {
+    nock.restore();
+  });
+
+  beforeEach(() => {
+    nock.restore();
+    nock.activate();
+    nock.cleanAll();
+  });
+
+  it('html.pipe processes data embeds', async () => {
+    const action = coerce({
+      request: { params },
+      secrets,
+      logger,
+    });
+
+    const context = {
+      request: crequest,
+      content: {
+        body: `Hello World
+Here comes a data embed.
+
+https://docs.google.com/spreadsheets/d/e/2PACX-1vQ78BeYUV4gFee4bSxjN8u86aV853LGYZlwv1jAUMZFnPn5TnIZteDJwjGr2GNu--zgnpTY1E_KHXcF/pubhtml
+
+![Easy!](easy.png)
+`,
+      },
+    };
+
+    action.downloader = new Downloader(context, action, { forceHttp1: true });
+
+    const result = await pipe(
+      (mycontext) => {
+        mycontext.response = { status: 201, body: mycontext.content.document.body.innerHTML };
+      },
+      context,
+      action,
+    );
+
+    assert.equal(result.response.status, 201);
+    assert.equal(result.response.headers['Content-Type'], 'text/html');
+    assertEquivalentNode(
+      result.response.document.body,
+      new JSDOM(`
+        <p>Hello World
+        Here comes an embed.</p>
+        <esi:include src="https://example-embed-service.com/https://www.youtube.com/watch?v=KOxbO0EI4MA"></esi:include>
+        <esi:remove><p><a href="https://www.youtube.com/watch?v=KOxbO0EI4MA">https://www.youtube.com/watch?v=KOxbO0EI4MA</a></p></esi:remove>
+        <p><img src="easy.png" alt="Easy!"></p>
+      `).window.document.body,
+    );
+  });
+});