From 7af93567006d7ab79d7552681c71558f5c4ae587 Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Wed, 17 Apr 2019 16:53:36 +0000 Subject: [PATCH 1/5] feat(embeds): detect internal embeds So far only the detection of local embeds has been implemented. As this requires access to some additional properties, many tests are broken for the time being See #267 --- src/html/find-embeds.js | 65 +- src/schemas/secrets.schema.json | 5 + test/fixtures/embeds.json | 1331 +++++++------------------------ test/fixtures/embeds.md | 47 +- test/testFindEmbeds.js | 87 +- 5 files changed, 505 insertions(+), 1030 deletions(-) diff --git a/src/html/find-embeds.js b/src/html/find-embeds.js index 56a893bd2..b94947df2 100644 --- a/src/html/find-embeds.js +++ b/src/html/find-embeds.js @@ -13,6 +13,7 @@ const map = require('unist-util-map'); const URI = require('uri-js'); const mm = require('micromatch'); +const p = require('path'); /** * Finds embeds like `video: https://www.youtube.com/embed/2Xc9gXyf2G4` @@ -26,6 +27,16 @@ function gatsbyEmbed(text) { return false; } +function internalGatsbyEmbed(text, base, contentext, resourceext) { + const matches = new RegExp(`^(markdown|html|embed): ?(.*(${contentext}|${resourceext}))$`) + .exec(text); + if (matches && matches[2]) { + const uri = URI.parse(URI.resolve(base, matches[2])); + return uri.reference === 'relative' && uri.path ? uri : false; + } + return false; +} + /** * Finds embeds that are single absolute links in a paragraph * @param {*} node An MDAST node @@ -40,6 +51,21 @@ function iaEmbed({ type, children }) { return false; } +function internalIaEmbed({ type, children }, base, contentext, resourceext) { + if (type === 'paragraph' + && children.length === 1 + && children[0].type === 'text' + && children[0].value + && !children[0].value.match(/\n/) + && !children[0].value.match(/ /) + && (children[0].value.endsWith(contentext) || (children[0].value.endsWith(resourceext))) + ) { + const uri = URI.parse(URI.resolve(base, children[0].value)); + return uri.reference === 'relative' && uri.path ? uri : false; + } + return false; +} + function imgEmbed({ type, children }) { if (type === 'paragraph' && children.length === 1 @@ -51,6 +77,18 @@ function imgEmbed({ type, children }) { return false; } +function internalImgEmbed({ type, children }, base, contentext, resourceext) { + if (type === 'paragraph' + && children.length === 1 + && children[0].type === 'image' + && URI.parse(children[0].url).reference === 'relative' + && (children[0].url.endsWith(contentext) || (children[0].url.endsWith(resourceext)))) { + const uri = URI.parse(URI.resolve(base, children[0].url)); + return uri.reference === 'relative' && uri.path ? uri : false; + } + return false; +} + function embed(uri, node, whitelist = '', warn = () => {}) { if (mm.some(uri.host, whitelist.split(','))) { const children = [Object.assign({}, node)]; @@ -65,7 +103,20 @@ function embed(uri, node, whitelist = '', warn = () => {}) { } } -function find({ content: { mdast } }, { logger, secrets: { EMBED_WHITELIST } }) { +function internalembed(uri, node, extension) { + const children = [Object.assign({}, node)]; + node.type = 'embed'; + node.children = children; + node.url = p.resolve(p.dirname(uri.path), p.basename(uri.path, p.extname(uri.path)) + extension); + if (node.value) { + delete node.value; + } +} + +function find({ content: { mdast }, request: { extension, url } }, + { logger, secrets: { EMBED_WHITELIST, EMBED_SELECTOR }, request: { path } }) { + const resourceext = `.${extension}`; + const contentext = p.extname(path); map(mdast, (node) => { if (node.type === 'inlineCode' && gatsbyEmbed(node.value)) { embed(gatsbyEmbed(node.value), node, EMBED_WHITELIST, logger.warn); @@ -73,6 +124,15 @@ function find({ content: { mdast } }, { logger, secrets: { EMBED_WHITELIST } }) embed(iaEmbed(node), node, EMBED_WHITELIST, logger.warn); } else if (node.type === 'paragraph' && imgEmbed(node)) { embed(imgEmbed(node), node, EMBED_WHITELIST, logger.warn); + } else if (node.type === 'inlineCode' + && internalGatsbyEmbed(node.value, url, contentext, resourceext)) { + internalembed(internalGatsbyEmbed(node.value, url, contentext, resourceext), node, `.${EMBED_SELECTOR}.${extension}`); + } else if (node.type === 'paragraph' + && internalIaEmbed(node, url, contentext, resourceext)) { + internalembed(internalIaEmbed(node, url, contentext, resourceext), node, `.${EMBED_SELECTOR}.${extension}`); + } else if (node.type === 'paragraph' + && internalImgEmbed(node, url, contentext, resourceext)) { + internalembed(internalImgEmbed(node, url, contentext, resourceext), node, `.${EMBED_SELECTOR}.${extension}`); } }); @@ -80,3 +140,6 @@ function find({ content: { mdast } }, { logger, secrets: { EMBED_WHITELIST } }) } module.exports = find; +module.exports.internalGatsbyEmbed = internalGatsbyEmbed; +module.exports.internalIaEmbed = internalIaEmbed; +module.exports.internalImgEmbed = internalImgEmbed; diff --git a/src/schemas/secrets.schema.json b/src/schemas/secrets.schema.json index e8165100d..daef4ca67 100644 --- a/src/schemas/secrets.schema.json +++ b/src/schemas/secrets.schema.json @@ -45,6 +45,11 @@ "description": "URL of an Embed Service that takes the appended URL and returns an embeddable HTML representation.", "default": "https://adobeioruntime.net/api/v1/web/helix/default/embed/" }, + "EMBED_SELECTOR": { + "type": "string", + "description": "Selector to be used when resolving internal embeds.", + "default": "embed" + }, "IMAGES_MIN_SIZE": { "type": "integer", "description": "Minimum physical width of responsive images to generate", diff --git a/test/fixtures/embeds.json b/test/fixtures/embeds.json index 068dacd72..73a147789 100644 --- a/test/fixtures/embeds.json +++ b/test/fixtures/embeds.json @@ -7,35 +7,9 @@ "children": [ { "type": "text", - "value": "Hello \"World\"", - "position": { - "start": { - "line": 1, - "column": 3, - "offset": 2 - }, - "end": { - "line": 1, - "column": 16, - "offset": 15 - }, - "indent": [] - } + "value": "Hello \"World\"" } - ], - "position": { - "start": { - "line": 1, - "column": 1, - "offset": 0 - }, - "end": { - "line": 1, - "column": 16, - "offset": 15 - }, - "indent": [] - } + ] }, { "type": "heading", @@ -43,262 +17,67 @@ "children": [ { "type": "text", - "value": "Gasby-Style-Embeds", - "position": { - "start": { - "line": 3, - "column": 4, - "offset": 20 - }, - "end": { - "line": 3, - "column": 22, - "offset": 38 - }, - "indent": [] - } + "value": "Gatsby-Style-Embeds" } - ], - "position": { - "start": { - "line": 3, - "column": 1, - "offset": 17 - }, - "end": { - "line": 3, - "column": 22, - "offset": 38 - }, - "indent": [] - } + ] }, { "type": "paragraph", "children": [ { "type": "embed", - "position": { - "start": { - "line": 5, - "column": 1, - "offset": 40 - }, - "end": { - "line": 5, - "column": 51, - "offset": 90 - }, - "indent": [] - }, "children": [ { "type": "inlineCode", - "value": "video: https://www.youtube.com/embed/2Xc9gXyf2G4", - "position": { - "start": { - "line": 5, - "column": 1, - "offset": 40 - }, - "end": { - "line": 5, - "column": 51, - "offset": 90 - }, - "indent": [] - } + "value": "video: https://www.youtube.com/embed/2Xc9gXyf2G4" } ], "url": "https://www.youtube.com/embed/2Xc9gXyf2G4" } - ], - "position": { - "start": { - "line": 5, - "column": 1, - "offset": 40 - }, - "end": { - "line": 5, - "column": 51, - "offset": 90 - }, - "indent": [] - } + ] }, { "type": "paragraph", "children": [ { "type": "text", - "value": "is an embed, but", - "position": { - "start": { - "line": 7, - "column": 1, - "offset": 92 - }, - "end": { - "line": 7, - "column": 17, - "offset": 108 - }, - "indent": [] - } + "value": "is an embed, but" } - ], - "position": { - "start": { - "line": 7, - "column": 1, - "offset": 92 - }, - "end": { - "line": 7, - "column": 17, - "offset": 108 - }, - "indent": [] - } + ] }, { "type": "paragraph", "children": [ { "type": "inlineCode", - "value": "this", - "position": { - "start": { - "line": 9, - "column": 1, - "offset": 110 - }, - "end": { - "line": 9, - "column": 7, - "offset": 116 - }, - "indent": [] - } + "value": "this" }, { "type": "text", - "value": " is simple inline code and ", - "position": { - "start": { - "line": 9, - "column": 7, - "offset": 116 - }, - "end": { - "line": 9, - "column": 34, - "offset": 143 - }, - "indent": [] - } + "value": " is simple inline code and " }, { "type": "inlineCode", - "value": "video: www.youtube.com", - "position": { - "start": { - "line": 9, - "column": 34, - "offset": 143 - }, - "end": { - "line": 9, - "column": 58, - "offset": 167 - }, - "indent": [] - } + "value": "video: www.youtube.com" }, { "type": "text", - "value": " isn't an embed either.", - "position": { - "start": { - "line": 9, - "column": 58, - "offset": 167 - }, - "end": { - "line": 9, - "column": 81, - "offset": 190 - }, - "indent": [] - } + "value": " isn't an embed either." } - ], - "position": { - "start": { - "line": 9, - "column": 1, - "offset": 110 - }, - "end": { - "line": 9, - "column": 81, - "offset": 190 - }, - "indent": [] - } + ] }, { "type": "paragraph", "children": [ { "type": "inlineCode", - "value": "video: http foo bar", - "position": { - "start": { - "line": 11, - "column": 1, - "offset": 192 - }, - "end": { - "line": 11, - "column": 25, - "offset": 216 - }, - "indent": [] - } + "value": "video: http foo bar" }, { "type": "text", - "value": " looks interesting, but doesn't work either.", - "position": { - "start": { - "line": 11, - "column": 25, - "offset": 216 - }, - "end": { - "line": 11, - "column": 69, - "offset": 260 - }, - "indent": [] - } + "value": " looks interesting, but doesn't work either." } - ], - "position": { - "start": { - "line": 11, - "column": 1, - "offset": 192 - }, - "end": { - "line": 11, - "column": 69, - "offset": 260 - }, - "indent": [] - } + ] }, { "type": "heading", @@ -306,35 +85,9 @@ "children": [ { "type": "text", - "value": "Link + Image-Style Embeds", - "position": { - "start": { - "line": 13, - "column": 4, - "offset": 265 - }, - "end": { - "line": 13, - "column": 29, - "offset": 290 - }, - "indent": [] - } + "value": "Link + Image-Style Embeds" } - ], - "position": { - "start": { - "line": 13, - "column": 1, - "offset": 262 - }, - "end": { - "line": 13, - "column": 29, - "offset": 290 - }, - "indent": [] - } + ] }, { "type": "embed", @@ -351,65 +104,13 @@ "type": "image", "title": null, "url": "http://img.youtube.com/vi/KOxbO0EI4MA/0.jpg", - "alt": "Audi R8", - "position": { - "start": { - "line": 15, - "column": 2, - "offset": 293 - }, - "end": { - "line": 15, - "column": 57, - "offset": 348 - }, - "indent": [] - } + "alt": "Audi R8" } - ], - "position": { - "start": { - "line": 15, - "column": 1, - "offset": 292 - }, - "end": { - "line": 15, - "column": 113, - "offset": 404 - }, - "indent": [] - } + ] } - ], - "position": { - "start": { - "line": 15, - "column": 1, - "offset": 292 - }, - "end": { - "line": 15, - "column": 113, - "offset": 404 - }, - "indent": [] - } + ] } ], - "position": { - "start": { - "line": 15, - "column": 1, - "offset": 292 - }, - "end": { - "line": 15, - "column": 113, - "offset": 404 - }, - "indent": [] - }, "url": "https://www.youtube.com/watch?v=KOxbO0EI4MA" }, { @@ -417,20 +118,7 @@ "children": [ { "type": "text", - "value": "is an embed, but ", - "position": { - "start": { - "line": 17, - "column": 1, - "offset": 406 - }, - "end": { - "line": 17, - "column": 18, - "offset": 423 - }, - "indent": [] - } + "value": "is an embed, but " }, { "type": "link", @@ -439,103 +127,25 @@ "children": [ { "type": "text", - "value": "this", - "position": { - "start": { - "line": 17, - "column": 19, - "offset": 424 - }, - "end": { - "line": 17, - "column": 23, - "offset": 428 - }, - "indent": [] - } + "value": "this" } - ], - "position": { - "start": { - "line": 17, - "column": 18, - "offset": 423 - }, - "end": { - "line": 17, - "column": 79, - "offset": 484 - }, - "indent": [] - } + ] }, { "type": "text", - "value": " is just a link and ", - "position": { - "start": { - "line": 17, - "column": 79, - "offset": 484 - }, - "end": { - "line": 17, - "column": 99, - "offset": 504 - }, - "indent": [] - } + "value": " is just a link and " }, { "type": "image", "title": null, "url": "http://img.youtube.com/vi/KOxbO0EI4MA/0.jpg", - "alt": "Audi R8", - "position": { - "start": { - "line": 17, - "column": 99, - "offset": 504 - }, - "end": { - "line": 17, - "column": 154, - "offset": 559 - }, - "indent": [] - } + "alt": "Audi R8" }, { "type": "text", - "value": " is just an image.", - "position": { - "start": { - "line": 17, - "column": 154, - "offset": 559 - }, - "end": { - "line": 17, - "column": 172, - "offset": 577 - }, - "indent": [] - } + "value": " is just an image." } - ], - "position": { - "start": { - "line": 17, - "column": 1, - "offset": 406 - }, - "end": { - "line": 17, - "column": 172, - "offset": 577 - }, - "indent": [] - } + ] }, { "type": "heading", @@ -543,35 +153,9 @@ "children": [ { "type": "text", - "value": "Image-Style Embeds", - "position": { - "start": { - "line": 19, - "column": 4, - "offset": 582 - }, - "end": { - "line": 19, - "column": 22, - "offset": 600 - }, - "indent": [] - } + "value": "Image-Style Embeds" } - ], - "position": { - "start": { - "line": 19, - "column": 1, - "offset": 579 - }, - "end": { - "line": 19, - "column": 22, - "offset": 600 - }, - "indent": [] - } + ] }, { "type": "embed", @@ -583,50 +167,11 @@ "type": "image", "title": null, "url": "https://www.youtube.com/watch?v=KOxbO0EI4MA", - "alt": null, - "position": { - "start": { - "line": 21, - "column": 1, - "offset": 602 - }, - "end": { - "line": 21, - "column": 49, - "offset": 650 - }, - "indent": [] - } + "alt": null } - ], - "position": { - "start": { - "line": 21, - "column": 1, - "offset": 602 - }, - "end": { - "line": 21, - "column": 49, - "offset": 650 - }, - "indent": [] - } + ] } ], - "position": { - "start": { - "line": 21, - "column": 1, - "offset": 602 - }, - "end": { - "line": 21, - "column": 49, - "offset": 650 - }, - "indent": [] - }, "url": "https://www.youtube.com/watch?v=KOxbO0EI4MA" }, { @@ -634,35 +179,9 @@ "children": [ { "type": "text", - "value": "is an embed, but ", - "position": { - "start": { - "line": 23, - "column": 1, - "offset": 652 - }, - "end": { - "line": 23, - "column": 18, - "offset": 669 - }, - "indent": [] - } + "value": "is an embed, but " } - ], - "position": { - "start": { - "line": 23, - "column": 1, - "offset": 652 - }, - "end": { - "line": 23, - "column": 18, - "offset": 669 - }, - "indent": [] - } + ] }, { "type": "paragraph", @@ -671,176 +190,33 @@ "type": "image", "title": null, "url": "https://www.gstatic.com/youtube/img/promos/growth/b74c9f83bf1704acff7677e46adde6cf59f23f4be85261468c1b1c7fa992ec18_120x120.jpeg", - "alt": null, - "position": { - "start": { - "line": 25, - "column": 1, - "offset": 671 - }, - "end": { - "line": 25, - "column": 133, - "offset": 803 - }, - "indent": [] - } + "alt": null }, { "type": "text", - "value": " is just an image. Even when ", - "position": { - "start": { - "line": 25, - "column": 133, - "offset": 803 - }, - "end": { - "line": 25, - "column": 162, - "offset": 832 - }, - "indent": [] - } + "value": " is just an image. Even when " } - ], - "position": { - "start": { - "line": 25, - "column": 1, - "offset": 671 - }, - "end": { - "line": 25, - "column": 162, - "offset": 832 - }, - "indent": [] - } + ] }, { "type": "paragraph", "children": [ { - "type": "text", - "value": "(", - "position": { - "start": { - "line": 27, - "column": 1, - "offset": 834 - }, - "end": { - "line": 27, - "column": 2, - "offset": 835 - }, - "indent": [] - } - }, - { - "type": "link", + "type": "image", "title": null, "url": "https://www.gstatic.com/youtube/img/promos/growth/b74c9f83bf1704acff7677e46adde6cf59f23f4be85261468c1b1c7fa992ec18_120x120.jpeg", - "children": [ - { - "type": "text", - "value": "https://www.gstatic.com/youtube/img/promos/growth/b74c9f83bf1704acff7677e46adde6cf59f23f4be85261468c1b1c7fa992ec18_120x120.jpeg", - "position": { - "start": { - "line": 27, - "column": 2, - "offset": 835 - }, - "end": { - "line": 27, - "column": 129, - "offset": 962 - }, - "indent": [] - } - } - ], - "position": { - "start": { - "line": 27, - "column": 2, - "offset": 835 - }, - "end": { - "line": 27, - "column": 129, - "offset": 962 - }, - "indent": [] - } - }, - { - "type": "text", - "value": ")", - "position": { - "start": { - "line": 27, - "column": 129, - "offset": 962 - }, - "end": { - "line": 27, - "column": 130, - "offset": 963 - }, - "indent": [] - } + "alt": null } - ], - "position": { - "start": { - "line": 27, - "column": 1, - "offset": 834 - }, - "end": { - "line": 27, - "column": 130, - "offset": 963 - }, - "indent": [] - } + ] }, { "type": "paragraph", "children": [ { "type": "text", - "value": "is on a paragraph of its own.", - "position": { - "start": { - "line": 29, - "column": 1, - "offset": 965 - }, - "end": { - "line": 29, - "column": 30, - "offset": 994 - }, - "indent": [] - } + "value": "is on a paragraph of its own." } - ], - "position": { - "start": { - "line": 29, - "column": 1, - "offset": 965 - }, - "end": { - "line": 29, - "column": 30, - "offset": 994 - }, - "indent": [] - } + ] }, { "type": "heading", @@ -848,35 +224,9 @@ "children": [ { "type": "text", - "value": "IA Writer-Style Embeds", - "position": { - "start": { - "line": 31, - "column": 4, - "offset": 999 - }, - "end": { - "line": 31, - "column": 26, - "offset": 1021 - }, - "indent": [] - } + "value": "IA Writer-Style Embeds" } - ], - "position": { - "start": { - "line": 31, - "column": 1, - "offset": 996 - }, - "end": { - "line": 31, - "column": 26, - "offset": 1021 - }, - "indent": [] - } + ] }, { "type": "embed", @@ -891,65 +241,13 @@ "children": [ { "type": "text", - "value": "https://www.youtube.com/watch?v=KOxbO0EI4MA", - "position": { - "start": { - "line": 33, - "column": 1, - "offset": 1023 - }, - "end": { - "line": 33, - "column": 44, - "offset": 1066 - }, - "indent": [] - } + "value": "https://www.youtube.com/watch?v=KOxbO0EI4MA" } - ], - "position": { - "start": { - "line": 33, - "column": 1, - "offset": 1023 - }, - "end": { - "line": 33, - "column": 44, - "offset": 1066 - }, - "indent": [] - } + ] } - ], - "position": { - "start": { - "line": 33, - "column": 1, - "offset": 1023 - }, - "end": { - "line": 33, - "column": 44, - "offset": 1066 - }, - "indent": [] - } + ] } ], - "position": { - "start": { - "line": 33, - "column": 1, - "offset": 1023 - }, - "end": { - "line": 33, - "column": 44, - "offset": 1066 - }, - "indent": [] - }, "url": "https://www.youtube.com/watch?v=KOxbO0EI4MA" }, { @@ -957,20 +255,7 @@ "children": [ { "type": "text", - "value": "is an embed, but ", - "position": { - "start": { - "line": 35, - "column": 1, - "offset": 1068 - }, - "end": { - "line": 35, - "column": 18, - "offset": 1085 - }, - "indent": [] - } + "value": "is an embed, but " }, { "type": "link", @@ -979,67 +264,15 @@ "children": [ { "type": "text", - "value": "https://www.youtube.com/watch?v=KOxbO0EI4MA", - "position": { - "start": { - "line": 35, - "column": 18, - "offset": 1085 - }, - "end": { - "line": 35, - "column": 61, - "offset": 1128 - }, - "indent": [] - } + "value": "https://www.youtube.com/watch?v=KOxbO0EI4MA" } - ], - "position": { - "start": { - "line": 35, - "column": 18, - "offset": 1085 - }, - "end": { - "line": 35, - "column": 61, - "offset": 1128 - }, - "indent": [] - } + ] }, { "type": "text", - "value": " is just a link.", - "position": { - "start": { - "line": 35, - "column": 61, - "offset": 1128 - }, - "end": { - "line": 35, - "column": 77, - "offset": 1144 - }, - "indent": [] - } + "value": " is just a link." } - ], - "position": { - "start": { - "line": 35, - "column": 1, - "offset": 1068 - }, - "end": { - "line": 35, - "column": 77, - "offset": 1144 - }, - "indent": [] - } + ] }, { "type": "heading", @@ -1047,105 +280,27 @@ "children": [ { "type": "text", - "value": "Whitelisting", - "position": { - "start": { - "line": 37, - "column": 3, - "offset": 1148 - }, - "end": { - "line": 37, - "column": 15, - "offset": 1160 - }, - "indent": [] - } + "value": "Whitelisting" } - ], - "position": { - "start": { - "line": 37, - "column": 1, - "offset": 1146 - }, - "end": { - "line": 37, - "column": 15, - "offset": 1160 - }, - "indent": [] - } + ] }, { "type": "paragraph", "children": [ { "type": "text", - "value": "All embed hostnames must be whitelisted. Therefore, the following are not embeds:", - "position": { - "start": { - "line": 39, - "column": 1, - "offset": 1162 - }, - "end": { - "line": 39, - "column": 82, - "offset": 1243 - }, - "indent": [] - } + "value": "All embed hostnames must be whitelisted. Therefore, the following are not embeds:" } - ], - "position": { - "start": { - "line": 39, - "column": 1, - "offset": 1162 - }, - "end": { - "line": 39, - "column": 82, - "offset": 1243 - }, - "indent": [] - } + ] }, { "type": "paragraph", "children": [ { "type": "inlineCode", - "value": "video: https://www.example.com/embed/2Xc9gXyf2G4", - "position": { - "start": { - "line": 41, - "column": 1, - "offset": 1245 - }, - "end": { - "line": 41, - "column": 51, - "offset": 1295 - }, - "indent": [] - } + "value": "video: https://www.example.com/embed/2Xc9gXyf2G4" } - ], - "position": { - "start": { - "line": 41, - "column": 1, - "offset": 1245 - }, - "end": { - "line": 41, - "column": 51, - "offset": 1295 - }, - "indent": [] - } + ] }, { "type": "paragraph", @@ -1159,50 +314,11 @@ "type": "image", "title": null, "url": "http://img.youtube.com/vi/KOxbO0EI4MA/0.jpg", - "alt": "Audi R8", - "position": { - "start": { - "line": 43, - "column": 2, - "offset": 1298 - }, - "end": { - "line": 43, - "column": 57, - "offset": 1353 - }, - "indent": [] - } + "alt": "Audi R8" } - ], - "position": { - "start": { - "line": 43, - "column": 1, - "offset": 1297 - }, - "end": { - "line": 43, - "column": 113, - "offset": 1409 - }, - "indent": [] - } + ] } - ], - "position": { - "start": { - "line": 43, - "column": 1, - "offset": 1297 - }, - "end": { - "line": 43, - "column": 113, - "offset": 1409 - }, - "indent": [] - } + ] }, { "type": "paragraph", @@ -1211,35 +327,9 @@ "type": "image", "title": null, "url": "https://www.example.com/watch?v=KOxbO0EI4MA", - "alt": null, - "position": { - "start": { - "line": 45, - "column": 1, - "offset": 1411 - }, - "end": { - "line": 45, - "column": 49, - "offset": 1459 - }, - "indent": [] - } + "alt": null } - ], - "position": { - "start": { - "line": 45, - "column": 1, - "offset": 1411 - }, - "end": { - "line": 45, - "column": 49, - "offset": 1459 - }, - "indent": [] - } + ] }, { "type": "paragraph", @@ -1251,62 +341,255 @@ "children": [ { "type": "text", - "value": "https://www.example.com/watch?v=KOxbO0EI4MA", - "position": { - "start": { - "line": 47, - "column": 1, - "offset": 1461 - }, - "end": { - "line": 47, - "column": 44, - "offset": 1504 - }, - "indent": [] - } + "value": "https://www.example.com/watch?v=KOxbO0EI4MA" + } + ] + } + ] + }, + { + "type": "heading", + "depth": 1, + "children": [ + { + "type": "text", + "value": "Internal Embeds" + } + ] + }, + { + "type": "heading", + "depth": 2, + "children": [ + { + "type": "text", + "value": "Gatsby-Style-Embeds" + } + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "embed", + "children": [ + { + "type": "inlineCode", + "value": "embed: foo.md" } ], - "position": { - "start": { - "line": 47, - "column": 1, - "offset": 1461 - }, - "end": { - "line": 47, - "column": 44, - "offset": 1504 - }, - "indent": [] - } + "url": "/docs/foo.embed.html" } - ], - "position": { - "start": { - "line": 47, - "column": 1, - "offset": 1461 + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "embed", + "children": [ + { + "type": "inlineCode", + "value": "html: bar.html" + } + ], + "url": "/docs/bar.embed.html" + } + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "inlineCode", + "value": "markdowm: /docs/test.md" + } + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "embed", + "children": [ + { + "type": "inlineCode", + "value": "embed:../index.html" + } + ], + "url": "/index.embed.html" + } + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "Are all valid embeds, but " }, - "end": { - "line": 47, - "column": 44, - "offset": 1504 + { + "type": "inlineCode", + "value": "test: foo.md" }, - "indent": [] - } - } - ], - "position": { - "start": { - "line": 1, - "column": 1, - "offset": 0 + { + "type": "text", + "value": " isn't one." + } + ] + }, + { + "type": "heading", + "depth": 2, + "children": [ + { + "type": "text", + "value": "Image-Style Embeds" + } + ] }, - "end": { - "line": 47, - "column": 44, - "offset": 1504 + { + "type": "embed", + "children": [ + { + "type": "paragraph", + "children": [ + { + "type": "image", + "title": null, + "url": "foo.md", + "alt": null + } + ] + } + ], + "url": "/docs/foo.embed.html" + }, + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "is an embed, but " + } + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "image", + "title": null, + "url": "foo.png", + "alt": null + }, + { + "type": "text", + "value": " is just an image. Even when " + } + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "image", + "title": null, + "url": "foo.png", + "alt": null + } + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "is on a paragraph of its own." + } + ] + }, + { + "type": "heading", + "depth": 2, + "children": [ + { + "type": "text", + "value": "IA Writer-Style Embeds" + } + ] + }, + { + "type": "embed", + "children": [ + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "/foo.md" + } + ] + } + ], + "url": "/foo.embed.html" + }, + { + "type": "embed", + "children": [ + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "../foo.html" + } + ] + } + ], + "url": "/foo.embed.html" + }, + { + "type": "embed", + "children": [ + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "../readme/docs.html" + } + ] + } + ], + "url": "/readme/docs.embed.html" + }, + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "Are all valid embeds, but" + } + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "/foo.txt" + } + ] + }, + { + "type": "paragraph", + "children": [ + { + "type": "text", + "value": "isn't one." + } + ] } - } + ] } diff --git a/test/fixtures/embeds.md b/test/fixtures/embeds.md index f99267957..b4b6a8de3 100644 --- a/test/fixtures/embeds.md +++ b/test/fixtures/embeds.md @@ -1,6 +1,6 @@ # Hello "World" -## Gasby-Style-Embeds +## Gatsby-Style-Embeds `video: https://www.youtube.com/embed/2Xc9gXyf2G4` @@ -24,7 +24,7 @@ is an embed, but ![](https://www.gstatic.com/youtube/img/promos/growth/b74c9f83bf1704acff7677e46adde6cf59f23f4be85261468c1b1c7fa992ec18_120x120.jpeg) is just an image. Even when -(https://www.gstatic.com/youtube/img/promos/growth/b74c9f83bf1704acff7677e46adde6cf59f23f4be85261468c1b1c7fa992ec18_120x120.jpeg) +![](https://www.gstatic.com/youtube/img/promos/growth/b74c9f83bf1704acff7677e46adde6cf59f23f4be85261468c1b1c7fa992ec18_120x120.jpeg) is on a paragraph of its own. @@ -44,4 +44,45 @@ All embed hostnames must be whitelisted. Therefore, the following are not embeds ![](https://www.example.com/watch?v=KOxbO0EI4MA) -https://www.example.com/watch?v=KOxbO0EI4MA \ No newline at end of file +https://www.example.com/watch?v=KOxbO0EI4MA + +# Internal Embeds + +## Gatsby-Style-Embeds + +`embed: foo.md` + +`html: bar.html` + +`markdowm: /docs/test.md` + +`embed:../index.html` + +Are all valid embeds, but `test: foo.md` isn't one. + + +## Image-Style Embeds + +![](foo.md) + +is an embed, but + +![](foo.png) is just an image. Even when + +![](foo.png) + +is on a paragraph of its own. + +## IA Writer-Style Embeds + +/foo.md + +../foo.html + +../readme/docs.html + +Are all valid embeds, but + +/foo.txt + +isn't one. \ No newline at end of file diff --git a/test/testFindEmbeds.js b/test/testFindEmbeds.js index 38971d95a..774e0a20b 100644 --- a/test/testFindEmbeds.js +++ b/test/testFindEmbeds.js @@ -11,6 +11,9 @@ */ /* eslint-env mocha */ const { Logger } = require('@adobe/helix-shared'); +const assert = require('assert'); +const u = require('unist-builder'); +const inspect = require('unist-util-inspect'); const parse = require('../src/html/parse-markdown'); const embeds = require('../src/html/find-embeds'); const { assertMatch, assertValid } = require('./markdown-utils'); @@ -23,16 +26,19 @@ const logger = Logger.getTestLogger({ const action = { logger, + request: { + path: '/index.md', + }, }; function mdast(body) { const parsed = parse({ content: { body } }, { logger }); - return embeds(parsed, action).content.mdast; + return embeds({ content: parsed.content, request: { extension: 'html', url: '/docs/index.html' } }, action).content.mdast; } function context(body) { const parsed = parse({ content: { body } }, { logger }); - return embeds(parsed, action); + return embeds({ content: parsed.content, request: { extension: 'html', url: '/docs/index.html' } }, action); } describe('Test Embed Detection Processing', () => { @@ -50,3 +56,80 @@ describe('Validate Embed Examples In Pipeline', () => { assertValid('embeds', context, done); }); }); + +describe('Find Embeds #unit', () => { + const base = '/docs/index.html'; + + it('internalGatsbyEmbed', () => { + ['foo: foo.md', + 'embed:', + 'embed: ?foo=bar', + 'embed: foo.txt', + ] + .forEach(t => assert.ok(!embeds.internalGatsbyEmbed(t, base, 'html', 'md'), `${t} should be invalid`)); + + ['embed: foo.md', + 'html: foo.md', + 'markdown: foo.md', + 'embed:markdown.md', + 'embed: foo.html', + 'embed: ../foo.html', + 'embed: /foo.html', + 'embed: ../docs/rocks/../foo.html'] + .forEach(t => assert.ok(embeds.internalGatsbyEmbed(t, base, 'html', 'md'), `${t} should be valid`)); + }); + + it('internalIaEmbed', () => { + [ + u('paragraph', [ + u('text', {}, 'I\'m just a text. Don\'t mind me.'), + ]), + u('paragraph', [ + u('text', {}, '/foo.md'), + u('text', {}, '/foo.md'), + ]), + u('paragraph', [ + u('text', {}, '/foo.txt'), + ]), + u('paragraph', [ + u('text', {}, ' foo.md'), + ]), + ].forEach(t => assert.ok(!embeds.internalIaEmbed(t, base, 'html', 'md'), `Expected invalid:\n${inspect(t)}`)); + [ + u('paragraph', [ + u('text', {}, '/foo.md'), + ]), + u('paragraph', [ + u('text', {}, '/foo.html'), + ]), + u('paragraph', [ + u('text', {}, '../foo.md'), + ]), + u('paragraph', [ + u('text', {}, '/readme/foo.md'), + ]), + u('paragraph', [ + u('text', {}, '../help/foo.md'), + ]), + ].forEach(t => assert.ok(embeds.internalIaEmbed(t, base, 'html', 'md'), `Expected valid:\n${inspect(t)}`)); + }); + + it('internalImgEmbed', () => { + [ + u('paragraph', [ + u('image', { url: 'test.png' }), + ]), + ].forEach(t => assert.ok(!embeds.internalImgEmbed(t, base, 'html', 'md'), `Expected invalid:\n${inspect(t)}`)); + [ + u('paragraph', [ + u('image', { url: 'test.md' }), + ]), + u('paragraph', [ + u('image', { url: '../test.md' }), + ]), + u('paragraph', [ + u('image', { url: '../readme/test.html' }), + ]), + ].forEach(t => assert.ok(embeds.internalImgEmbed(t, base, 'html', 'md'), `Expected valid:\n${inspect(t)}`)); + }); +}); From 8ceef49dc813b9b366376d4103b3d77517bf57e7 Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Thu, 18 Apr 2019 06:55:18 +0000 Subject: [PATCH 2/5] test(embeds): fix unit and integration tests for internal embeds The internal embed handler now requires access to some additional request properties that were not mocked in all test cases for #267 --- src/html/find-embeds.js | 2 +- test/testConditionalSections.js | 9 +++++++++ test/testEmbedHandler.js | 7 +++++++ test/testFindEmbeds.js | 4 +++- test/testHTML.js | 19 +++++++++++++++++-- test/testHTMLFromMarkdown.js | 7 ++++++- 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/html/find-embeds.js b/src/html/find-embeds.js index b94947df2..41e97de40 100644 --- a/src/html/find-embeds.js +++ b/src/html/find-embeds.js @@ -114,7 +114,7 @@ function internalembed(uri, node, extension) { } function find({ content: { mdast }, request: { extension, url } }, - { logger, secrets: { EMBED_WHITELIST, EMBED_SELECTOR }, request: { path } }) { + { logger, secrets: { EMBED_WHITELIST, EMBED_SELECTOR }, request: { params: { path } } }) { const resourceext = `.${extension}`; const contentext = p.extname(path); map(mdast, (node) => { diff --git a/test/testConditionalSections.js b/test/testConditionalSections.js index 90b87bd36..a0f94c86c 100644 --- a/test/testConditionalSections.js +++ b/test/testConditionalSections.js @@ -66,6 +66,11 @@ const secrets = { REPO_RAW_ROOT: 'https://raw.githubusercontent.com/', }; +const crequest = { + extension: 'html', + url: '/test/test.html', +}; + // return only sections that are not hidden function nonhidden(section) { if (section.meta && section.meta) { @@ -86,6 +91,7 @@ describe('Integration Test Section Strain Filtering', () => { return { response: { body: content.document.body.innerHTML } }; }, { + request: crequest, content: { body: `--- frontmatter: true @@ -125,6 +131,7 @@ And this one only in strain "B" describe('Unit Test Section Strain Filtering', () => { it('Works with empty section lists', () => { const context = { + request: crequest, content: { sections: [], }, @@ -342,6 +349,7 @@ describe('Integration Test A/B Testing', () => { return { response: { body: content.document.body.innerHTML } }; }, { + request: crequest, content: { body: `--- frontmatter: true @@ -393,6 +401,7 @@ Or this one at the same time. return { response: { body: content.document.body.innerHTML } }; }, { + request: crequest, content: { body: `--- frontmatter: true diff --git a/test/testEmbedHandler.js b/test/testEmbedHandler.js index 1d0d4dea2..b1a87195c 100644 --- a/test/testEmbedHandler.js +++ b/test/testEmbedHandler.js @@ -68,6 +68,12 @@ const logger = Logger.getTestLogger({ level: 'info', }); + +const crequest = { + extension: 'html', + url: '/test/test.html', +}; + describe('Test Embed Handler', () => { it('Creates ESI', async () => { const node = { @@ -93,6 +99,7 @@ describe('Integration Test with Embeds', () => { const result = await pipe( ({ content }) => ({ response: { status: 201, body: content.document.body.innerHTML } }), { + request: crequest, content: { body: `Hello World Here comes an embed. diff --git a/test/testFindEmbeds.js b/test/testFindEmbeds.js index 774e0a20b..571d35471 100644 --- a/test/testFindEmbeds.js +++ b/test/testFindEmbeds.js @@ -27,7 +27,9 @@ const logger = Logger.getTestLogger({ const action = { logger, request: { - path: '/index.md', + params: { + path: '/index.md', + }, }, }; diff --git a/test/testHTML.js b/test/testHTML.js index bfeb3eab2..ea876ac24 100644 --- a/test/testHTML.js +++ b/test/testHTML.js @@ -111,6 +111,12 @@ const secrets = { REPO_RAW_ROOT: 'https://raw.githubusercontent.com/', }; + +const crequest = { + extension: 'html', + url: '/test/test.html', +}; + describe('Testing HTML Pipeline', () => { setupPolly({ logging: false, @@ -139,6 +145,7 @@ describe('Testing HTML Pipeline', () => { return { response: { status: 201, body: content.document.body.innerHTML } }; }, { + request: crequest, content: { body: 'Hello World', }, @@ -206,6 +213,7 @@ ${content.document.body.innerHTML}`, const res = await pipe( myfunc, { + request: crequest, content: { body: 'Hello World', }, @@ -236,6 +244,7 @@ ${content.document.body.innerHTML}`, return { response: { status: 201, body: content.document.body.innerHTML } }; }, { + request: crequest, content: { body: fs.readFileSync(path.resolve(__dirname, 'fixtures/index-unmodified.md')).toString(), }, @@ -262,6 +271,7 @@ ${content.document.body.innerHTML}`, return { response: { status: 201, body: content.document.body.innerHTML } }; }, { + request: crequest, content: { body: fs.readFileSync(path.resolve(__dirname, 'fixtures/index-projecthelixio.md')).toString(), }, @@ -288,6 +298,7 @@ ${content.document.body.innerHTML}`, return { response: { status: 201, body: content.document.body.innerHTML } }; }, { + request: crequest, content: { body: fs.readFileSync(path.resolve(__dirname, 'fixtures/index-modified.md')).toString(), }, @@ -306,6 +317,7 @@ ${content.document.body.innerHTML}`, const result = await pipe( ({ content }) => ({ response: { status: 201, body: content.document.body.innerHTML } }), { + request: crequest, content: { foo: 'Hello World', }, @@ -325,6 +337,7 @@ ${content.document.body.innerHTML}`, const result = await pipe( ({ content }) => ({ response: { status: 201, body: content.document.body.innerHTML } }), { + request: crequest, content: { sections: [{ type: 'notroot' }], }, @@ -344,6 +357,7 @@ ${content.document.body.innerHTML}`, const result = await pipe( ({ content }) => ({ response: { status: 201, body: content.document.body.innerHTML } }), { + request: crequest, content: { sections: [{ type: 'root', custom: 'notallowed' }], }, @@ -363,6 +377,7 @@ ${content.document.body.innerHTML}`, const result = await pipe( ({ content }) => ({ response: { status: 201, body: content.document.body.innerHTML } }), { + request: crequest, content: { body: 'Hello World', }, @@ -498,7 +513,7 @@ ${content.document.body.innerHTML}`, }, }, }), - {}, + { request: crequest }, { request: { params }, secrets, @@ -524,7 +539,7 @@ ${content.document.body.innerHTML}`, }, }, }), - {}, + { request: crequest }, { request: { params }, secrets, diff --git a/test/testHTMLFromMarkdown.js b/test/testHTMLFromMarkdown.js index 9d824593d..4125c1195 100644 --- a/test/testHTMLFromMarkdown.js +++ b/test/testHTMLFromMarkdown.js @@ -68,6 +68,11 @@ const logger = winston.createLogger({ ], }); +const crequest = { + extension: 'html', + url: '/test/test.html', +}; + /** * Assert that a specific html dom is generated from the given markdown * using our html pipeline. @@ -86,7 +91,7 @@ const assertMd = async (md, html) => { const generated = await pipe( fromHTML, - { content: { body: multiline(md) } }, + { content: { body: multiline(md) }, request: crequest }, { logger, request: { params }, From 73cb0fcde8fddb399dd33ae811dcc7b6c88498d8 Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Thu, 18 Apr 2019 07:08:44 +0000 Subject: [PATCH 3/5] test(embed): provide integration test for internal embeds #267 --- src/utils/embed-handler.js | 5 ++++- test/testEmbedHandler.js | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/utils/embed-handler.js b/src/utils/embed-handler.js index bd30a72ed..a8926597d 100644 --- a/src/utils/embed-handler.js +++ b/src/utils/embed-handler.js @@ -13,11 +13,14 @@ * Handles `embed` MDAST nodes by converting them into `` tags * @param {string} EMBED_SERVICE the URL of an embedding service compatible with https://github.com/adobe/helix-embed that returns HTML */ +const URI = require('uri-js'); + function embed({ EMBED_SERVICE }) { return function handler(h, node) { const { url } = node; const props = { - src: EMBED_SERVICE + url, + // prepend the embed service for absolute URLs + src: (URI.parse(url).reference === 'absolute' ? EMBED_SERVICE : '') + url, }; const retval = h(node, 'esi:include', props); return retval; diff --git a/test/testEmbedHandler.js b/test/testEmbedHandler.js index b1a87195c..60293c6c4 100644 --- a/test/testEmbedHandler.js +++ b/test/testEmbedHandler.js @@ -122,6 +122,36 @@ https://www.youtube.com/watch?v=KOxbO0EI4MA assert.equal(result.response.body, `

Hello World Here comes an embed.

+

Easy!

`); + }); + + it('html.pipe processes internal embeds', async () => { + const result = await pipe( + ({ content }) => ({ response: { status: 201, body: content.document.body.innerHTML } }), + { + request: crequest, + content: { + body: `Hello World +Here comes an embed. + +./foo.md + +![Easy!](easy.png) +`, + }, + }, + { + request: { params }, + secrets, + logger, + }, + ); + + assert.equal(result.response.status, 201); + assert.equal(result.response.headers['Content-Type'], 'text/html'); + assert.equal(result.response.body, `

Hello World +Here comes an embed.

+

Easy!

`); }); }); From 0d5193a8cbee8c06988a23bde247fa99c5eed52f Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Thu, 18 Apr 2019 08:14:25 +0000 Subject: [PATCH 4/5] feat(transformer): enable recursive processing in custom handler functions The VDOMTransformer's `handler` functions did not allow for recursive processing of child nodes so far, i.e. when a handler function wanted to process only a single MDAST node, but hand the processing of all MDAST child nodes back to the original transformer, it had no way of doing so, making each `handler` function a terminal node. With this change, the signature of `handler` functions changes to `handler(callback, node, parent, handlechild)` where `handlechild(callback, childnode, mdastparent, hastparent)` is a callback itself that takes the current HTAST-constructing `callback`, the MDAST `childnode` that should get transformed, the `mdastparent`, i.e. the current node from the vantage point of the `handler` function, and with `hastparent` a HAST parent node that the output of the child processing will be attached to. --- src/utils/mdast-to-vdom.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/utils/mdast-to-vdom.js b/src/utils/mdast-to-vdom.js index 21e49ee42..379876d65 100644 --- a/src/utils/mdast-to-vdom.js +++ b/src/utils/mdast-to-vdom.js @@ -106,7 +106,23 @@ class VDOMTransformer { const handlefn = that.matches(node); // process the node - const result = handlefn(cb, node, parent); + + /** + * A function that enables the recursive processing of MDAST child nodes + * in handler functions. + * @param {function} callback the HAST-constructing callback function + * @param {Node} childnode the MDAST child node that should be handled + * @param {Node} mdastparent the MDAST parent node, usually the current MDAST node + * processed by the handler function + * @param {*} hastparent the HAST parent node that the transformed child will be appended to + */ + function handlechild(callback, childnode, mdastparent, hastparent) { + if (hastparent && hastparent.children) { + hastparent.children.push(VDOMTransformer.handle(callback, childnode, mdastparent, that)); + } + } + + const result = handlefn(cb, node, parent, handlechild); if (result && typeof result === 'string') { return VDOMTransformer.toHTAST(result, cb, node); From 575391dae8d4994b6ddf0a794cbed8ae609e104f Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Thu, 18 Apr 2019 08:16:55 +0000 Subject: [PATCH 5/5] fix(embed): provide fallback with esi:remove when esi include fails Until now, a failed ESI include would fail silently, without a possible fallback. This change falls back to the markdown rendered as HTML in case the ESI does not work, giving at least an approximation of the content created. fixes #267 --- src/utils/embed-handler.js | 11 +++++++++-- test/testEmbedHandler.js | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/utils/embed-handler.js b/src/utils/embed-handler.js index a8926597d..1faa0a1db 100644 --- a/src/utils/embed-handler.js +++ b/src/utils/embed-handler.js @@ -16,13 +16,20 @@ const URI = require('uri-js'); function embed({ EMBED_SERVICE }) { - return function handler(h, node) { + return function handler(h, node, _, handlechild) { const { url } = node; const props = { // prepend the embed service for absolute URLs src: (URI.parse(url).reference === 'absolute' ? EMBED_SERVICE : '') + url, }; - const retval = h(node, 'esi:include', props); + const retval = [h(node, 'esi:include', props)]; + + if (node.children && node.children.length) { + const rem = h(node, 'esi:remove', {}); + node.children.forEach(childnode => handlechild(h, childnode, node, rem)); + retval.push(rem); + } + return retval; }; } diff --git a/test/testEmbedHandler.js b/test/testEmbedHandler.js index 60293c6c4..43fadd7a4 100644 --- a/test/testEmbedHandler.js +++ b/test/testEmbedHandler.js @@ -122,6 +122,7 @@ https://www.youtube.com/watch?v=KOxbO0EI4MA assert.equal(result.response.body, `

Hello World Here comes an embed.

+

https://www.youtube.com/watch?v=KOxbO0EI4MA

Easy!

`); }); @@ -152,6 +153,7 @@ Here comes an embed. assert.equal(result.response.body, `

Hello World Here comes an embed.

+

./foo.md

Easy!

`); }); });