From bcb5157a73df02fbb1171b6d305f8aa9c976dc1c Mon Sep 17 00:00:00 2001 From: Jesus David Garcia Gomez Date: Fri, 15 Feb 2019 09:40:54 -0800 Subject: [PATCH 1/7] Fix: Add utils to traverse HTML --- packages/hint/package.json | 1 + .../hint/src/lib/types/jsdom-async-html.ts | 4 + .../src/lib/utils/dom/create-async-window.ts | 9 + .../hint/src/lib/utils/dom/create-jsdom.ts | 16 ++ packages/hint/src/lib/utils/dom/traverse.ts | 36 +++ yarn.lock | 215 ++++++++---------- 6 files changed, 166 insertions(+), 115 deletions(-) create mode 100644 packages/hint/src/lib/utils/dom/create-async-window.ts create mode 100644 packages/hint/src/lib/utils/dom/create-jsdom.ts create mode 100644 packages/hint/src/lib/utils/dom/traverse.ts diff --git a/packages/hint/package.json b/packages/hint/package.json index 3a9f9ca7d60..ace21ebfa3d 100644 --- a/packages/hint/package.json +++ b/packages/hint/package.json @@ -23,6 +23,7 @@ "globby": "^9.0.0", "is-ci": "^2.0.0", "is-svg": "^3.0.0", + "jsdom": "^13.2.0", "jsonc-parser": "^2.0.3", "lodash": "^4.17.11", "mime-db": "1.35.0", diff --git a/packages/hint/src/lib/types/jsdom-async-html.ts b/packages/hint/src/lib/types/jsdom-async-html.ts index ba5d018cb72..6fb9d140e07 100644 --- a/packages/hint/src/lib/types/jsdom-async-html.ts +++ b/packages/hint/src/lib/types/jsdom-async-html.ts @@ -134,6 +134,10 @@ export class JSDOMAsyncWindow implements IAsyncWindow { return this._document; } + public get dom(): JSDOM | undefined { + return this._dom; + } + public evaluate(source: string): Promise { return this._window.eval(source) as any; } diff --git a/packages/hint/src/lib/utils/dom/create-async-window.ts b/packages/hint/src/lib/utils/dom/create-async-window.ts new file mode 100644 index 00000000000..8a7bd472ad0 --- /dev/null +++ b/packages/hint/src/lib/utils/dom/create-async-window.ts @@ -0,0 +1,9 @@ +import { IAsyncWindow } from '../../types/async-html'; +import { JSDOMAsyncWindow } from '../../types/jsdom-async-html'; +import createJsdom from './create-jsdom'; + +export default (html: string, allowScripts: boolean = false): IAsyncWindow => { + const dom = createJsdom(html, allowScripts); + + return new JSDOMAsyncWindow(dom.window, dom); +}; diff --git a/packages/hint/src/lib/utils/dom/create-jsdom.ts b/packages/hint/src/lib/utils/dom/create-jsdom.ts new file mode 100644 index 00000000000..9a2dcf06d23 --- /dev/null +++ b/packages/hint/src/lib/utils/dom/create-jsdom.ts @@ -0,0 +1,16 @@ +import { JSDOM } from 'jsdom'; + +export default (html: string, allowScripts: boolean = false): JSDOM => { + return new JSDOM(html, { + + /** Needed to provide line/column positions for elements. */ + includeNodeLocations: true, + + /** + * Needed to let hints run script against the DOM. + * However the page itself is kept static because `connector-local` + * validates files individually without loading resources. + */ + runScripts: allowScripts ? 'outside-only' : undefined + }); +}; diff --git a/packages/hint/src/lib/utils/dom/traverse.ts b/packages/hint/src/lib/utils/dom/traverse.ts new file mode 100644 index 00000000000..7a8190c1da8 --- /dev/null +++ b/packages/hint/src/lib/utils/dom/traverse.ts @@ -0,0 +1,36 @@ +import { JSDOMAsyncHTMLElement } from '../../types/jsdom-async-html'; +import { JSDOM } from 'jsdom'; +import { Engine } from '../../engine'; +import { TraverseUp, TraverseDown, Event } from '../../types/events'; + +const traverseAndNotify = async (element: HTMLElement, dom: JSDOM, engine: Engine, resource: string): Promise => { + + await engine.emitAsync(`element::${element.tagName.toLowerCase()}` as 'element::*', { + element: new JSDOMAsyncHTMLElement(element, dom), + resource + }); + + const traverseEvent = { + element: new JSDOMAsyncHTMLElement(element, dom), + resource + } as TraverseDown | TraverseUp; + + await engine.emitAsync(`traverse::down`, traverseEvent); + + // Recursively traverse child elements. + for (let i = 0; i < element.children.length; i++) { + await traverseAndNotify(element.children[i] as HTMLElement, dom, engine, resource); + } + + await engine.emitAsync(`traverse::up`, traverseEvent); +}; + +export default async (dom: JSDOM, engine: Engine, resource: string): Promise => { + const documentElement = dom.window.document.documentElement; + + const event = { resource } as Event; + + await engine.emitAsync('traverse::start', event); + await traverseAndNotify(documentElement, dom, engine, resource); + await engine.emitAsync('traverse::end', event); +}; diff --git a/yarn.lock b/yarn.lock index ad258d1a2fa..41c3c93189e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -360,13 +360,13 @@ "@sinonjs/samsam" "^2 || ^3" "@sinonjs/samsam@^2 || ^3", "@sinonjs/samsam@^3.0.2": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@sinonjs/samsam/-/samsam-3.1.0.tgz#38146f7be732de96f9f599d7247d71e349bf4bdb" - integrity sha512-IXio+GWY+Q8XUjHUOgK7wx8fpvr7IFffgyXb1bnJFfX3001KmHt35Zq4tp7MXZyjJPCLPuadesDYNk41LYtVjw== + version "3.1.1" + resolved "https://registry.yarnpkg.com/@sinonjs/samsam/-/samsam-3.1.1.tgz#8e2eceb2353f6626e2867352e3def951d3366240" + integrity sha512-ILlwvQUwAiaVBzr3qz8oT1moM7AIUHqUc2UmEjQcH9lLe+E+BZPwUMuc9FFojMswRK4r96x5zDTTrowMLw/vuA== dependencies: "@sinonjs/commons" "^1.0.2" array-from "^2.1.1" - lodash.get "^4.4.2" + lodash "^4.17.11" "@types/amphtml-validator@^1.0.0": version "1.0.0" @@ -376,9 +376,9 @@ "@types/node" "*" "@types/anymatch@*": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@types/anymatch/-/anymatch-1.3.0.tgz#d1d55958d1fccc5527d4aba29fc9c4b942f563ff" - integrity sha512-7WcbyctkE8GTzogDb0ulRAEw7v8oIS54ft9mQTU7PfM0hp5e+8kpa+HeQ7IQrFbKtJXBKcZ4bh+Em9dTw5L6AQ== + version "1.3.1" + resolved "https://registry.yarnpkg.com/@types/anymatch/-/anymatch-1.3.1.tgz#336badc1beecb9dacc38bea2cf32adf627a8421a" + integrity sha512-/+CRPXpBDpo2RK9C68N3b2cOvO0Cf5B9aPijHsoDQTHivnGSObdOF2BRQOYjojWTDy6nQvMjmqRXIxH55VjxxA== "@types/applicationinsights-js@^1.0.9": version "1.0.9" @@ -460,6 +460,11 @@ resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.1.tgz#0be64e2fbe7c0bb2a6cad6a152462127a1b0d45a" integrity sha512-OKKcc+Tt1rLMg1DTnmgvWHRna3SN51GWs/ERxpgjFNRjCUUVhGG8FEKeFqVTzBv1jEGFqbkJRWfsHV6KAcLT+A== +"@types/debug@^4.1.0": + version "4.1.1" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.1.tgz#0be64e2fbe7c0bb2a6cad6a152462127a1b0d45a" + integrity sha512-OKKcc+Tt1rLMg1DTnmgvWHRna3SN51GWs/ERxpgjFNRjCUUVhGG8FEKeFqVTzBv1jEGFqbkJRWfsHV6KAcLT+A== + "@types/ejs@^2.6.2": version "2.6.2" resolved "https://registry.yarnpkg.com/@types/ejs/-/ejs-2.6.2.tgz#d59bd4b12114a62d203ed137dbee5bebc2333f93" @@ -609,9 +614,9 @@ parse5 "^4.0.0" "@types/json-schema@*": - version "7.0.2" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.2.tgz#39e4bb68007c8cfa56393d867d192917a59b0723" - integrity sha512-XawGUgMoPEBwyN+P8FzzH8RfoJcmkyNPdMdX/Ejeit8y5M/VllcBjyGS72a3SCAQedsSnVn2pnf3lQ0OVR7f5g== + version "7.0.3" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.3.tgz#bdfd69d61e464dcc81b25159c270d75a73c1a636" + integrity sha512-Il2DtDVRGDcqjDtE+rF8iqg1CArehSK84HZJCT7AMITlyXRBpuPhqGLDQMowraqqu1coEaimg4ZOqggt6L6L+A== "@types/listr@^0.13.0": version "0.13.0" @@ -1179,11 +1184,11 @@ ajv-errors@^1.0.0: integrity sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ== ajv-keywords@^3.1.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.3.0.tgz#cb6499da9b83177af8bc1732b2f0a1a1a3aacf8c" - integrity sha512-CMzN9S62ZOO4sA/mJZIO4S++ZM7KFWzH3PPWkveLhy4OZ9i1/VatgwWMD46w/XbGCBy7Ye0gCk+Za6mmyfKK7g== + version "3.4.0" + resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.4.0.tgz#4b831e7b531415a7cc518cd404e73f6193c6349d" + integrity sha512-aUjdRFISbuFOl0EIZc+9e4FfZp0bDZgAdOOf30bJmw8VM9v84SHyVyxDfbWxpGYbdZD/9XoKxfHVNmxPkhwyGw== -ajv@^6.1.0, ajv@^6.5.3, ajv@^6.5.5, ajv@^6.6.1, ajv@^6.9.1: +ajv@^6.1.0, ajv@^6.5.3, ajv@^6.5.5, ajv@^6.9.1: version "6.9.1" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.9.1.tgz#a4d3683d74abc5670e75f0b16520f70a20ea8dc1" integrity sha512-XDN92U311aINL77ieWHmqCcNlwjoP5cHXDxIxbf2MaPYuCXOHS7gHH8jktxeK5omgd52XbSTX6a4Piwd1pQmzA== @@ -2183,9 +2188,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000929: - version "1.0.30000935" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000935.tgz#d1b59df00b46f4921bb84a8a34c1d172b346df59" - integrity sha512-1Y2uJ5y56qDt3jsDTdBHL1OqiImzjoQcBG6Yl3Qizq8mcc2SgCFpi+ZwLLqkztYnk9l87IYqRlNBnPSOTbFkXQ== + version "1.0.30000938" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000938.tgz#b64bf1427438df40183fce910fe24e34feda7a3f" + integrity sha512-ekW8NQ3/FvokviDxhdKLZZAx7PptXNwxKgXtnR5y+PR3hckwuP3yJ1Ir+4/c97dsHNqtAyfKUGdw8P4EYzBNgw== canvas@2.2.0: version "2.2.0" @@ -2743,9 +2748,9 @@ copy-descriptor@^0.1.0: integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= core-js@^2.0.0, core-js@^2.4.0: - version "2.6.4" - resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.4.tgz#b8897c062c4d769dd30a0ac5c73976c47f92ea0d" - integrity sha512-05qQ5hXShcqGkPZpXEFLIpxayZscVD2kuMBZewxiIPPEagukO4mqgPA9CWhUvFBJfy3ODdK2p9xyHh7FTU9/7A== + version "2.6.5" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.5.tgz#44bc8d249e7fb2ff5d00e0341a7ffb94fbf67895" + integrity sha512-klh/kDpwX8hryYL14M9w/xei6vrv6sE8gTHDG7/T/+SEovB/G4ejwcfE/CBzO6Edsu+OETZMZ3wcX/EjUkrl5A== core-js@~2.3.0: version "2.3.0" @@ -2935,9 +2940,9 @@ css-selector-tokenizer@^0.7.0: regexpu-core "^1.0.0" css-what@2.1: - version "2.1.2" - resolved "https://registry.yarnpkg.com/css-what/-/css-what-2.1.2.tgz#c0876d9d0480927d7d4920dcd72af3595649554d" - integrity sha512-wan8dMWQ0GUeF7DGEPVjhHemVW/vy6xUYmFzRY8RYqgA0JtXC9rJmbScBjqSu6dg9q0lwPQy6ZAmJVr3PPTvqQ== + version "2.1.3" + resolved "https://registry.yarnpkg.com/css-what/-/css-what-2.1.3.tgz#a6d7604573365fe74686c3f311c56513d88285f2" + integrity sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg== cssesc@^0.1.0: version "0.1.0" @@ -3250,28 +3255,23 @@ doctrine@^2.1.0: esutils "^2.0.2" dom-serializer@0, dom-serializer@~0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.0.tgz#073c697546ce0780ce23be4a28e293e40bc30c82" - integrity sha1-BzxpdUbOB4DOI75KKOKT5AvDDII= + version "0.1.1" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0" + integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA== dependencies: - domelementtype "~1.1.1" - entities "~1.1.1" + domelementtype "^1.3.0" + entities "^1.1.1" domain-browser@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.2.0.tgz#3d31f50191a6749dd1375a7f522e823d42e54eda" integrity sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA== -domelementtype@1, domelementtype@^1.3.0: +domelementtype@1, domelementtype@^1.3.0, domelementtype@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f" integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w== -domelementtype@~1.1.1: - version "1.1.3" - resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.1.3.tgz#bd28773e2642881aec51544924299c5cd822185b" - integrity sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs= - domexception@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/domexception/-/domexception-1.0.1.tgz#937442644ca6a31261ef36e3ec677fe805582c90" @@ -4487,9 +4487,9 @@ global-prefix@^1.0.1: which "^1.2.14" globals@^11.1.0, globals@^11.7.0: - version "11.10.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-11.10.0.tgz#1e09776dffda5e01816b3bb4077c8b59c24eaa50" - integrity sha512-0GZF1RiPKU97IHUO5TORo9w1PwrH/NBPl+fS7oMLdaTRiYmYbwK4NWoZWrAdd0/abG9R2BU+OiwyQpTpE6pdfQ== + version "11.11.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-11.11.0.tgz#dcf93757fa2de5486fbeed7118538adf789e9c2e" + integrity sha512-WHq43gS+6ufNOEqlrDBxVEbb8ntfXrfAUU2ZOpCxrBdGKW3gyv8mCxAfIBD0DroPKGrJ2eSsXsLtY9MPntsyTw== globby@^6.1.0: version "6.1.0" @@ -4633,7 +4633,7 @@ gulp-vinyl-zip@^2.1.2: yauzl "^2.2.1" yazl "^2.2.1" -handlebars@^4.0.11, handlebars@^4.1.0: +handlebars@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.1.0.tgz#0d6a6f34ff1f63cecec8423aa4169827bf787c3a" integrity sha512-l2jRuU1NAWK6AW5qqcTATWQJvNPEwkM7NEKSiv/gqOsoSQbVoWyqVEY5GS+XPQ88zLNmqASRpzfdm8d79hJS+w== @@ -4812,16 +4812,16 @@ html-minifier@^3.5.21: uglify-js "3.4.x" htmlparser2@^3.9.1: - version "3.10.0" - resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.0.tgz#5f5e422dcf6119c0d983ed36260ce9ded0bee464" - integrity sha512-J1nEUGv+MkXS0weHNWVKJJ+UrLfePxRWpN3C9bEi9fLxL2+ggW94DQvgYVXsaT30PGwYRIZKNZXuyMhp3Di4bQ== + version "3.10.1" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f" + integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ== dependencies: - domelementtype "^1.3.0" + domelementtype "^1.3.1" domhandler "^2.3.0" domutils "^1.5.1" entities "^1.1.1" inherits "^2.0.1" - readable-stream "^3.0.6" + readable-stream "^3.1.1" http-cache-semantics@^3.8.1: version "3.8.1" @@ -5090,11 +5090,6 @@ invert-kv@^2.0.0: resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02" integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA== -ip-regex@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" - integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk= - ip@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a" @@ -5568,7 +5563,7 @@ istanbul-lib-hook@^2.0.3: dependencies: append-transform "^1.0.0" -istanbul-lib-instrument@^3.0.1: +istanbul-lib-instrument@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/istanbul-lib-instrument/-/istanbul-lib-instrument-3.1.0.tgz#a2b5484a7d445f1f311e93190813fa56dfb62971" integrity sha512-ooVllVGT38HIk8MxDj/OIHXSYvH+1tq/Vb38s8ixt9GoJadXska4WkGY+0wkmtYCZNYtaARniH/DixUGGLZ0uA== @@ -5601,12 +5596,12 @@ istanbul-lib-source-maps@^3.0.2: rimraf "^2.6.2" source-map "^0.6.1" -istanbul-reports@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-2.1.0.tgz#87b8b55cd1901ba1748964c98ddd8900ce306d59" - integrity sha512-azQdSX+dtTtkQEfqq20ICxWi6eOHXyHIgMFw1VOOVi8iIPWeCWRgCyFh/CsBKIhcgskMI8ExXmU7rjXTRCIJ+A== +istanbul-reports@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-2.1.1.tgz#72ef16b4ecb9a4a7bd0e2001e00f95d1eec8afa9" + integrity sha512-FzNahnidyEPBCI0HcufJoSEoKykesRlFcSzQqjH9x0+LC8tnnE/p/90PBLu8iZTxr8yYZNyTtiAujUqyN+CIxw== dependencies: - handlebars "^4.0.11" + handlebars "^4.1.0" jquery@2.1.4: version "2.1.4" @@ -6050,11 +6045,6 @@ lodash.flattendeep@^4.4.0: resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2" integrity sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI= -lodash.get@^4.4.2: - version "4.4.2" - resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" - integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk= - lodash.islength@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/lodash.islength/-/lodash.islength-4.0.1.tgz#4e9868d452575d750affd358c979543dc20ed577" @@ -6085,7 +6075,7 @@ lodash.uniq@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M= -lodash@^4.13.1, lodash@^4.15.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.5, lodash@^4.3.0, lodash@^4.8.0: +lodash@^4.15.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.5, lodash@^4.3.0, lodash@^4.8.0: version "4.17.11" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg== @@ -6119,9 +6109,13 @@ lolex@^2.3.2: integrity sha512-l9x0+1offnKKIzYVjyXU2SiwhXDLekRzKyhnbyldPHvC7BvLPVpdNUNR2KeMAiCN2D/kLNttZgQD5WjSxuBx3Q== lolex@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/lolex/-/lolex-3.0.0.tgz#f04ee1a8aa13f60f1abd7b0e8f4213ec72ec193e" - integrity sha512-hcnW80h3j2lbUfFdMArd5UPA/vxZJ+G8vobd+wg3nVEQA0EigStbYcrG030FJxL6xiDDPEkoMatV9xIh5OecQQ== + version "3.1.0" + resolved "https://registry.yarnpkg.com/lolex/-/lolex-3.1.0.tgz#1a7feb2fefd75b3e3a7f79f0e110d9476e294434" + integrity sha512-zFo5MgCJ0rZ7gQg69S4pqBsLURbFw11X68C18OcJjJQbqaXm2NoTrGl1IMM3TIz0/BnN1tIs2tzmmqvCsOMMjw== + +"long@git://github.com/dcodeIO/long.js.git#8181a6b50a2a230f0b2a1e4c4093f9b9d19c8b69": + version "4.0.1" + resolved "git://github.com/dcodeIO/long.js.git#8181a6b50a2a230f0b2a1e4c4093f9b9d19c8b69" "long@git://github.com/dcodeIO/long.js.git#8181a6b50a2a230f0b2a1e4c4093f9b9d19c8b69": version "4.0.1" @@ -6468,17 +6462,17 @@ mime-db@1.35.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.35.0.tgz#0569d657466491283709663ad379a99b90d9ab47" integrity sha512-JWT/IcCTsB0Io3AhWUMjRqucrHSPsSf2xKLaRldJVULioggvkJvggZ3VXNNSRkCddE6D+BUI4HEIZIA2OjwIvg== -mime-db@~1.37.0: - version "1.37.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.37.0.tgz#0b6a0ce6fdbe9576e25f1f2d2fde8830dc0ad0d8" - integrity sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg== +mime-db@~1.38.0: + version "1.38.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.38.0.tgz#1a2aab16da9eb167b49c6e4df2d9c68d63d8e2ad" + integrity sha512-bqVioMFFzc2awcdJZIzR3HjZFX20QhilVS7hytkKrv7xFAn8bM1gzc/FOX2awLISvWe0PV8ptFKcon+wZ5qYkg== mime-types@^2.1.12, mime-types@~2.1.18, mime-types@~2.1.19: - version "2.1.21" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.21.tgz#28995aa1ecb770742fe6ae7e58f9181c744b3f96" - integrity sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg== + version "2.1.22" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.22.tgz#fe6b355a190926ab7698c9a0556a11199b2199bd" + integrity sha512-aGl6TZGnhm/li6F7yx82bJiBZwgiEa4Hf6CNr8YO+r5UHr53tSTYZb102zyU50DOWWKeOv0uQLRL0/9EiKWCog== dependencies: - mime-db "~1.37.0" + mime-db "~1.38.0" mime@1.4.1: version "1.4.1" @@ -6747,9 +6741,9 @@ no-case@^2.2.0: lower-case "^1.1.1" node-abi@^2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.7.0.tgz#e2f814088ab97c85504ae2bacb8f93d5d77cbc2b" - integrity sha512-egTtvNoZLMjwxkL/5iiJKYKZgn2im0zP+G+PncMxICYGiD3aZtXUvEsDmu0pF8gpASvLZyD8v53qi1/ELaRZpg== + version "2.7.1" + resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.7.1.tgz#a8997ae91176a5fbaa455b194976e32683cda643" + integrity sha512-OV8Bq1OrPh6z+Y4dqwo05HqrRL9YNF7QVMRfq1/pguwKLG+q9UB/Lk0x5qXjO23JjJg+/jqCHSTaG1P3tfKfuw== dependencies: semver "^5.4.1" @@ -7008,9 +7002,9 @@ nwsapi@^2.0.9: integrity sha512-ZG3bLAvdHmhIjaQ/Db1qvBxsGvFMLIRpQszyqbg31VJ53UP++uZX1/gf3Ut96pdwN9AuDwlMqIYLm0UPCdUeHg== nyc@^13.1.0, nyc@^13.2.0: - version "13.2.0" - resolved "https://registry.yarnpkg.com/nyc/-/nyc-13.2.0.tgz#6a4a4b3f5f97b63ab491c665567557debcfa23d6" - integrity sha512-gQBlOqvfpYt9b2PZ7qElrHWt8x4y8ApNfbMBoDPdl3sY4/4RJwCxDGTSqhA9RnaguZjS5nW7taW8oToe86JLgQ== + version "13.3.0" + resolved "https://registry.yarnpkg.com/nyc/-/nyc-13.3.0.tgz#da4dbe91a9c8b9ead3f4f3344c76f353e3c78c75" + integrity sha512-P+FwIuro2aFG6B0Esd9ZDWUd51uZrAEoGutqZxzrVmYl3qSfkLgcQpBPBjtDFsUQLFY1dvTQJPOyeqr8S9GF8w== dependencies: archy "^1.0.0" arrify "^1.0.1" @@ -7022,10 +7016,10 @@ nyc@^13.1.0, nyc@^13.2.0: glob "^7.1.3" istanbul-lib-coverage "^2.0.3" istanbul-lib-hook "^2.0.3" - istanbul-lib-instrument "^3.0.1" + istanbul-lib-instrument "^3.1.0" istanbul-lib-report "^2.0.4" istanbul-lib-source-maps "^3.0.2" - istanbul-reports "^2.1.0" + istanbul-reports "^2.1.1" make-dir "^1.3.0" merge-source-map "^1.1.0" resolve-from "^4.0.0" @@ -7066,9 +7060,9 @@ object-extended@0.0.7: is-extended "~0.0.3" object-keys@^1.0.11, object-keys@^1.0.12: - version "1.0.12" - resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.12.tgz#09c53855377575310cca62f55bb334abff7b3ed2" - integrity sha512-FTMyFUm2wBcGHnH2eXmz7tC6IwlqQZ6mVZ+6dm6vZ4IQIHjs6FdNsQBuKGPuUUUY6NfJw2PshC08Tn6LzLDOag== + version "1.1.0" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.0.tgz#11bd22348dd2e096a045ab06f6c85bcc340fa032" + integrity sha512-6OO5X1+2tYkNyNEx6TsCxEqFfRWaqx6EtMiSbGrw8Ob8v9Ne+Hl8rBAgLBZn5wjEz3s/s6U1WXFUFOcxxAwUpg== object-visit@^1.0.0: version "1.0.1" @@ -7348,9 +7342,9 @@ parent-module@^1.0.0: callsites "^3.0.0" parse-asn1@^5.0.0: - version "5.1.3" - resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.3.tgz#1600c6cc0727365d68b97f3aa78939e735a75204" - integrity sha512-VrPoetlz7B/FqjBLD2f5wBVZvsZVLnRUrxVLfRYhGXCODa/NWE4p3Wp+6+aV3ZPL3KM7/OZmxDIwwijD7yuucg== + version "5.1.4" + resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.4.tgz#37f6628f823fbdeb2273b4d540434a22f3ef1fcc" + integrity sha512-Qs5duJcuvNExRfFZ99HDD3z4mAi3r9Wl/FOjEOijlxwCZs7E7mW2vjTpgQ4J8LpTF8x5v+1Vn5UQFejmWT11aw== dependencies: asn1.js "^4.0.0" browserify-aes "^1.0.0" @@ -8093,7 +8087,7 @@ read-pkg@^4.0.1: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@^3.0.6: +readable-stream@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.1.1.tgz#ed6bbc6c5ba58b090039ff18ce670515795aeb06" integrity sha512-DkN66hPyqDhnIQ6Jcsvx9bFjhw214O4poMBcIMgPVpQvNy9a0e0Uhg5SqySyDKAmUlwt8LonTBz1ezOnM8pUdA== @@ -8310,21 +8304,21 @@ replace-ext@1.0.0, replace-ext@^1.0.0: resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.0.tgz#de63128373fcbf7c3ccfa4de5a480c45a67958eb" integrity sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs= -request-promise-core@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.1.tgz#3eee00b2c5aa83239cfb04c5700da36f81cd08b6" - integrity sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY= +request-promise-core@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/request-promise-core/-/request-promise-core-1.1.2.tgz#339f6aababcafdb31c799ff158700336301d3346" + integrity sha512-UHYyq1MO8GsefGEt7EprS8UrXsm1TxEvFUX1IMTuSLU2Rh7fTIdFtl8xD7JiEYiWU2dl+NYAjCTksTehQUxPag== dependencies: - lodash "^4.13.1" + lodash "^4.17.11" request-promise-native@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/request-promise-native/-/request-promise-native-1.0.5.tgz#5281770f68e0c9719e5163fd3fab482215f4fda5" - integrity sha1-UoF3D2jgyXGeUWP9P6tIIhX0/aU= + version "1.0.7" + resolved "https://registry.yarnpkg.com/request-promise-native/-/request-promise-native-1.0.7.tgz#a49868a624bdea5069f1251d0a836e0d89aa2c59" + integrity sha512-rIMnbBdgNViL37nZ1b3L/VfPOpSi0TqVDQPAvO6U14lMzOLrt5nilxCQqtDKhZeDiW0/hkCXGoQjhgJd/tCh6w== dependencies: - request-promise-core "1.1.1" - stealthy-require "^1.1.0" - tough-cookie ">=2.3.3" + request-promise-core "1.1.2" + stealthy-require "^1.1.1" + tough-cookie "^2.3.3" request@^2.79.0, request@^2.88.0: version "2.88.0" @@ -8766,7 +8760,7 @@ slice-ansi@^1.0.0: dependencies: is-fullwidth-code-point "^2.0.0" -slice-ansi@^2.0.0: +slice-ansi@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-2.1.0.tgz#cacd7693461a637a5788d92a7dd4fba068e81636" integrity sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ== @@ -9001,7 +8995,7 @@ statuses@~1.4.0: resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.4.0.tgz#bb73d446da2796106efcc1b601a253d6c46bd087" integrity sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew== -stealthy-require@^1.1.0: +stealthy-require@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks= @@ -9265,14 +9259,14 @@ symbol-tree@^3.2.2: integrity sha1-rifbOPZgp64uHDt9G8KQgZuFGeY= table@^5.0.2: - version "5.2.2" - resolved "https://registry.yarnpkg.com/table/-/table-5.2.2.tgz#61d474c9e4d8f4f7062c98c7504acb3c08aa738f" - integrity sha512-f8mJmuu9beQEDkKHLzOv4VxVYlU68NpdzjbGPl69i4Hx0sTopJuNxuzJd17iV2h24dAfa93u794OnDA5jqXvfQ== + version "5.2.3" + resolved "https://registry.yarnpkg.com/table/-/table-5.2.3.tgz#cde0cc6eb06751c009efab27e8c820ca5b67b7f2" + integrity sha512-N2RsDAMvDLvYwFcwbPyF3VmVSSkuF+G1e+8inhBLtHpvwXGw4QRPEZhihQNeEN0i1up6/f6ObCJXNdlRG3YVyQ== dependencies: - ajv "^6.6.1" + ajv "^6.9.1" lodash "^4.17.11" - slice-ansi "^2.0.0" - string-width "^2.1.1" + slice-ansi "^2.1.0" + string-width "^3.0.0" tapable@^1.0.0, tapable@^1.1.0: version "1.1.1" @@ -9490,16 +9484,7 @@ to-through@^2.0.0: dependencies: through2 "^2.0.3" -tough-cookie@>=2.3.3: - version "3.0.1" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-3.0.1.tgz#9df4f57e739c26930a018184887f4adb7dca73b2" - integrity sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg== - dependencies: - ip-regex "^2.1.0" - psl "^1.1.28" - punycode "^2.1.1" - -tough-cookie@^2.5.0: +tough-cookie@^2.3.3, tough-cookie@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== From e1ded3a4a7076dd94af8a5df2b5b9f8fba87efb7 Mon Sep 17 00:00:00 2001 From: Jesus David Garcia Gomez Date: Fri, 15 Feb 2019 09:42:20 -0800 Subject: [PATCH 2/7] Breaking: Remove traversing html from `parser-html` --- packages/parser-html/README.md | 8 +++- packages/parser-html/src/parser.ts | 66 +++------------------------- packages/parser-html/tests/parser.ts | 15 ------- 3 files changed, 13 insertions(+), 76 deletions(-) diff --git a/packages/parser-html/README.md b/packages/parser-html/README.md index 70ad1641745..b277212a531 100644 --- a/packages/parser-html/README.md +++ b/packages/parser-html/README.md @@ -34,7 +34,7 @@ And then activate it via the [`.hintrc`][hintrc] configuration file: ## Events emitted -This `parser` emits the event `parse::html` of type `HTMLParse` +This `parser` emits the event `parse::end::html` of type `HTMLParse` which has the following information: * `window`: an [`IAsyncWindow`][asynchtml] object containing the @@ -42,6 +42,11 @@ which has the following information: * `html`: a string containing the raw HTML source code. * `resource`: the parsed resource. +And the event `parse::start::html` of type `Event` which has the +following information: + +* `resource`: the resource that is going to be parsed. + This `parser` also automatically traverses and emits events for elements in the tree (see [events][events] for details): @@ -50,7 +55,6 @@ elements in the tree (see [events][events] for details): * `traverse::end` * `traverse::start` * `traverse::up` -* `parse::html::end` diff --git a/packages/parser-html/src/parser.ts b/packages/parser-html/src/parser.ts index 3a73e2a97a6..fae737c73ee 100644 --- a/packages/parser-html/src/parser.ts +++ b/packages/parser-html/src/parser.ts @@ -26,82 +26,30 @@ try { // `canvas` is not installed, nothing to do } -import { JSDOM } from 'jsdom'; -import { JSDOMAsyncHTMLElement, JSDOMAsyncWindow } from 'hint/dist/src/lib/types/jsdom-async-html'; -import { Event, FetchEnd, Parser, TraverseDown, TraverseUp } from 'hint/dist/src/lib/types'; +import { Parser, FetchEnd } from 'hint/dist/src/lib/types'; import { Engine } from 'hint'; +import createAsyncWindow from 'hint/dist/src/lib/utils/dom/create-async-window'; import { HTMLEvents } from './types'; export * from './types'; export default class HTMLParser extends Parser { - private _url = ''; - public constructor(engine: Engine) { super(engine, 'html'); - engine.on('fetch::end::html', this.onFetchEndHtml.bind(this)); + engine.on('fetch::end::html', this.onFetchEnd.bind(this)); } - private async onFetchEndHtml(fetchEnd: FetchEnd) { - const resource = this._url = fetchEnd.response.url; + private async onFetchEnd(fetchEnd: FetchEnd) { + const resource = fetchEnd.resource; await this.engine.emitAsync(`parse::start::html`, { resource }); const html = fetchEnd.response.body.content; - const dom = new JSDOM(html, { - - /** Needed to provide line/column positions for elements. */ - includeNodeLocations: true, - - /** - * Needed to let hints run script against the DOM. - * However the page itself is kept static because `connector-local` - * validates files individually without loading resources. - */ - runScripts: 'outside-only' - - }); - - const window = new JSDOMAsyncWindow(dom.window, dom); - const documentElement = dom.window.document.documentElement; - - await this.engine.emitAsync(`parse::end::html`, { html, resource, window }); - - const event = { resource } as Event; - - /* istanbul ignore if */ - if (!documentElement) { - return; - } - - await this.engine.emitAsync('traverse::start', event); - await this.traverseAndNotify(documentElement, dom); - await this.engine.emitAsync('traverse::end', event); - } - - /** Traverses the DOM while sending `element::typeofelement` events. */ - private async traverseAndNotify(element: HTMLElement, dom: JSDOM): Promise { - - await this.engine.emitAsync(`element::${element.tagName.toLowerCase()}` as 'element::*', { - element: new JSDOMAsyncHTMLElement(element, dom), - resource: this._url - }); - - const traverseEvent = { - element: new JSDOMAsyncHTMLElement(element, dom), - resource: this._url - } as TraverseDown | TraverseUp; - - await this.engine.emitAsync(`traverse::down`, traverseEvent); - - // Recursively traverse child elements. - for (let i = 0; i < element.children.length; i++) { - await this.traverseAndNotify(element.children[i] as HTMLElement, dom); - } + const window = createAsyncWindow(html, true); - await this.engine.emitAsync(`traverse::up`, traverseEvent); + await this.engine.emitAsync('parse::end::html', { html, resource, window }); } } diff --git a/packages/parser-html/tests/parser.ts b/packages/parser-html/tests/parser.ts index 41892ce22ac..2b0a5992e8b 100644 --- a/packages/parser-html/tests/parser.ts +++ b/packages/parser-html/tests/parser.ts @@ -56,21 +56,6 @@ test('If `fetch::end::html` is received, then the code should be parsed and the t.is(id && id.value, 'test'); t.true(div.isSame(div2)); - t.is(args[3][0], 'traverse::start'); - t.is(args[4][0], 'element::html'); - t.is(args[5][0], 'traverse::down'); - t.is(args[6][0], 'element::head'); - t.is(args[7][0], 'traverse::down'); - t.is(args[8][0], 'traverse::up'); - t.is(args[9][0], 'element::body'); - t.is(args[10][0], 'traverse::down'); - t.is(args[11][0], 'element::div'); - t.is(args[12][0], 'traverse::down'); - t.is(args[13][0], 'traverse::up'); - t.is(args[14][0], 'traverse::up'); - t.is(args[15][0], 'traverse::up'); - t.is(args[16][0], 'traverse::end'); - sandbox.restore(); }); From 80043c1af48ac9d0bf9842bd8ffbb036b5c02b44 Mon Sep 17 00:00:00 2001 From: Jesus David Garcia Gomez Date: Fri, 15 Feb 2019 09:45:08 -0800 Subject: [PATCH 3/7] Breaking: Update jsdom to use a copy of the DOM Fix #1881 --- packages/connector-jsdom/src/connector.ts | 59 +++++------------------ 1 file changed, 11 insertions(+), 48 deletions(-) diff --git a/packages/connector-jsdom/src/connector.ts b/packages/connector-jsdom/src/connector.ts index 54ed30abddf..891bc5bb128 100644 --- a/packages/connector-jsdom/src/connector.ts +++ b/packages/connector-jsdom/src/connector.ts @@ -36,13 +36,16 @@ import { getContentTypeData, getType } from 'hint/dist/src/lib/utils/content-typ import { HttpHeaders, IConnector, - ElementFound, Event, FetchEnd, FetchError, TraverseDown, TraverseUp, + Event, FetchEnd, FetchError, NetworkData } from 'hint/dist/src/lib/types'; -import { JSDOMAsyncHTMLElement, JSDOMAsyncHTMLDocument } from 'hint/dist/src/lib/types/jsdom-async-html'; +import { JSDOMAsyncHTMLElement, JSDOMAsyncHTMLDocument, JSDOMAsyncWindow } from 'hint/dist/src/lib/types/jsdom-async-html'; import { Engine } from 'hint/dist/src/lib/engine'; import isHTMLDocument from 'hint/dist/src/lib/utils/network/is-html-document'; +import createAsyncWindow from 'hint/dist/src/lib/utils/dom/create-async-window'; +import traverse from 'hint/dist/src/lib/utils/dom/traverse'; import { Requester } from '@hint/utils-connector-tools/dist/src/requester'; + import CustomResourceLoader from './resource-loader'; import { beforeParse } from './before-parse'; @@ -109,47 +112,6 @@ export default class JSDOMConnector implements IConnector { return r.get(uri); } - /** Traverses the DOM while sending `element::typeofelement` events. */ - private async traverseAndNotify(element: HTMLElement) { - const eventName = `element::${element.nodeName.toLowerCase()}` as 'element::*'; - - debug(`emitting ${eventName}`); - /* - * should we freeze it? what about the other siblings, children, - * parents? We should have an option to not allow modifications - * maybe we create a custom object that only exposes read only - * properties? - */ - const event: ElementFound = { - element: new JSDOMAsyncHTMLElement(element), - resource: this.finalHref - }; - - await this.server.emitAsync(eventName, event); - for (let i = 0; i < element.children.length; i++) { - const child: HTMLElement = element.children[i] as HTMLElement; - - debug('next children'); - const traverseDown: TraverseDown = { - element: new JSDOMAsyncHTMLElement(element), - resource: this.finalHref - }; - - await this.server.emitAsync(`traverse::down`, traverseDown); - await this.traverseAndNotify(child); - - } - - const traverseUp: TraverseUp = { - element: new JSDOMAsyncHTMLElement(element), - resource: this.finalHref - }; - - await this.server.emitAsync(`traverse::up`, traverseUp); - - return Promise.resolve(); - } - /** * JSDOM doesn't download the favicon automatically, this method: * @@ -200,11 +162,10 @@ export default class JSDOMConnector implements IConnector { try { this._targetNetworkData = await this.fetchContent(target); - } catch (err) { + } catch (err) /* istanbul ignore next */ { const hops: string[] = this.request.getRedirects(err.uri); const fetchError: FetchError = { element: null as any, - /* istanbul ignore next */ error: err.error ? err.error : err, hops, resource: href @@ -293,9 +254,11 @@ export default class JSDOMConnector implements IConnector { debug(`${this.finalHref} loaded, traversing`); try { - await this.server.emitAsync('traverse::start', event); - await this.traverseAndNotify(window.document.children[0] as HTMLElement); - await this.server.emitAsync('traverse::end', event); + const html = await this.html; + + const asyncWindow = createAsyncWindow(html); + + await traverse((asyncWindow as JSDOMAsyncWindow).dom!, this.server, this.finalHref); // We download only the first favicon found await this.getFavicon(window.document.querySelector('link[rel~="icon"]')); From e69627642b699b019e8fe3b54779164a6c218d91 Mon Sep 17 00:00:00 2001 From: Jesus David Garcia Gomez Date: Fri, 15 Feb 2019 09:45:43 -0800 Subject: [PATCH 4/7] Breaking: Update CDP to use a copy of the DOM Fix #1879 --- .../package.json | 6 +- .../src/cdp-async-html.ts | 233 ------------------ .../src/debugging-protocol-connector.ts | 93 ++----- .../src/request-response.ts | 6 +- 4 files changed, 30 insertions(+), 308 deletions(-) delete mode 100644 packages/utils-debugging-protocol-common/src/cdp-async-html.ts diff --git a/packages/utils-debugging-protocol-common/package.json b/packages/utils-debugging-protocol-common/package.json index a44dff17486..383917c79ed 100644 --- a/packages/utils-debugging-protocol-common/package.json +++ b/packages/utils-debugging-protocol-common/package.json @@ -10,17 +10,17 @@ "devDependencies": { "@types/lodash": "^4.14.121", "@types/node": "11.9.4", + "@typescript-eslint/eslint-plugin": "^1.3.0", + "@typescript-eslint/parser": "1.3.0", "chrome-remote-debug-protocol": "^1.2.20180422", "cpx": "^1.5.0", "eslint": "^5.13.0", "eslint-plugin-import": "^2.16.0", "eslint-plugin-markdown": "^1.0.0", - "@typescript-eslint/eslint-plugin": "^1.3.0", "hint": "^4.4.0", "npm-run-all": "^4.1.5", "rimraf": "^2.6.3", - "typescript": "^3.3.3", - "@typescript-eslint/parser": "1.3.0" + "typescript": "^3.3.3" }, "engines": { "node": ">=8.0.0" diff --git a/packages/utils-debugging-protocol-common/src/cdp-async-html.ts b/packages/utils-debugging-protocol-common/src/cdp-async-html.ts deleted file mode 100644 index 7962c89231b..00000000000 --- a/packages/utils-debugging-protocol-common/src/cdp-async-html.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { Crdp } from 'chrome-remote-debug-protocol'; - -import { IAsyncHTMLDocument, IAsyncHTMLElement } from 'hint/dist/src/lib/types/async-html'; //eslint-disable-line -import { debug as d } from 'hint/dist/src/lib/utils/debug'; -import { ProblemLocation } from 'hint/dist/src/lib/types'; - -const debug: debug.IDebugger = d(__filename); - -/** An implementation of AsyncHTMLDocument on top of the Chrome Debugging Protocol */ -export class CDPAsyncHTMLDocument implements IAsyncHTMLDocument { - /** The DOM domain of the CDP client. */ - private _DOM: Crdp.DOMClient; - /** The root element of the real DOM. */ - private _dom!: Crdp.DOM.Node; - /** A map with all the nodes accessible using `nodeId`. */ - private _nodes: Map = new Map(); - /** outerHTML of the page. */ - private _outerHTML: string = ''; - - public constructor(DOM: Crdp.DOMClient) { - this._DOM = DOM; - } - - /** - * When doing requests like `querySelectorAll`, we receive an array of nodeIds. Instead - * of having to do another request for the node, and because we are getting the whole DOM - * initially, we store them in a Map using the `nodeId` as the key so we can access to them - * later. - */ - private trackNodes(root: Crdp.DOM.Node) { - this._nodes.set(root.nodeId, root); - if (!root.children) { - return; - } - - root.children.forEach((child) => { - this.trackNodes(child); - }); - } - - private getHTMLChildren(children: Crdp.DOM.Node[]) { - return children.find((item) => { - return item.nodeType === 1 && item.nodeName === 'HTML'; - }); - } - - /* - * ------------------------------------------------------------------------------ - * Public methods - * ------------------------------------------------------------------------------ - */ - - public async querySelectorAll(selector: string): Promise { - let nodeIds; - - try { - nodeIds = (await this._DOM.querySelectorAll!({ nodeId: this._dom.nodeId, selector })).nodeIds; - } catch (e) { - debug(e); - - return []; - } - - const nodes: AsyncHTMLElement[] = []; - - for (let i = 0; i < nodeIds.length; i++) { - const nodeId = nodeIds[i]; - const node = this._nodes.get(nodeId); - - if (node) { - nodes.push(new AsyncHTMLElement(node, this, this._DOM)); // eslint-disable-line no-use-before-define, @typescript-eslint/no-use-before-define - } else { - /* - * This node was added in the DOM and we don't have it cached - * TODO: uncomment once chrome 62 is released. - * See https://github.com/cyrus-and/chrome-remote-interface/issues/255 for more information - * node = await this._DOM.describeNode({ nodeId }); - * this._nodes.set(nodeId, node); - */ - } - } - - return nodes; - } - - public async pageHTML(): Promise { - if (this._outerHTML) { - return Promise.resolve(this._outerHTML); - } - - let { outerHTML } = await this._DOM.getOuterHTML!({ nodeId: this._dom.nodeId }); - - /* - * Some browsers like Edge don't have the property outerHTML in the root element - * so we need to find the html element - */ - if (!outerHTML) { - const htmlElement = this.getHTMLChildren(this._dom.children!); - - ({ outerHTML } = await this._DOM.getOuterHTML!({ nodeId: htmlElement!.nodeId })); // HTML documents always have a root (the parser creates one) - } - - this._outerHTML = outerHTML; - - return outerHTML; - } - - public async load() { - const { root: dom } = await this._DOM.getDocument!({ depth: -1 }); - - this.trackNodes(dom); - this._dom = dom; - } - - /* - * ------------------------------------------------------------------------------ - * Getters - * ------------------------------------------------------------------------------ - */ - - public get root() { - return this._dom; - } -} - -export const createCDPAsyncHTMLDocument = async (DOM: Crdp.DOMClient) => { - const dom = new CDPAsyncHTMLDocument(DOM); - - await dom.load(); - - return dom; -}; - -/** An implementation of AsyncHTMLElement on top of the Chrome Debugging Protocol */ -export class AsyncHTMLElement implements IAsyncHTMLElement { - protected _htmlelement: Crdp.DOM.Node; - private _ownerDocument: CDPAsyncHTMLDocument; - private _DOM: Crdp.DOMClient; - private _outerHTML: string = ''; - private _attributesArray: { name: string; value: string }[] = []; - private _attributesMap: Map = new Map(); - - public constructor(htmlelement: Crdp.DOM.Node, ownerDocument: CDPAsyncHTMLDocument, DOM: Crdp.DOMClient) { - if (typeof htmlelement === 'number') { - throw new Error(); - } - this._htmlelement = htmlelement; - this._DOM = DOM; - this._ownerDocument = ownerDocument; - } - - private initializeAttributes() { - const attributes = this._htmlelement.attributes || []; - - for (let i = 0; i < attributes.length; i += 2) { - this._attributesArray.push({ name: attributes[i], value: attributes[i + 1] }); - this._attributesMap.set(attributes[i], attributes[i + 1]); - } - } - - /* - * ------------------------------------------------------------------------------ - * Public methods - * ------------------------------------------------------------------------------ - */ - - public getAttribute(name: string): string | null { - if (this._attributesArray.length === 0) { - this.initializeAttributes(); - } - - const value = this._attributesMap.get(name); - - return typeof value === 'string' ? value : null; - } - - public getLocation(): ProblemLocation | null { - return null; - } - - public isSame(element: AsyncHTMLElement): boolean { - return this._htmlelement.nodeId === element._htmlelement.nodeId; - } - - public async outerHTML(): Promise { - if (this._outerHTML) { - return Promise.resolve(this._outerHTML); - } - - let outerHTML = ''; - - try { - ({ outerHTML } = await this._DOM.getOuterHTML!({ nodeId: this._htmlelement.nodeId })); - } catch (e) { - debug(`Error trying to get outerHTML for node ${this._htmlelement.nodeId}`); - debug(e); - } - - this._outerHTML = outerHTML; - - return outerHTML; - } - - /* - * ------------------------------------------------------------------------------ - * Getters - * ------------------------------------------------------------------------------ - */ - - public get attributes() { - if (this._attributesArray.length === 0) { - this.initializeAttributes(); - } - - return this._attributesArray; - } - - public get children() { - if (this._htmlelement.children) { - return this._htmlelement.children; - } - - return []; - } - - public get nodeName(): string { - return this._htmlelement.nodeName; - } - - public get ownerDocument(): IAsyncHTMLDocument { - return this._ownerDocument; - } -} diff --git a/packages/utils-debugging-protocol-common/src/debugging-protocol-connector.ts b/packages/utils-debugging-protocol-common/src/debugging-protocol-connector.ts index 2e753bc4045..b2449ff9a2c 100644 --- a/packages/utils-debugging-protocol-common/src/debugging-protocol-connector.ts +++ b/packages/utils-debugging-protocol-common/src/debugging-protocol-connector.ts @@ -21,17 +21,19 @@ import { compact, filter } from 'lodash'; import { Crdp } from 'chrome-remote-debug-protocol'; -import { createCDPAsyncHTMLDocument, CDPAsyncHTMLDocument, AsyncHTMLElement } from './cdp-async-html'; import { getType } from 'hint/dist/src/lib/utils/content-type'; import { debug as d } from 'hint/dist/src/lib/utils/debug'; import cutString from 'hint/dist/src/lib/utils/misc/cut-string'; import delay from 'hint/dist/src/lib/utils/misc/delay'; import isHTMLDocument from 'hint/dist/src/lib/utils/network/is-html-document'; +import createAsyncWindow from 'hint/dist/src/lib/utils/dom/create-async-window'; +import { JSDOMAsyncWindow } from 'hint/dist/src/lib/types/jsdom-async-html'; +import traverse from 'hint/dist/src/lib/utils/dom/traverse'; import { BrowserInfo, IConnector, - ElementFound, Event, FetchEnd, FetchError, ILauncher, TraverseUp, TraverseDown, - Response, Request, NetworkData, HttpHeaders, IAsyncHTMLElement + Event, FetchEnd, FetchError, ILauncher, + Response, Request, NetworkData, HttpHeaders, IAsyncHTMLElement, IAsyncHTMLDocument } from 'hint/dist/src/lib/types'; import { normalizeHeaders } from '@hint/utils-connector-tools/dist/src/normalize-headers'; @@ -63,7 +65,7 @@ export class Connector implements IConnector { /** Indicates if there has been an error loading the page (e.g.: it doesn't exists). */ private _errorWithPage: boolean = false; /** The DOM abstraction on top of adapter. */ - private _dom: CDPAsyncHTMLDocument | undefined; + private _dom: IAsyncHTMLDocument | undefined; /** A collection of requests with their initial data. */ private _pendingResponseReceived: Function[]; /** List of all the tabs used by the connector. */ @@ -114,9 +116,9 @@ export class Connector implements IConnector { * ------------------------------------------------------------------------------ */ - private async getElementFromParser(parts: string[], dom: CDPAsyncHTMLDocument): Promise { + private async getElementFromParser(parts: string[], dom: IAsyncHTMLDocument): Promise { let basename: string | null = null; - let elements: AsyncHTMLElement[] = []; + let elements: IAsyncHTMLElement[] = []; while (parts.length > 0) { basename = !basename ? parts.pop()! : `${parts.pop()}/${basename}`; @@ -137,7 +139,7 @@ export class Connector implements IConnector { decodeBasename = basename; } const query: string = `[src$="${basename}" i],[href$="${basename}" i],[src$="${decodeBasename}" i],[href$="${decodeBasename}" i]`; - const newElements: AsyncHTMLElement[] = await dom.querySelectorAll(query); + const newElements: IAsyncHTMLElement[] = await dom.querySelectorAll(query); if (newElements.length === 0) { if (elements.length > 0) { @@ -166,7 +168,7 @@ export class Connector implements IConnector { } /** Returns the IAsyncHTMLElement that initiated a request */ - private async getElementFromRequest(requestId: string, dom: CDPAsyncHTMLDocument): Promise { + private async getElementFromRequest(requestId: string, dom: IAsyncHTMLDocument): Promise { const sourceRequest = this._requests.get(requestId); if (!sourceRequest) { @@ -271,7 +273,7 @@ export class Connector implements IConnector { await this._server.emitAsync(eventName, event); } - private async emitFetchEnd(requestResponse: RequestResponse, dom: CDPAsyncHTMLDocument | null) { + private async emitFetchEnd(requestResponse: RequestResponse, dom: IAsyncHTMLDocument | null) { const resourceUrl: string = requestResponse.finalUrl; const hops: string[] = requestResponse.hops; const originalUrl: string = hops[0] || resourceUrl; @@ -424,54 +426,6 @@ export class Connector implements IConnector { await this.emitFetchEnd(requestResponse, this._dom); } - /** Traverses the DOM notifying when a new element is traversed. */ - private async traverseAndNotify(element: Crdp.DOM.Node) { - /* - * CDP returns more elements than the ones we want. For example there - * are 2 HTML elements. One has children and has `nodeType === 1`, - * while the other doesn't have children and `nodeType === 10`. - * We ignore those elements. - * - * 10: `HTML` with no children - */ - const ignoredNodeTypes: number[] = [10]; - - if (ignoredNodeTypes.includes(element.nodeType)) { - return; - } - - const eventName = `element::${element.nodeName.toLowerCase()}` as 'element::*'; - - // If we are traversing, we know `this._dom` exists already - const wrappedElement: AsyncHTMLElement = new AsyncHTMLElement(element, this._dom!, this._client.DOM); - - const event: ElementFound = { - element: wrappedElement, - resource: this._finalHref - }; - - await this._server.emitAsync(eventName, event); - - const elementChildren = wrappedElement.children; - - for (const child of elementChildren) { - const traverseDown: TraverseDown = { - element: wrappedElement, - resource: this._finalHref - }; - - await this._server.emitAsync(`traverse::down`, traverseDown); - await this.traverseAndNotify(child); - } - - const traverseUp: TraverseUp = { - element: wrappedElement, - resource: this._finalHref - }; - - await this._server.emitAsync(`traverse::up`, traverseUp); - } - /** Wait until the browser load the first tab */ private getClient(port: number, tab: number): Promise { let retries: number = 0; @@ -615,7 +569,7 @@ export class Connector implements IConnector { * * uses the `src` attribute of `` if present. * * uses `favicon.ico` and the final url after redirects. */ - private async getFavicon(dom: CDPAsyncHTMLDocument) { + private async getFavicon(dom: IAsyncHTMLDocument) { const element = (await dom.querySelectorAll('link[rel~="icon"]'))[0]; const href = (element && element.getAttribute('href')) || '/favicon.ico'; @@ -677,12 +631,8 @@ export class Connector implements IConnector { this._client.Network.disable!(); } - const { DOM } = this._client; const event: Event = { resource: this._finalHref }; - this._dom = await createCDPAsyncHTMLDocument(DOM); - - await this.processPendingResponses(); /* * If the target is not an HTML we don't need to * traverse it. @@ -693,6 +643,16 @@ export class Connector implements IConnector { return callback(); } + const node = await this._client.DOM.getDocument!({ depth: -1 }); + const html = (await this._client.DOM.getOuterHTML!({ nodeId: node.root.nodeId })).outerHTML; + const window = createAsyncWindow(html); + + this._dom = window.document; + + await this.processPendingResponses(); + + await traverse((window as JSDOMAsyncWindow).dom!, this._server, node.root.documentURL! || this._finalHref); + /* * Headless chrome does not download the favicon automatically. * Also we can do it at the end because it uses `fetchContent` @@ -703,12 +663,9 @@ export class Connector implements IConnector { if (this._launcher.options && this._launcher.options.flags && this._launcher.options.flags.includes('--headless')) { - await this.getFavicon(this._dom); + await this.getFavicon(this._dom!); } - await this._server.emitAsync('traverse::start', event); - await this.traverseAndNotify(this._dom.root); - await this._server.emitAsync('traverse::end', event); await this._server.emitAsync('can-evaluate::script', event); // We let time to any pending things (like error networks and so) to happen in the next second @@ -969,7 +926,7 @@ export class Connector implements IConnector { }); } - public querySelectorAll(selector: string): Promise { + public querySelectorAll(selector: string): Promise { if (!this._dom) { return Promise.resolve([]); } @@ -983,7 +940,7 @@ export class Connector implements IConnector { * ------------------------------------------------------------------------------ */ - public get dom(): CDPAsyncHTMLDocument | undefined { + public get dom(): IAsyncHTMLDocument | undefined { return this._dom; } diff --git a/packages/utils-debugging-protocol-common/src/request-response.ts b/packages/utils-debugging-protocol-common/src/request-response.ts index 88885e57428..1681f0070de 100644 --- a/packages/utils-debugging-protocol-common/src/request-response.ts +++ b/packages/utils-debugging-protocol-common/src/request-response.ts @@ -2,13 +2,11 @@ import { atob } from 'abab'; import { Crdp } from 'chrome-remote-debug-protocol'; import { getContentTypeData } from 'hint/dist/src/lib/utils/content-type'; -import { HttpHeaders, Response } from 'hint/dist/src/lib/types'; +import { HttpHeaders, Response, IAsyncHTMLElement } from 'hint/dist/src/lib/types'; import { debug as d } from 'hint/dist/src/lib/utils/debug'; import { Requester } from '@hint/utils-connector-tools/dist/src/requester'; import { normalizeHeaders } from '@hint/utils-connector-tools/dist/src/normalize-headers'; -import { AsyncHTMLElement } from './cdp-async-html'; - const debug: debug.IDebugger = d(__filename); export enum RequestStatus { @@ -161,7 +159,7 @@ export class RequestResponse { private _response: Response | undefined; /** The `Response` associated to this request to be sent on the `fetch::end` event. */ - public getResponse(element: AsyncHTMLElement | null): Response { + public getResponse(element: IAsyncHTMLElement | null): Response { if (!this._response) { const { headers, status } = this.responseReceived!.response; From a592080c47510a88f3425857f97d8e0299423b3c Mon Sep 17 00:00:00 2001 From: Jesus David Garcia Gomez Date: Fri, 15 Feb 2019 09:46:32 -0800 Subject: [PATCH 5/7] Fix: Update `connector-local` to traverse using the tools in `hint` --- packages/connector-local/src/connector.ts | 5 +++++ packages/connector-local/tests/tests.ts | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/connector-local/src/connector.ts b/packages/connector-local/src/connector.ts index 23950ebf5fc..aeb4ab95994 100644 --- a/packages/connector-local/src/connector.ts +++ b/packages/connector-local/src/connector.ts @@ -24,6 +24,8 @@ import { getAsUri } from 'hint/dist/src/lib/utils/network/as-uri'; import asPathString from 'hint/dist/src/lib/utils/network/as-path-string'; import { getContentTypeData, isTextMediaType, getType } from 'hint/dist/src/lib/utils/content-type'; import { IAsyncHTMLDocument, IAsyncHTMLElement, IAsyncWindow } from 'hint/dist/src/lib/types/async-html'; +import { JSDOMAsyncWindow } from 'hint/src/lib/types/jsdom-async-html'; +import traverse from 'hint/dist/src/lib/utils/dom/traverse'; import isFile from 'hint/dist/src/lib/utils/fs/is-file'; import cwd from 'hint/dist/src/lib/utils/fs/cwd'; @@ -258,6 +260,9 @@ export default class LocalConnector implements IConnector { /* istanbul ignore next */ private async onParseHTML(event: HTMLParse) { this._window = event.window; + + await traverse((event.window as JSDOMAsyncWindow).dom!, this.engine, event.resource); + await this.engine.emitAsync('can-evaluate::script', { resource: this._href } as CanEvaluateScript); } diff --git a/packages/connector-local/tests/tests.ts b/packages/connector-local/tests/tests.ts index 30d9aa2e176..83db447da61 100644 --- a/packages/connector-local/tests/tests.ts +++ b/packages/connector-local/tests/tests.ts @@ -9,7 +9,8 @@ import delay from 'hint/dist/src/lib/utils/misc/delay'; import asPathString from 'hint/dist/src/lib/utils/network/as-path-string'; import { getAsUri } from 'hint/dist/src/lib/utils/network/as-uri'; import { Engine } from 'hint'; -import { Events, FetchEnd, Problem } from 'hint/dist/src/lib/types'; +import { FetchEnd, Problem } from 'hint/dist/src/lib/types'; +import { HTMLEvents } from '@hint/parser-html'; type SandboxContext = { sandbox: sinon.SinonSandbox; @@ -23,10 +24,10 @@ const mockContext = () => { clear() { }, async emitAsync(event: Event, data: Problem[]): Promise { }, async notify() { }, - on(): Engine { + on(): Engine { return null as any; } - } as Partial>; + } as Partial>; const chokidar = { watch(target: string, options: Chokidar.WatchOptions): Stream { From 2d686ca5c6fba2713a88fe501b49513f00a0d23e Mon Sep 17 00:00:00 2001 From: Jesus David Garcia Gomez Date: Fri, 15 Feb 2019 09:47:35 -0800 Subject: [PATCH 6/7] Fix: Test for the new way to traverse the HTML --- packages/hint-compat-api/tests/html-next.ts | 14 +++++++------- packages/hint-compat-api/tests/html.ts | 16 ++++++++-------- .../tests/tests.ts | 8 ++++---- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/hint-compat-api/tests/html-next.ts b/packages/hint-compat-api/tests/html-next.ts index c75f05be9db..017dde39f96 100644 --- a/packages/hint-compat-api/tests/html-next.ts +++ b/packages/hint-compat-api/tests/html-next.ts @@ -41,7 +41,7 @@ hintRunner.testHint(hintPath, elementAttrAddedAlwaysTrue, { browserslist: ['> 1% const elementAttrVersionAddedFalse: HintTest[] = [ { name: 'Element attributes that have version added as false and not deprecated should fail.', - reports: [{ message: 'srcset attribute of the img element is not supported by ie.', position: { column: 9, line: 5 }}], + reports: [{ message: 'srcset attribute of the img element is not supported by ie.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('img-srcset') } ]; @@ -87,7 +87,7 @@ hintRunner.testHint(hintPath, elementAddedVersionOfTargetedBrowser, { browsersli const elementAddedInVersionAfterTargetedBrowserVersion: HintTest[] = [ { name: 'Elements added in version after targeted browser should fail.', - reports: [{ message: 'video element is not supported by ie 8.', position: { column: 9, line: 5 }}], + reports: [{ message: 'video element is not supported by ie 8.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('video') } ]; @@ -118,7 +118,7 @@ hintRunner.testHint(hintPath, globalAttrVersionAddedNull, { browserslist: ['last const globalAttrVersionAddedFalse: HintTest[] = [ { name: 'Global attributes that have version added as false and not deprecated should fail.', - reports: [{ message: 'global attribute dropzone is not supported by edge, firefox, ie.', position: { column: 9, line: 5 }}], + reports: [{ message: 'global attribute dropzone is not supported by edge, firefox, ie.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('global-attr-dropzone') } ]; @@ -146,7 +146,7 @@ hintRunner.testHint(hintPath, globalAttrAddedVersionOfTargetedBrowser, { browser const globalAttrAddedInVersionAfterTargetedBrowserVersion: HintTest[] = [ { name: 'Global attributes added in version after targeted browser should fail.', - reports: [{ message: 'global attribute class is not supported by firefox 31.', position: { column: 9, line: 5 }}], + reports: [{ message: 'global attribute class is not supported by firefox 31.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('div') } ]; @@ -169,7 +169,7 @@ hintRunner.testHint(hintPath, inputTypeVersionAddedNull, { browserslist: ['last const inputTypeVersionAddedFalse: HintTest[] = [ { name: 'Input types that have version added as false and not deprecated should fail.', - reports: [{ message: 'input type color is not supported by ie.', position: { column: 9, line: 5 }}], + reports: [{ message: 'input type color is not supported by ie.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('input-color') } ]; @@ -179,7 +179,7 @@ hintRunner.testHint(hintPath, inputTypeVersionAddedFalse, { browserslist: ['ie 9 const inputTypeVersionAddedAfterTargetedBrowsers: HintTest[] = [ { name: 'Input types added in a version after the targeted browsers should fail.', - reports: [{ message: 'input type color is not supported by chrome 19, firefox 28.', position: { column: 9, line: 5 }}], + reports: [{ message: 'input type color is not supported by chrome 19, firefox 28.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('input-color') } ]; @@ -193,7 +193,7 @@ hintRunner.testHint(hintPath, inputTypeVersionAddedAfterTargetedBrowsers, { brow const mixedFeaturedCompatibility: HintTest[] = [ { name: 'Features with mixed compatibility (not supported for specific version and never supported) and not deprecated should throw errors for browsers in which the feature is not supported.', - reports: [{ message: 'integrity attribute of the link element is not supported by edge, ie, safari, safari_ios, samsunginternet_android 4, webview_android 4.', position: { column: 9, line: 5 }}], + reports: [{ message: 'integrity attribute of the link element is not supported by edge, ie, safari, safari_ios, samsunginternet_android 4, webview_android 4.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('link-integrity') } ]; diff --git a/packages/hint-compat-api/tests/html.ts b/packages/hint-compat-api/tests/html.ts index e6cb0c5f1f4..7e3adc0f515 100644 --- a/packages/hint-compat-api/tests/html.ts +++ b/packages/hint-compat-api/tests/html.ts @@ -50,7 +50,7 @@ hintRunner.testHint(hintPath, removedForFlags, { browserslist: ['firefox 34'] }) const onlySupportedByFlags: HintTest[] = [ { name: 'Elements only supported by flags should fail.', - reports: [{ message: 'shadow element is not supported by firefox 60.', position: { column: 9, line: 5 }}], + reports: [{ message: 'shadow element is not supported by firefox 60.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('shadow') } ]; @@ -79,7 +79,7 @@ hintRunner.testHint(hintPath, elementRemovedVersionOfTargetedBrowser, { browsers const elementRemovedVersionEarlierThanMultipleTargetedBrowser: HintTest[] = [ { name: 'Elements that were removed in a version before the targeted browser should fail.', - reports: [{ message: 'blink element is not supported by firefox 24-26.'}], + reports: [{ message: 'blink element is not supported by firefox 24-26.' }], serverConfig: generateHTMLConfig('blink') } ]; @@ -99,7 +99,7 @@ hintRunner.testHint(hintPath, elementRemovedVersionEarlierThanTargetedBrowser, { const elementVersionAddedFalse: HintTest[] = [ { name: 'Elements that have version added as false should fail.', - reports: [{ message: 'blink element is not supported by chrome.', position: { column: 9, line: 5 }}], + reports: [{ message: 'blink element is not supported by chrome.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('blink') } ]; @@ -109,7 +109,7 @@ hintRunner.testHint(hintPath, elementVersionAddedFalse, { browserslist: ['last 2 const featureVersionAddedFalseForAllTargetedBrowsers: HintTest[] = [ { name: 'Features with no support (version added is false) for multiple targeted browsers should fail.', - reports: [{ message: 'element element is not supported by any of your target browsers.', position: { column: 9, line: 5 }}], + reports: [{ message: 'element element is not supported by any of your target browsers.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('element') } ]; @@ -119,7 +119,7 @@ hintRunner.testHint(hintPath, featureVersionAddedFalseForAllTargetedBrowsers, { const elementVersionAddedFalseForMultipleBrowsers: HintTest[] = [ { name: 'Elements that have version added as false for multiple browsers should fail with one error.', - reports: [{ message: 'blink element is not supported by chrome, edge, ie.', position: { column: 9, line: 5 }}], + reports: [{ message: 'blink element is not supported by chrome, edge, ie.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('blink') } ]; @@ -129,7 +129,7 @@ hintRunner.testHint(hintPath, elementVersionAddedFalseForMultipleBrowsers, { bro const featureVersionAddedMixedFalseAndNullForDifferentBrowsers: HintTest[] = [ { name: 'Features with unknown support (version added is null) and no support (version added is false) for different browsers should fail for unsupported browsers.', - reports: [{ message: 'element element is not supported by edge, firefox_android.', position: { column: 9, line: 5 }}], + reports: [{ message: 'element element is not supported by edge, firefox_android.', position: { column: 9, line: 4 } }], serverConfig: generateHTMLConfig('element') } ]; @@ -148,7 +148,7 @@ hintRunner.testHint(hintPath, elementAttrRemovedVersionLaterThanTargetedBrowser, const elementAttrRemovedVersionOfTargetedBrowser: HintTest[] = [ { name: 'Element attributes that were removed the version of the targeted browser should fail.', - reports: [{ message: 'scoped attribute of the style element is not supported by firefox 55.'}], + reports: [{ message: 'scoped attribute of the style element is not supported by firefox 55.' }], serverConfig: generateHTMLConfig('style-scoped') } ]; @@ -158,7 +158,7 @@ hintRunner.testHint(hintPath, elementAttrRemovedVersionOfTargetedBrowser, { brow const elementAttrRemovedVersionEarlierThanTargetedBrowser: HintTest[] = [ { name: 'Element attributes that were removed in a version before the targeted browser should fail.', - reports: [{ message: 'scoped attribute of the style element is not supported by firefox 56.'}], + reports: [{ message: 'scoped attribute of the style element is not supported by firefox 56.' }], serverConfig: generateHTMLConfig('style-scoped') } ]; diff --git a/packages/hint-no-protocol-relative-urls/tests/tests.ts b/packages/hint-no-protocol-relative-urls/tests/tests.ts index 6847d14e01c..c1dd154a6a9 100644 --- a/packages/hint-no-protocol-relative-urls/tests/tests.ts +++ b/packages/hint-no-protocol-relative-urls/tests/tests.ts @@ -26,7 +26,7 @@ const tests: HintTest[] = [ name: `'link' with initial // fails the hint`, reports: [{ message: generateErrorMessage('//site.webmanifest'), - position: { column: 37, line: 2 } + position: { column: 37, line: 1 } }], serverConfig: generateHTMLPage('') }, @@ -46,7 +46,7 @@ const tests: HintTest[] = [ name: `'script' with initial // fails the hint`, reports: [{ message: generateErrorMessage('//script.js'), - position: { column: 23, line: 5 } + position: { column: 23, line: 4 } }], serverConfig: generateHTMLPage(undefined, '') }, @@ -66,7 +66,7 @@ const tests: HintTest[] = [ name: `'a' with initial // fails the hint`, reports: [{ message: generateErrorMessage('//home'), - position: { column: 19, line: 5 } + position: { column: 19, line: 4 } }], serverConfig: generateHTMLPage(undefined, 'home') }, @@ -76,4 +76,4 @@ const tests: HintTest[] = [ } ]; -hintRunner.testHint(getHintPath(__filename), tests); +hintRunner.testHint(getHintPath(__filename), tests, { parsers: ['html'] }); From 9d40fa679099bb70ed375748dc39a279648062f3 Mon Sep 17 00:00:00 2001 From: Jesus David Garcia Gomez Date: Fri, 15 Feb 2019 13:49:23 -0800 Subject: [PATCH 7/7] Fix: Update browser extension to use a copy of the DOM Fix #1880 --- .../src/content-script/connector.ts | 27 ++++--------------- packages/extension-browser/webpack.config.js | 7 ++++- yarn.lock | 10 ------- 3 files changed, 11 insertions(+), 33 deletions(-) diff --git a/packages/extension-browser/src/content-script/connector.ts b/packages/extension-browser/src/content-script/connector.ts index a62b7e0c569..9a047685ff1 100644 --- a/packages/extension-browser/src/content-script/connector.ts +++ b/packages/extension-browser/src/content-script/connector.ts @@ -11,6 +11,9 @@ import { FetchEnd, NetworkData } from 'hint/dist/src/lib/types'; +import { JSDOMAsyncWindow } from 'hint/dist/src/lib/types/jsdom-async-html'; +import createAsyncWindow from 'hint/dist/src/lib/utils/dom/create-async-window'; +import traverse from 'hint/dist/src/lib/utils/dom/traverse'; import { Events } from '../shared/types'; import { AsyncWindow, AsyncHTMLDocument, AsyncHTMLElement } from './web-async-html'; @@ -47,12 +50,9 @@ export default class WebExtensionConnector implements IConnector { await this._engine.emitAsync('can-evaluate::script', { resource }); setTimeout(async () => { + const asyncWindow = createAsyncWindow(document.documentElement.outerHTML); - if (this._window && document.documentElement) { - await this._engine.emitAsync('traverse::start', { resource }); - await this.traverseAndNotify(document.documentElement, this._window.document); - await this._engine.emitAsync('traverse::end', { resource }); - } + await traverse((asyncWindow as JSDOMAsyncWindow).dom!, this._engine, resource); this._onComplete(resource); }, this._options.waitFor); @@ -69,23 +69,6 @@ export default class WebExtensionConnector implements IConnector { browser.runtime.sendMessage(message); } - /** Traverses the DOM while sending `element::*` events. */ - private async traverseAndNotify(node: Element, doc: IAsyncHTMLDocument): Promise { - const element = new AsyncHTMLElement(node, doc); - const name = node.tagName.toLowerCase(); - const resource = location.href; - - await this._engine.emitAsync(`element::${name}` as 'element::*', { element, resource }); - await this._engine.emitAsync(`traverse::down`, { element, resource }); - - // Recursively traverse child elements. - for (let i = 0; i < node.children.length; i++) { - await this.traverseAndNotify(node.children[i], doc); - } - - await this._engine.emitAsync(`traverse::up`, { element, resource }); - } - private setFetchElement(event: FetchEnd) { const url = event.request.url; const elements = Array.from(document.querySelectorAll('[href],[src]')).filter((element: any) => { diff --git a/packages/extension-browser/webpack.config.js b/packages/extension-browser/webpack.config.js index bd0c8768029..8a9b7781c2b 100644 --- a/packages/extension-browser/webpack.config.js +++ b/packages/extension-browser/webpack.config.js @@ -45,7 +45,12 @@ const baseConfig = { } ] }, - node: { fs: 'empty' }, + node: { + child_process: 'empty', // eslint-disable-line camelcase + fs: 'empty', + net: 'mock', + tls: 'mock' + }, output: { filename: '[name].js', path: path.resolve(__dirname, 'dist/bundle') diff --git a/yarn.lock b/yarn.lock index 41c3c93189e..1589aef96dd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -460,11 +460,6 @@ resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.1.tgz#0be64e2fbe7c0bb2a6cad6a152462127a1b0d45a" integrity sha512-OKKcc+Tt1rLMg1DTnmgvWHRna3SN51GWs/ERxpgjFNRjCUUVhGG8FEKeFqVTzBv1jEGFqbkJRWfsHV6KAcLT+A== -"@types/debug@^4.1.0": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.1.tgz#0be64e2fbe7c0bb2a6cad6a152462127a1b0d45a" - integrity sha512-OKKcc+Tt1rLMg1DTnmgvWHRna3SN51GWs/ERxpgjFNRjCUUVhGG8FEKeFqVTzBv1jEGFqbkJRWfsHV6KAcLT+A== - "@types/ejs@^2.6.2": version "2.6.2" resolved "https://registry.yarnpkg.com/@types/ejs/-/ejs-2.6.2.tgz#d59bd4b12114a62d203ed137dbee5bebc2333f93" @@ -6117,11 +6112,6 @@ lolex@^3.0.0: version "4.0.1" resolved "git://github.com/dcodeIO/long.js.git#8181a6b50a2a230f0b2a1e4c4093f9b9d19c8b69" -"long@git://github.com/dcodeIO/long.js.git#8181a6b50a2a230f0b2a1e4c4093f9b9d19c8b69": - version "4.0.1" - uid "8181a6b50a2a230f0b2a1e4c4093f9b9d19c8b69" - resolved "git://github.com/dcodeIO/long.js.git#8181a6b50a2a230f0b2a1e4c4093f9b9d19c8b69" - loud-rejection@^1.0.0, loud-rejection@^1.2.0: version "1.6.0" resolved "https://registry.yarnpkg.com/loud-rejection/-/loud-rejection-1.6.0.tgz#5b46f80147edee578870f086d04821cf998e551f"