From 9fa11bc523bbfed3f77f5dd1b51c0f7a4aaf8ac2 Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Sat, 22 Jun 2024 10:30:19 -0700 Subject: [PATCH 1/3] Proposal: Moving hooks on thread --- doc/design/proposal-moving-hooks-on-thread.md | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 doc/design/proposal-moving-hooks-on-thread.md diff --git a/doc/design/proposal-moving-hooks-on-thread.md b/doc/design/proposal-moving-hooks-on-thread.md new file mode 100644 index 0000000..724c62f --- /dev/null +++ b/doc/design/proposal-moving-hooks-on-thread.md @@ -0,0 +1,155 @@ +# Moving Hooks on Thread + +This is a proposal for how to move the existing hooks API back from a separate thread onto the same thread (main or worker) as the user’s application code, while preserving the ability for the hooks API to customize both CommonJS and ES modules. + +## Motivation + +The current off-thread design is proving very difficult to complete and maintain, and will not satisfy an important goal of sunsetting monkey-patching. The primary motivation for the off-thread design was so that users could write asynchronous customization hooks, but this seems to be a benefit that few users need; and the subset of users who need this ability can achieve it on their own, without such support needing to be built into the overall hooks API. Also, there are things that cannot be achieved without being on-thread, such as modifying exports from modules, that are important for instrumentation. + +## Goals + +1. Preserve the design of the current `resolve` and `load` hooks as much as possible, to ease migration for existing users of the off-thread hooks API. + +1. Support most or all of the use cases currently covered by the off-thread hooks. + +1. Complement @joyeecheung’s [Universal, synchronous and in-thread loader hooks proposal](https://github.com/nodejs/loaders/pull/198). We want one set of hooks for both CommonJS and ES modules, and that single set of hooks should both allow for both sunsetting monkey-patching and sunsetting the off-thread hooks. + +## Design + +This proposal builds on the [Universal, synchronous and in-thread loader hooks proposal](https://github.com/nodejs/loaders/pull/198). First we would implement the new `registerHooks` API from that proposal, which creates sync `resolve`, `load` and other hooks. That should cover that proposal’s primary goal of providing new APIs for monkey-patchers to migrate to, allowing us to sunset the need to support monkey-patching the CommonJS loader. + +Once that is achieved, there is one large gap left to fill before we can remove the off-thread hooks: provide a way to perform async work, now that the hooks are all sync. + +### `startGraph` hook (module graph entry point only) + +We would create a new _async_ hook that runs for the main entry point, any worker entry point, and any dynamic `import()`: in other words, before any new module graph is created. This would be an opportunity to do async work that the later sync hooks could have access to, for example to make network calls in a non-blocking way. One of the use cases for this hook is to support network imports, such as the [Import from HTTPS example](https://nodejs.org/api/module.html#import-from-https) in our docs. See [`nodejs/node` #43245](https://github.com/nodejs/node/pull/43245). + +Because this hook might need to resolve and load the entire module graph to do its work, such as to resolve and fetch HTTPS URLs, it will be provided with the `resolve` and `load` hooks in its `context`. It would also receive `context.entry` to distinguish main/worker entry points from dynamic `import()`. Its signature would be something like: + +```js +/** + * @typedef {{ + * parentURL?: string, + * entry: boolean, + * conditions?: string[], + * importAttributes?: Record, + * resolve: Function, + * load: Function, + * }} ModuleStartGraphContext + */ +/** + * @typedef {Promise<{ + * shortCircuit?: boolean + * }>} ModuleStartGraphResult + */ +/** + * @param {string} specifier + * @param {ModuleStartGraphContext} context + * @param {(specifier: string, context: ModuleStartGraphContext) => ModuleStartGraphResult} nextStartGraph + * @returns {ModuleStartGraphResult} + */ +async function startGraph(specifier, context, nextStartGraph) {} +``` + +## Examples + +### Import from HTTPS + +the [Import from HTTPS example](https://nodejs.org/api/module.html#import-from-https) in our docs could use `startGraph` to handle the async work: + +```js +import { parseAllImportSpecifiersFromSource } from 'some-library'; + +/** @type {Map 0) { + const specifier = specifiers.pop(); + const { url } = resolve(specifier, context); + if (httpImportsCache.has(url)) { + continue; + } + if (url.startsWith('https://')) { + const response = await fetch(url); + const responseSource = await response.text(); + httpImportsCache.set(url, responseSource); + const { source } = load(url, context); + httpImportsCache.set(url, source); + specifiers.push(...parseAllImportSpecifiersFromSource(source)); + } + } + return nextStartGraph(specifier, context); +} + +export function resolve(specifier, context, defaultResolve) { + if (specifier.startsWith('https://')) { + return { + url: specifier, + shortCircuit: true, + }; + } + return defaultResolve(specifier, context); +} + +export function load(url, context, defaultLoad) { + const cached = httpImportsCache.get(url); + if (cached) { + return { + source: cached, + format: 'module', // This example assumes all fetched sources are modules for simplicity + // No shortCircuit so that the load chain possibly transforms the source + } + } + return defaultLoad(url, context); +} +``` + + +### Transpilation + +The [transpilation example from our docs](https://nodejs.org/api/module.html#transpilation) needs only to be changed so that async calls are now synchronous: + +```js +// coffeescript-hooks.mjs +import { readFileSync } from 'node:fs'; +import { dirname, extname, resolve as resolvePath } from 'node:path'; +import { cwd } from 'node:process'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import coffeescript from 'coffeescript'; + +const extensionsRegex = /\.(coffee|litcoffee|coffee\.md)$/; + +export function load(url, context, nextLoad) { + if (extensionsRegex.test(url)) { + const format = getPackageType(url); + + const { source: rawSource } = nextLoad(url, { ...context, format }); + const transformedSource = coffeescript.compile(rawSource.toString(), url); + + return { + format, + shortCircuit: true, + source: transformedSource, + }; + } + + return nextLoad(url); +} + +function getPackageType(url) { + const isFilePath = !!extname(url); + const dir = isFilePath ? dirname(fileURLToPath(url)) : url; + const packagePath = resolvePath(dir, 'package.json'); + try { + const fileContents = readFileSync(packagePath, { encoding: 'utf8' }) + const { type } = JSON.parse(fileContents); + if (type) { return type }; + } catch {} + return dir.length > 1 && getPackageType(resolvePath(dir, '..')); +} +``` + +This could be used via `node --import 'data:text/javascript,import { registerHooks } from "node:module"; import { load } from "./coffeescript-hooks.mjs"; registerHooks({ load });' ./main.coffee`. From 4d10f8cbcb45713274b851a7cf09dee04257de7d Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Sun, 23 Jun 2024 15:02:58 -0700 Subject: [PATCH 2/3] Improve HTTPS example Co-authored-by: Guy Bedford --- doc/design/proposal-moving-hooks-on-thread.md | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/doc/design/proposal-moving-hooks-on-thread.md b/doc/design/proposal-moving-hooks-on-thread.md index 724c62f..35ca9f7 100644 --- a/doc/design/proposal-moving-hooks-on-thread.md +++ b/doc/design/proposal-moving-hooks-on-thread.md @@ -65,20 +65,25 @@ const httpImportsCache = new Map(); export async function startGraph(specifier, context, nextStartGraph) { const { resolve, load } = context; - const specifiers = [specifier]; - while (specifiers.length > 0) { - const specifier = specifiers.pop(); - const { url } = resolve(specifier, context); - if (httpImportsCache.has(url)) { - continue; - } + const { url } = resolve(specifier, context); + const urls = [url]; + while (urls.length > 0) { + const url = urls.shift(); if (url.startsWith('https://')) { const response = await fetch(url); const responseSource = await response.text(); httpImportsCache.set(url, responseSource); - const { source } = load(url, context); - httpImportsCache.set(url, source); - specifiers.push(...parseAllImportSpecifiersFromSource(source)); + } + // load API constructs context automatically from registry data + const { source } = load(url); + httpImportsCache.set(url, source); + for (const specifier of parseAllImportSpecifiersFromSource(source)) { + // resolve API also constructs context automatically from registry data + // including determining if the parent is CJS or ESM and setting conditions + // correctly. Even when conditions is provided, although explicit import + // or require conditions can override this process. + const depUrl = resolve(specifier, { parentURL: url }); + urls.push(depUrl); } } return nextStartGraph(specifier, context); From 842b7c7f1579fa673c3041049dc90b1590ef149e Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Sun, 23 Jun 2024 15:03:22 -0700 Subject: [PATCH 3/3] Add note that hook delays evaluation Co-authored-by: Guy Bedford --- doc/design/proposal-moving-hooks-on-thread.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/proposal-moving-hooks-on-thread.md b/doc/design/proposal-moving-hooks-on-thread.md index 35ca9f7..8b1eacc 100644 --- a/doc/design/proposal-moving-hooks-on-thread.md +++ b/doc/design/proposal-moving-hooks-on-thread.md @@ -22,7 +22,7 @@ Once that is achieved, there is one large gap left to fill before we can remove ### `startGraph` hook (module graph entry point only) -We would create a new _async_ hook that runs for the main entry point, any worker entry point, and any dynamic `import()`: in other words, before any new module graph is created. This would be an opportunity to do async work that the later sync hooks could have access to, for example to make network calls in a non-blocking way. One of the use cases for this hook is to support network imports, such as the [Import from HTTPS example](https://nodejs.org/api/module.html#import-from-https) in our docs. See [`nodejs/node` #43245](https://github.com/nodejs/node/pull/43245). +We would create a new _async_ hook that runs for the main entry point, any worker entry point, and any dynamic `import()`: in other words, before any new module graph is created. This hook would delay any further hooks for that module graph operation, until the `startGraph` hook promise has resolved successfully. This provides an opportunity to do async work that the later sync hooks could have access to, for example to make network calls in a non-blocking way. One of the use cases for this hook is to support network imports, such as the [Import from HTTPS example](https://nodejs.org/api/module.html#import-from-https) in our docs. See [`nodejs/node` #43245](https://github.com/nodejs/node/pull/43245). Because this hook might need to resolve and load the entire module graph to do its work, such as to resolve and fetch HTTPS URLs, it will be provided with the `resolve` and `load` hooks in its `context`. It would also receive `context.entry` to distinguish main/worker entry points from dynamic `import()`. Its signature would be something like: