diff --git a/package.json b/package.json index 4c02840be264..5747d7770d95 100644 --- a/package.json +++ b/package.json @@ -66,12 +66,14 @@ "@types/react-dom@18>@types/react": "^18", "@types/react-tabs>@types/react": "^18", "@types/react-transition-group>@types/react": "^18", - "@cloudflare/elements>@types/react": "^18" + "@cloudflare/elements>@types/react": "^18", + "capnpc-ts>typescript": "4.2.4" }, "patchedDependencies": { "ink@3.2.0": "patches/ink@3.2.0.patch", "toucan-js@3.2.2": "patches/toucan-js@3.2.2.patch", - "@cloudflare/component-listbox@1.10.6": "patches/@cloudflare__component-listbox@1.10.6.patch" + "@cloudflare/component-listbox@1.10.6": "patches/@cloudflare__component-listbox@1.10.6.patch", + "capnp-ts@0.7.0": "patches/capnp-ts@0.7.0.patch" } } } diff --git a/packages/miniflare/.gitignore b/packages/miniflare/.gitignore new file mode 100644 index 000000000000..2c29241f43df --- /dev/null +++ b/packages/miniflare/.gitignore @@ -0,0 +1,3 @@ +.mf +.tmp +*.metafile.json diff --git a/packages/miniflare/ava.config.mjs b/packages/miniflare/ava.config.mjs new file mode 100644 index 000000000000..d9f838e12972 --- /dev/null +++ b/packages/miniflare/ava.config.mjs @@ -0,0 +1,15 @@ +export default { + files: ["test/**/*.spec.ts"], + nodeArguments: ["--no-warnings", "--experimental-vm-modules"], + require: ["./test/setup.mjs"], + workerThreads: false, + typescript: { + compile: false, + rewritePaths: { + "test/": "dist/test/", + }, + }, + environmentVariables: { + MINIFLARE_ASSERT_BODIES_CONSUMED: "true", + }, +}; diff --git a/packages/miniflare/package.json b/packages/miniflare/package.json index 65e8a006b1ea..532492bc549a 100644 --- a/packages/miniflare/package.json +++ b/packages/miniflare/package.json @@ -9,14 +9,14 @@ "local", "cloudworker" ], - "homepage": "https://github.com/cloudflare/miniflare/tree/master/packages/tre#readme", + "homepage": "https://github.com/cloudflare/workers-sdk/tree/main/packages/miniflare#readme", "bugs": { - "url": "https://github.com/cloudflare/miniflare/issues" + "url": "https://github.com/cloudflare/workers-sdk/issues" }, "repository": { "type": "git", - "url": "git+https://github.com/cloudflare/miniflare.git", - "directory": "packages/tre" + "url": "https://github.com/cloudflare/workers-sdk.git", + "directory": "packages/miniflare" }, "license": "MIT", "author": "MrBBot ", @@ -26,6 +26,17 @@ "dist/src", "bootstrap.js" ], + "scripts": { + "build": "node scripts/build.mjs && pnpm run types:build", + "capnp:workerd": "capnpc -o ts src/runtime/config/workerd.capnp", + "clean": "rimraf ./dist ./dist-types", + "dev": "concurrently -n esbuild,typechk,typewrk -c yellow,blue,blue.dim \"node scripts/build.mjs watch\" \"node scripts/types.mjs tsconfig.json watch\" \"node scripts/types.mjs src/workers/tsconfig.json watch\"", + "test": "node scripts/build.mjs && ava && rimraf ./.tmp", + "test:ci": "pnpm run test", + "check:lint": "eslint \"{src,test}/**/*.ts\" \"scripts/**/*.{js,mjs}\" \"types/**/*.ts\"", + "lint:fix": "pnpm run lint -- --fix", + "types:build": "node scripts/types.mjs tsconfig.json && node scripts/types.mjs src/workers/tsconfig.json" + }, "dependencies": { "acorn": "^8.8.0", "acorn-walk": "^8.2.0", @@ -41,19 +52,31 @@ "zod": "^3.20.6" }, "devDependencies": { + "@ava/typescript": "^4.0.0", "@cloudflare/kv-asset-handler": "^0.3.0", "@cloudflare/workers-types": "^4.20231002.0", + "@microsoft/api-extractor": "^7.36.3", "@types/debug": "^4.1.7", "@types/estree": "^1.0.0", "@types/glob-to-regexp": "^0.4.1", "@types/http-cache-semantics": "^4.0.1", + "@types/node": "^18.11.9", + "@types/rimraf": "^3.0.2", "@types/source-map-support": "^0.5.6", "@types/stoppable": "^1.1.1", + "@types/which": "^2.0.1", "@types/ws": "^8.5.3", + "ava": "^5.2.0", + "capnpc-ts": "^0.7.0", "devalue": "^4.3.0", "devtools-protocol": "^0.0.1182435", + "esbuild": "^0.16.17", + "expect-type": "^0.15.0", "http-cache-semantics": "^4.1.0", - "kleur": "^4.1.5" + "kleur": "^4.1.5", + "rimraf": "^3.0.2", + "source-map": "^0.6.0", + "which": "^2.0.2" }, "engines": { "node": ">=16.13" diff --git a/packages/miniflare/scripts/build.mjs b/packages/miniflare/scripts/build.mjs new file mode 100644 index 000000000000..9004ce38248f --- /dev/null +++ b/packages/miniflare/scripts/build.mjs @@ -0,0 +1,197 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import esbuild from "esbuild"; +import { getPackage, pkgRoot } from "./common.mjs"; + +const argv = process.argv.slice(2); +const watch = argv[0] === "watch"; + +/** + * Recursively walks a directory, returning a list of all files contained within + * @param {string} rootPath + * @returns {Promise} + */ +async function walk(rootPath) { + const fileNames = await fs.readdir(rootPath); + const walkPromises = fileNames.map(async (fileName) => { + const filePath = path.join(rootPath, fileName); + return (await fs.stat(filePath)).isDirectory() + ? await walk(filePath) + : [filePath]; + }); + return (await Promise.all(walkPromises)).flat(); +} + +/** + * Gets a list of dependency names from the passed package + * @param {~Package} pkg + * @param {boolean} [includeDev] + * @returns {string[]} + */ +function getPackageDependencies(pkg, includeDev) { + return [ + ...(pkg.dependencies ? Object.keys(pkg.dependencies) : []), + ...(includeDev && pkg.devDependencies + ? Object.keys(pkg.devDependencies) + : []), + ...(pkg.peerDependencies ? Object.keys(pkg.peerDependencies) : []), + ...(pkg.optionalDependencies ? Object.keys(pkg.optionalDependencies) : []), + ]; +} + +const workersRoot = path.join(pkgRoot, "src", "workers"); + +const miniflareSharedExtensionPath = path.join( + workersRoot, + "shared", + "index.worker.ts" +); +const miniflareZodExtensionPath = path.join( + workersRoot, + "shared", + "zod.worker.ts" +); +/** + * `workerd` `extensions` don't have access to "built-in" modules like + * `node:buffer`, but do have access to "internal" modules like + * `node-internal:internal_buffer`, which usually provide the same exports. + * So that we can use `node:assert` and `node:buffer` in our shared extension, + * rewrite built-in names to internal. + * @type {esbuild.Plugin} + */ +const rewriteNodeToInternalPlugin = { + name: "rewrite-node-to-internal", + setup(build) { + build.onResolve({ filter: /^node:(assert|buffer)$/ }, async (args) => { + const module = args.path.substring("node:".length); + return { path: `node-internal:internal_${module}`, external: true }; + }); + }, +}; + +/** + * @type {Map} + */ +const workersBuilders = new Map(); +/** + * @type {esbuild.Plugin} + */ +const embedWorkersPlugin = { + name: "embed-workers", + setup(build) { + const namespace = "embed-worker"; + build.onResolve({ filter: /^worker:/ }, async (args) => { + let name = args.path.substring("worker:".length); + // Allow `.worker` to be omitted + if (!name.endsWith(".worker")) name += ".worker"; + // Use `build.resolve()` API so Workers can be written as `m?[jt]s` files + const result = await build.resolve("./" + name, { + kind: "import-statement", + resolveDir: workersRoot, + }); + if (result.errors.length > 0) return { errors: result.errors }; + return { path: result.path, namespace }; + }); + build.onLoad({ filter: /.*/, namespace }, async (args) => { + let builder = workersBuilders.get(args.path); + if (builder === undefined) { + builder = await esbuild.build({ + platform: "node", // Marks `node:*` imports as external + format: "esm", + target: "esnext", + bundle: true, + sourcemap: true, + sourcesContent: false, + external: ["miniflare:shared", "miniflare:zod"], + metafile: true, + incremental: watch, // Allow `rebuild()` calls if watching + entryPoints: [args.path], + minifySyntax: true, + outdir: build.initialOptions.outdir, + outbase: pkgRoot, + plugins: + args.path === miniflareSharedExtensionPath || + args.path === miniflareZodExtensionPath + ? [rewriteNodeToInternalPlugin] + : [], + }); + } else { + builder = await builder.rebuild(); + } + workersBuilders.set(args.path, builder); + await fs.mkdir("worker-metafiles", { recursive: true }); + await fs.writeFile( + path.join( + "worker-metafiles", + path.basename(args.path) + ".metafile.json" + ), + JSON.stringify(builder.metafile) + ); + let outPath = args.path.substring(workersRoot.length + 1); + outPath = outPath.substring(0, outPath.lastIndexOf(".")) + ".js"; + outPath = JSON.stringify(outPath); + const watchFiles = Object.keys(builder.metafile.inputs); + const contents = ` + import fs from "fs"; + import path from "path"; + import url from "url"; + let contents; + export default function() { + if (contents !== undefined) return contents; + const filePath = path.join(__dirname, "workers", ${outPath}); + contents = fs.readFileSync(filePath, "utf8") + "//# sourceURL=" + url.pathToFileURL(filePath); + return contents; + } + `; + return { contents, loader: "js", watchFiles }; + }); + }, +}; + +async function buildPackage() { + const pkg = await getPackage(pkgRoot); + + const indexPath = path.join(pkgRoot, "src", "index.ts"); + // Look for test files ending with .spec.ts in the test directory, default to + // empty array if not found + let testPaths = []; + try { + testPaths = (await walk(path.join(pkgRoot, "test"))).filter((testPath) => + testPath.endsWith(".spec.ts") + ); + } catch (e) { + if (e.code !== "ENOENT") throw e; + } + const outPath = path.join(pkgRoot, "dist"); + + await esbuild.build({ + platform: "node", + format: "cjs", + target: "esnext", + bundle: true, + sourcemap: true, + sourcesContent: false, + tsconfig: path.join(pkgRoot, "tsconfig.json"), + // Mark root package's dependencies as external, include root devDependencies + // (e.g. test runner) as we don't want these bundled + external: [ + // Make sure we're not bundling any packages we're building, we want to + // test against the actual code we'll publish for instance + "miniflare", + // Mark `dependencies` as external, but not `devDependencies` (we use them + // to signal single-use/small packages we want inlined in the bundle) + ...getPackageDependencies(pkg), + // Mark test dependencies as external + "ava", + "esbuild", + ], + plugins: [embedWorkersPlugin], + logLevel: watch ? "info" : "warning", + watch, + outdir: outPath, + outbase: pkgRoot, + entryPoints: [indexPath, ...testPaths], + }); +} + +await buildPackage(); diff --git a/packages/miniflare/scripts/common.mjs b/packages/miniflare/scripts/common.mjs new file mode 100644 index 000000000000..81e1af46c201 --- /dev/null +++ b/packages/miniflare/scripts/common.mjs @@ -0,0 +1,29 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +export const pkgRoot = path.resolve(__dirname, ".."); + +/** + * @typedef {object} ~Package + * @property {string} name + * @property {string} version + * @property {Record} [dependencies] + * @property {Record} [devDependencies] + * @property {Record} [peerDependencies] + * @property {Record} [optionalDependencies] + * @property {string[]} [entryPoints] + */ + +/** + * Gets the contents of the package.json file in + * @param {string} pkgRoot + * @returns {Promise<~Package>} + */ +export async function getPackage(pkgRoot) { + return JSON.parse( + await fs.readFile(path.join(pkgRoot, "package.json"), "utf8") + ); +} diff --git a/packages/miniflare/test/plugins/core/errors/index.spec.ts b/packages/miniflare/test/plugins/core/errors/index.spec.ts index d76d6931a09e..24f718fc1161 100644 --- a/packages/miniflare/test/plugins/core/errors/index.spec.ts +++ b/packages/miniflare/test/plugins/core/errors/index.spec.ts @@ -6,7 +6,7 @@ import test from "ava"; import Protocol from "devtools-protocol"; import esbuild from "esbuild"; import { DeferredPromise, Miniflare } from "miniflare"; -import { RawSourceMap } from "source-map"; +import type { RawSourceMap } from "source-map"; import NodeWebSocket from "ws"; import { escapeRegexp, useTmp } from "../../../test-shared"; @@ -136,7 +136,7 @@ addEventListener("fetch", (event) => { message: "unnamed", }); const serviceWorkerEntryRegexp = escapeRegexp( - `${SERVICE_WORKER_ENTRY_PATH}:6:17` + `${SERVICE_WORKER_ENTRY_PATH}:6:16` ); t.regex(String(error?.stack), serviceWorkerEntryRegexp); error = await t.throwsAsync(mf.dispatchFetch("http://localhost/a"), { @@ -148,7 +148,7 @@ addEventListener("fetch", (event) => { error = await t.throwsAsync(mf.dispatchFetch("http://localhost/b"), { message: "b", }); - const modulesEntryRegexp = escapeRegexp(`${MODULES_ENTRY_PATH}:5:19`); + const modulesEntryRegexp = escapeRegexp(`${MODULES_ENTRY_PATH}:5:17`); t.regex(String(error?.stack), modulesEntryRegexp); error = await t.throwsAsync(mf.dispatchFetch("http://localhost/c"), { message: "c", @@ -174,7 +174,7 @@ addEventListener("fetch", (event) => { instanceOf: TypeError, message: "Dependency error", }); - const nestedRegexp = escapeRegexp(`${DEP_ENTRY_PATH}:4:17`); + const nestedRegexp = escapeRegexp(`${DEP_ENTRY_PATH}:4:16`); t.regex(String(error?.stack), nestedRegexp); // Check source mapping URLs rewritten diff --git a/packages/miniflare/test/plugins/core/index.spec.ts b/packages/miniflare/test/plugins/core/index.spec.ts index 7ff1016eb536..afcaf26ad448 100644 --- a/packages/miniflare/test/plugins/core/index.spec.ts +++ b/packages/miniflare/test/plugins/core/index.spec.ts @@ -52,12 +52,13 @@ opensslTest("NODE_EXTRA_CA_CERTS: loads certificates", async (t) => { // Start Miniflare with NODE_EXTRA_CA_CERTS environment variable // (cannot use sync process methods here as that would block HTTPS server) + const miniflarePath = require.resolve("miniflare"); const result = childProcess.spawn( process.execPath, [ "-e", ` - const { Miniflare } = require("miniflare"); + const { Miniflare } = require(${JSON.stringify(miniflarePath)}); const mf = new Miniflare({ verbose: true, modules: true, diff --git a/packages/miniflare/test/plugins/core/modules.spec.ts b/packages/miniflare/test/plugins/core/modules.spec.ts index 8544486edb2b..a1f0ed7a6fe1 100644 --- a/packages/miniflare/test/plugins/core/modules.spec.ts +++ b/packages/miniflare/test/plugins/core/modules.spec.ts @@ -241,7 +241,7 @@ You must manually define your modules when constructing Miniflare: ... ] }) - at ${scriptPath}:14:17` + at ${scriptPath}:14:15` ); // Check with dynamic require diff --git a/packages/miniflare/test/setup.mjs b/packages/miniflare/test/setup.mjs index 5094c300c130..766ec2519e8f 100644 --- a/packages/miniflare/test/setup.mjs +++ b/packages/miniflare/test/setup.mjs @@ -1,4 +1,20 @@ -import { _initialiseInstanceRegistry } from "miniflare"; +import Module from "node:module"; +import { _initialiseInstanceRegistry } from "../dist/src/index.js"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const pkgRoot = path.resolve(__dirname, ".."); + +// Monkeypatch Node's resolver to require built `miniflare` package when we call +// `require("miniflare")`. We could fix this by adding `miniflare` as a +// dev dependency of itself, but Turborepo doesn't allow this. +const originalResolveFilename = Module._resolveFilename; +Module._resolveFilename = function (spec, ...args) { + if (spec === "miniflare") spec = pkgRoot; + return originalResolveFilename.call(this, spec, ...args); +}; const registry = _initialiseInstanceRegistry(); const bigSeparator = "=".repeat(80); diff --git a/packages/miniflare/tsconfig.json b/packages/miniflare/tsconfig.json new file mode 100644 index 000000000000..00e4e088a7c8 --- /dev/null +++ b/packages/miniflare/tsconfig.json @@ -0,0 +1,39 @@ +{ + "compilerOptions": { + "module": "esnext", + "target": "esnext", + "lib": ["esnext"], + "strict": true, + "moduleResolution": "bundler", + "esModuleInterop": true, + "isolatedModules": true, + "experimentalDecorators": true, + "resolveJsonModule": true, + "types": ["node"], + "outDir": "dist-types", + "sourceMap": true, + "declaration": true, + "declarationMap": true, + "noEmit": false, + "emitDeclarationOnly": true, + "rootDir": ".", + "checkJs": true, + "allowJs": true, + "skipLibCheck": false, + "paths": { + "miniflare": ["./src"], + "miniflare:shared": ["./src/workers/shared/index.ts"], + "miniflare:zod": ["./src/workers/shared/zod.worker.ts"] + } + }, + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "types" + ], + "exclude": [ + "src/workers/**/*.worker.ts", + "src/workers/node.d.ts", + "test/fixtures" + ] +} diff --git a/packages/miniflare/turbo.json b/packages/miniflare/turbo.json new file mode 100644 index 000000000000..1a6c74c9def8 --- /dev/null +++ b/packages/miniflare/turbo.json @@ -0,0 +1,9 @@ +{ + "$schema": "http://turbo.build/schema.json", + "extends": ["//"], + "pipeline": { + "build": { + "outputs": ["dist/**"] + } + } +} diff --git a/patches/capnp-ts@0.7.0.patch b/patches/capnp-ts@0.7.0.patch new file mode 100644 index 000000000000..f67623c9fbb9 --- /dev/null +++ b/patches/capnp-ts@0.7.0.patch @@ -0,0 +1,81 @@ +diff --git a/src/serialization/arena/index.d.ts b/src/serialization/arena/index.d.ts +index 86f6210387fc832609209f992a1db951cc488fcc..19d9d1c3a22a0a70ef015485d9f9726a43a8d4c5 100644 +--- a/src/serialization/arena/index.d.ts ++++ b/src/serialization/arena/index.d.ts +@@ -1,7 +1,7 @@ + /** + * @author jdiaz5513 + */ +-export { AnyArena } from "./any-arena"; ++export type { AnyArena } from "./any-arena"; + export { Arena } from "./arena"; + export { ArenaKind } from "./arena-kind"; + export { MultiSegmentArena } from "./multi-segment-arena"; +diff --git a/src/serialization/arena/index.ts b/src/serialization/arena/index.ts +index 5128c6ed5bb0ba309e4e0462a540b0dde957010e..f2407b9d5d05e33250653c239950521c4f8e5f25 100644 +--- a/src/serialization/arena/index.ts ++++ b/src/serialization/arena/index.ts +@@ -2,7 +2,7 @@ + * @author jdiaz5513 + */ + +-export { AnyArena } from "./any-arena"; ++export type { AnyArena } from "./any-arena"; + export { Arena } from "./arena"; + export { ArenaKind } from "./arena-kind"; + export { MultiSegmentArena } from "./multi-segment-arena"; +diff --git a/src/serialization/pointers/index.d.ts b/src/serialization/pointers/index.d.ts +index dc013aa2766afaaa36a46107c27c531b02abd744..2835a013c529344833ef92f597bddbf8d19e711e 100644 +--- a/src/serialization/pointers/index.d.ts ++++ b/src/serialization/pointers/index.d.ts +@@ -14,12 +14,14 @@ export { Int32List } from "./int32-list"; + export { Int64List } from "./int64-list"; + export { Interface } from "./interface"; + export { InterfaceList } from "./interface-list"; +-export { List, ListCtor } from "./list"; ++export { List } from "./list"; ++export type { ListCtor } from "./list"; + export { Orphan } from "./orphan"; + export { PointerList } from "./pointer-list"; + export { PointerType } from "./pointer-type"; + export { Pointer } from "./pointer"; +-export { _StructCtor, Struct, StructCtor } from "./struct"; ++export { Struct } from "./struct"; ++export type { _StructCtor, StructCtor } from "./struct"; + export { Text } from "./text"; + export { TextList } from "./text-list"; + export { Uint8List } from "./uint8-list"; +diff --git a/src/serialization/pointers/index.ts b/src/serialization/pointers/index.ts +index 5a94ada76b013c4d23b2a6edfb966d93bceed6fd..4e2f49d15c3b7ee6a8f551e8867353d311c92424 100644 +--- a/src/serialization/pointers/index.ts ++++ b/src/serialization/pointers/index.ts +@@ -15,12 +15,14 @@ export { Int32List } from "./int32-list"; + export { Int64List } from "./int64-list"; + export { Interface } from "./interface"; + export { InterfaceList } from "./interface-list"; +-export { List, ListCtor } from "./list"; ++export { List } from "./list"; ++export type { ListCtor } from "./list"; + export { Orphan } from "./orphan"; + export { PointerList } from "./pointer-list"; + export { PointerType } from "./pointer-type"; + export { Pointer } from "./pointer"; +-export { _StructCtor, Struct, StructCtor } from "./struct"; ++export { Struct } from "./struct"; ++export type { _StructCtor, StructCtor } from "./struct"; + export { Text } from "./text"; + export { TextList } from "./text-list"; + export { Uint8List } from "./uint8-list"; +diff --git a/src/serialization/pointers/struct.ts b/src/serialization/pointers/struct.ts +index 30b3fc0d2b855e569e1b5d522fe7f76bde82050f..c7464310742cb5ef5bd6ef0df496a099839cb4a9 100644 +--- a/src/serialization/pointers/struct.ts ++++ b/src/serialization/pointers/struct.ts +@@ -107,8 +107,6 @@ export class Struct extends Pointer { + static readonly setText = setText; + static readonly testWhich = testWhich; + +- readonly _capnp!: _Struct; +- + /** + * Create a new pointer to a struct. + * diff --git a/turbo.json b/turbo.json index 8c689cba44ea..6cc24334132e 100644 --- a/turbo.json +++ b/turbo.json @@ -8,6 +8,10 @@ "persistent": true, "cache": false }, + "miniflare#dev": { + "persistent": true, + "cache": false + }, "build": { "dependsOn": ["^build"] }, "test": { "dependsOn": ["^build"],