[migration] Fix Miniflare build and test scripts

package.json

@@ -66,12 +66,14 @@
 			"@types/react-dom@18>@types/react": "^18",
 			"@types/react-tabs>@types/react": "^18",
 			"@types/react-transition-group>@types/react": "^18",
-			"@cloudflare/elements>@types/react": "^18"
+			"@cloudflare/elements>@types/react": "^18",
+			"capnpc-ts>typescript": "4.2.4"
 		},
 		"patchedDependencies": {
 			"[email protected]": "patches/[email protected]",
 			"[email protected]": "patches/[email protected]",
-			"@cloudflare/[email protected]": "patches/@[email protected]"
+			"@cloudflare/[email protected]": "patches/@[email protected]",
+			"[email protected]": "patches/[email protected]"
 		}
 	}
 }

packages/miniflare/.gitignore

@@ -0,0 +1,3 @@
+.mf
+.tmp
+*.metafile.json

packages/miniflare/ava.config.mjs

@@ -0,0 +1,15 @@
+export default {
+  files: ["test/**/*.spec.ts"],
+  nodeArguments: ["--no-warnings", "--experimental-vm-modules"],
+  require: ["./test/setup.mjs"],
+  workerThreads: false,
+  typescript: {
+    compile: false,
+    rewritePaths: {
+      "test/": "dist/test/",
+    },
+  },
+  environmentVariables: {
+    MINIFLARE_ASSERT_BODIES_CONSUMED: "true",
+  },
+};

packages/miniflare/package.json

@@ -9,14 +9,14 @@
 		"local",
 		"cloudworker"
 	],
-	"homepage": "https://github.com/cloudflare/miniflare/tree/master/packages/tre#readme",
+	"homepage": "https://github.com/cloudflare/workers-sdk/tree/main/packages/miniflare#readme",
 	"bugs": {
-		"url": "https://github.com/cloudflare/miniflare/issues"
+		"url": "https://github.com/cloudflare/workers-sdk/issues"
 	},
 	"repository": {
 		"type": "git",
-		"url": "git+https://github.com/cloudflare/miniflare.git",
-		"directory": "packages/tre"
+		"url": "https://github.com/cloudflare/workers-sdk.git",
+		"directory": "packages/miniflare"
 	},
 	"license": "MIT",
 	"author": "MrBBot <[email protected]>",
@@ -26,6 +26,17 @@
 		"dist/src",
 		"bootstrap.js"
 	],
+	"scripts": {
+		"build": "node scripts/build.mjs && pnpm run types:build",
+		"capnp:workerd": "capnpc -o ts src/runtime/config/workerd.capnp",
+		"clean": "rimraf ./dist ./dist-types",
+		"dev": "concurrently -n esbuild,typechk,typewrk -c yellow,blue,blue.dim \"node scripts/build.mjs watch\" \"node scripts/types.mjs tsconfig.json watch\" \"node scripts/types.mjs src/workers/tsconfig.json watch\"",
+		"test": "node scripts/build.mjs && ava && rimraf ./.tmp",
+		"test:ci": "pnpm run test",
+		"check:lint": "eslint \"{src,test}/**/*.ts\" \"scripts/**/*.{js,mjs}\" \"types/**/*.ts\"",
+		"lint:fix": "pnpm run lint -- --fix",
+		"types:build": "node scripts/types.mjs tsconfig.json && node scripts/types.mjs src/workers/tsconfig.json"
+	},
 	"dependencies": {
 		"acorn": "^8.8.0",
 		"acorn-walk": "^8.2.0",
@@ -41,19 +52,31 @@
 		"zod": "^3.20.6"
 	},
 	"devDependencies": {
+		"@ava/typescript": "^4.0.0",
 		"@cloudflare/kv-asset-handler": "^0.3.0",
 		"@cloudflare/workers-types": "^4.20231002.0",
+		"@microsoft/api-extractor": "^7.36.3",
 		"@types/debug": "^4.1.7",
 		"@types/estree": "^1.0.0",
 		"@types/glob-to-regexp": "^0.4.1",
 		"@types/http-cache-semantics": "^4.0.1",
+		"@types/node": "^18.11.9",
+		"@types/rimraf": "^3.0.2",
 		"@types/source-map-support": "^0.5.6",
 		"@types/stoppable": "^1.1.1",
+		"@types/which": "^2.0.1",
 		"@types/ws": "^8.5.3",
+		"ava": "^5.2.0",
+		"capnpc-ts": "^0.7.0",
 		"devalue": "^4.3.0",
 		"devtools-protocol": "^0.0.1182435",
+		"esbuild": "^0.16.17",
+		"expect-type": "^0.15.0",
 		"http-cache-semantics": "^4.1.0",
-		"kleur": "^4.1.5"
+		"kleur": "^4.1.5",
+		"rimraf": "^3.0.2",
+		"source-map": "^0.6.0",
+		"which": "^2.0.2"
 	},
 	"engines": {
 		"node": ">=16.13"

packages/miniflare/scripts/build.mjs

@@ -0,0 +1,197 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import esbuild from "esbuild";
+import { getPackage, pkgRoot } from "./common.mjs";
+
+const argv = process.argv.slice(2);
+const watch = argv[0] === "watch";
+
+/**
+ * Recursively walks a directory, returning a list of all files contained within
+ * @param {string} rootPath
+ * @returns {Promise<string[]>}
+ */
+async function walk(rootPath) {
+	const fileNames = await fs.readdir(rootPath);
+	const walkPromises = fileNames.map(async (fileName) => {
+		const filePath = path.join(rootPath, fileName);
+		return (await fs.stat(filePath)).isDirectory()
+			? await walk(filePath)
+			: [filePath];
+	});
+	return (await Promise.all(walkPromises)).flat();
+}
+
+/**
+ * Gets a list of dependency names from the passed package
+ * @param {~Package} pkg
+ * @param {boolean} [includeDev]
+ * @returns {string[]}
+ */
+function getPackageDependencies(pkg, includeDev) {
+	return [
+		...(pkg.dependencies ? Object.keys(pkg.dependencies) : []),
+		...(includeDev && pkg.devDependencies
+			? Object.keys(pkg.devDependencies)
+			: []),
+		...(pkg.peerDependencies ? Object.keys(pkg.peerDependencies) : []),
+		...(pkg.optionalDependencies ? Object.keys(pkg.optionalDependencies) : []),
+	];
+}
+
+const workersRoot = path.join(pkgRoot, "src", "workers");
+
+const miniflareSharedExtensionPath = path.join(
+	workersRoot,
+	"shared",
+	"index.worker.ts"
+);
+const miniflareZodExtensionPath = path.join(
+	workersRoot,
+	"shared",
+	"zod.worker.ts"
+);
+/**
+ * `workerd` `extensions` don't have access to "built-in" modules like
+ * `node:buffer`, but do have access to "internal" modules like
+ * `node-internal:internal_buffer`, which usually provide the same exports.
+ * So that we can use `node:assert` and `node:buffer` in our shared extension,
+ * rewrite built-in names to internal.
+ * @type {esbuild.Plugin}
+ */
+const rewriteNodeToInternalPlugin = {
+	name: "rewrite-node-to-internal",
+	setup(build) {
+		build.onResolve({ filter: /^node:(assert|buffer)$/ }, async (args) => {
+			const module = args.path.substring("node:".length);
+			return { path: `node-internal:internal_${module}`, external: true };
+		});
+	},
+};
+
+/**
+ * @type {Map<string, esbuild.BuildResult>}
+ */
+const workersBuilders = new Map();
+/**
+ * @type {esbuild.Plugin}
+ */
+const embedWorkersPlugin = {
+	name: "embed-workers",
+	setup(build) {
+		const namespace = "embed-worker";
+		build.onResolve({ filter: /^worker:/ }, async (args) => {
+			let name = args.path.substring("worker:".length);
+			// Allow `.worker` to be omitted
+			if (!name.endsWith(".worker")) name += ".worker";
+			// Use `build.resolve()` API so Workers can be written as `m?[jt]s` files
+			const result = await build.resolve("./" + name, {
+				kind: "import-statement",
+				resolveDir: workersRoot,
+			});
+			if (result.errors.length > 0) return { errors: result.errors };
+			return { path: result.path, namespace };
+		});
+		build.onLoad({ filter: /.*/, namespace }, async (args) => {
+			let builder = workersBuilders.get(args.path);
+			if (builder === undefined) {
+				builder = await esbuild.build({
+					platform: "node", // Marks `node:*` imports as external
+					format: "esm",
+					target: "esnext",
+					bundle: true,
+					sourcemap: true,
+					sourcesContent: false,
+					external: ["miniflare:shared", "miniflare:zod"],
+					metafile: true,
+					incremental: watch, // Allow `rebuild()` calls if watching
+					entryPoints: [args.path],
+					minifySyntax: true,
+					outdir: build.initialOptions.outdir,
+					outbase: pkgRoot,
+					plugins:
+						args.path === miniflareSharedExtensionPath ||
+						args.path === miniflareZodExtensionPath
+							? [rewriteNodeToInternalPlugin]
+							: [],
+				});
+			} else {
+				builder = await builder.rebuild();
+			}
+			workersBuilders.set(args.path, builder);
+			await fs.mkdir("worker-metafiles", { recursive: true });
+			await fs.writeFile(
+				path.join(
+					"worker-metafiles",
+					path.basename(args.path) + ".metafile.json"
+				),
+				JSON.stringify(builder.metafile)
+			);
+			let outPath = args.path.substring(workersRoot.length + 1);
+			outPath = outPath.substring(0, outPath.lastIndexOf(".")) + ".js";
+			outPath = JSON.stringify(outPath);
+			const watchFiles = Object.keys(builder.metafile.inputs);
+			const contents = `
+      import fs from "fs";
+      import path from "path";
+      import url from "url";
+      let contents;
+      export default function() {
+         if (contents !== undefined) return contents;
+         const filePath = path.join(__dirname, "workers", ${outPath});
+         contents = fs.readFileSync(filePath, "utf8") + "//# sourceURL=" + url.pathToFileURL(filePath);
+         return contents;
+      }
+      `;
+			return { contents, loader: "js", watchFiles };
+		});
+	},
+};
+
+async function buildPackage() {
+	const pkg = await getPackage(pkgRoot);
+
+	const indexPath = path.join(pkgRoot, "src", "index.ts");
+	// Look for test files ending with .spec.ts in the test directory, default to
+	// empty array if not found
+	let testPaths = [];
+	try {
+		testPaths = (await walk(path.join(pkgRoot, "test"))).filter((testPath) =>
+			testPath.endsWith(".spec.ts")
+		);
+	} catch (e) {
+		if (e.code !== "ENOENT") throw e;
+	}
+	const outPath = path.join(pkgRoot, "dist");
+
+	await esbuild.build({
+		platform: "node",
+		format: "cjs",
+		target: "esnext",
+		bundle: true,
+		sourcemap: true,
+		sourcesContent: false,
+		tsconfig: path.join(pkgRoot, "tsconfig.json"),
+		// Mark root package's dependencies as external, include root devDependencies
+		// (e.g. test runner) as we don't want these bundled
+		external: [
+			// Make sure we're not bundling any packages we're building, we want to
+			// test against the actual code we'll publish for instance
+			"miniflare",
+			// Mark `dependencies` as external, but not `devDependencies` (we use them
+			// to signal single-use/small packages we want inlined in the bundle)
+			...getPackageDependencies(pkg),
+			// Mark test dependencies as external
+			"ava",
+			"esbuild",
+		],
+		plugins: [embedWorkersPlugin],
+		logLevel: watch ? "info" : "warning",
+		watch,
+		outdir: outPath,
+		outbase: pkgRoot,
+		entryPoints: [indexPath, ...testPaths],
+	});
+}
+
+await buildPackage();

packages/miniflare/scripts/common.mjs

@@ -0,0 +1,29 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+export const pkgRoot = path.resolve(__dirname, "..");
+
+/**
+ * @typedef {object} ~Package
+ * @property {string} name
+ * @property {string} version
+ * @property {Record<string, string>} [dependencies]
+ * @property {Record<string, string>} [devDependencies]
+ * @property {Record<string, string>} [peerDependencies]
+ * @property {Record<string, string>} [optionalDependencies]
+ * @property {string[]} [entryPoints]
+ */
+
+/**
+ * Gets the contents of the package.json file in <pkgRoot>
+ * @param {string} pkgRoot
+ * @returns {Promise<~Package>}
+ */
+export async function getPackage(pkgRoot) {
+	return JSON.parse(
+		await fs.readFile(path.join(pkgRoot, "package.json"), "utf8")
+	);
+}

packages/miniflare/test/plugins/core/errors/index.spec.ts

@@ -6,7 +6,7 @@ import test from "ava";
 import Protocol from "devtools-protocol";
 import esbuild from "esbuild";
 import { DeferredPromise, Miniflare } from "miniflare";
-import { RawSourceMap } from "source-map";
+import type { RawSourceMap } from "source-map";
 import NodeWebSocket from "ws";
 import { escapeRegexp, useTmp } from "../../../test-shared";
 
@@ -136,7 +136,7 @@ addEventListener("fetch", (event) => {
 		message: "unnamed",
 	});
 	const serviceWorkerEntryRegexp = escapeRegexp(
-		`${SERVICE_WORKER_ENTRY_PATH}:6:17`
+		`${SERVICE_WORKER_ENTRY_PATH}:6:16`
 	);
 	t.regex(String(error?.stack), serviceWorkerEntryRegexp);
 	error = await t.throwsAsync(mf.dispatchFetch("http://localhost/a"), {
@@ -148,7 +148,7 @@ addEventListener("fetch", (event) => {
 	error = await t.throwsAsync(mf.dispatchFetch("http://localhost/b"), {
 		message: "b",
 	});
-	const modulesEntryRegexp = escapeRegexp(`${MODULES_ENTRY_PATH}:5:19`);
+	const modulesEntryRegexp = escapeRegexp(`${MODULES_ENTRY_PATH}:5:17`);
 	t.regex(String(error?.stack), modulesEntryRegexp);
 	error = await t.throwsAsync(mf.dispatchFetch("http://localhost/c"), {
 		message: "c",
@@ -174,7 +174,7 @@ addEventListener("fetch", (event) => {
 		instanceOf: TypeError,
 		message: "Dependency error",
 	});
-	const nestedRegexp = escapeRegexp(`${DEP_ENTRY_PATH}:4:17`);
+	const nestedRegexp = escapeRegexp(`${DEP_ENTRY_PATH}:4:16`);
 	t.regex(String(error?.stack), nestedRegexp);
 
 	// Check source mapping URLs rewritten

packages/miniflare/test/plugins/core/index.spec.ts

@@ -52,12 +52,13 @@ opensslTest("NODE_EXTRA_CA_CERTS: loads certificates", async (t) => {
 
 	// Start Miniflare with NODE_EXTRA_CA_CERTS environment variable
 	// (cannot use sync process methods here as that would block HTTPS server)
+	const miniflarePath = require.resolve("miniflare");
 	const result = childProcess.spawn(
 		process.execPath,
 		[
 			"-e",
 			`
-      const { Miniflare } = require("miniflare");
+      const { Miniflare } = require(${JSON.stringify(miniflarePath)});
       const mf = new Miniflare({
         verbose: true,
         modules: true,

packages/miniflare/test/plugins/core/modules.spec.ts

@@ -241,7 +241,7 @@ You must manually define your modules when constructing Miniflare:
       ...
     ]
   })
-    at ${scriptPath}:14:17`
+    at ${scriptPath}:14:15`
 	);
 
 	// Check with dynamic require