Skip to content

Commit

Permalink
feat: isomorphic wasi (again) and patched wasi-js (#1584)
Browse files Browse the repository at this point in the history
This PR reintroduces #1571 with 3 additional changes to avoid [the last regression](#1578):
- `patch-package` is used to mess with wasi-js, to remove the preinstall script from their package.json. The patching is performed during packaging, where it's actually effective (for downstream consumers).
- wasi-js is included as a bundled dep for winglang to ensure that the patched `package.json` is appropriately patched in time to avoid any hooks. Normally this isn't needed with patch package, but due to the particular mutation I'm not sure this is avoidable
- Added checks to hangar to ensure we avoid introducing script hooks. I verified that these tests pass with the latest version of the repo but fail the version noted in the regression.

The changes from the original PR are untouched.

*By submitting this pull request, I confirm that my contribution is made under the terms of the [Monada Contribution License](https://docs.winglang.io/terms-and-policies/contribution-license.html)*.
  • Loading branch information
MarkMcCulloh authored Feb 20, 2023
1 parent 5332d6e commit e8e8860
Show file tree
Hide file tree
Showing 16 changed files with 166,324 additions and 5,963 deletions.
99,402 changes: 98,343 additions & 1,059 deletions apps/wing-playground/package-lock.json

Large diffs are not rendered by default.

9 changes: 4 additions & 5 deletions apps/wing-playground/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,12 @@
},
"author": "Monada",
"dependencies": {
"@wasmer/wasi": "^1.2.2",
"@cowasm/memfs": "^3.5.1",
"winglang": "file:../../apps/wing",
"@winglang/sdk": "file:../../libs/wingsdk"
},
"devDependencies": {
"@esbuild-plugins/node-globals-polyfill": "^0.2.3",
"@esbuild-plugins/node-modules-polyfill": "^0.2.2",
"rollup-plugin-polyfill-node": "^0.12.0",
"vite": "^4.1.1"
"vite": "^4.1.1",
"vite-plugin-node-polyfills": "^0.7.0"
}
}
34 changes: 6 additions & 28 deletions apps/wing-playground/project.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,48 +2,26 @@
"name": "wing-playground",
"$schema": "../../node_modules/nx/schemas/project-schema.json",
"implicitDependencies": [
"wingc",
"winglang",
"sdk"
],
"targets": {
"copy": {
"executor": "nx:run-commands",
"inputs": [
"{workspaceRoot}/target/*/wingc.wasm"
],
"dependsOn": [
"^build"
],
"options": {
"command": "cp -v ../../target/wasm32-wasi/debug/wingc.wasm ./",
"cwd": "apps/wing-playground"
},
"configurations": {
"release": {
"command": "cp -v ../../target/wasm32-wasi/release/wingc.wasm ./"
}
}
},
"build": {
"executor": "nx:run-script",
"dependsOn": [
"copy",
"^build"
],
"executor": "nx:run-commands",
"options": {
"cwd": "apps/wing-playground",
"script": "build"
"command": "npm run build"
}
},
"dev": {
"executor": "nx:run-script",
"executor": "nx:run-commands",
"dependsOn": [
"copy",
"build",
"^build"
],
"options": {
"cwd": "apps/wing-playground",
"script": "dev"
"command": "npm run dev"
}
}
}
Expand Down
40 changes: 22 additions & 18 deletions apps/wing-playground/vite.config.js
Original file line number Diff line number Diff line change
@@ -1,36 +1,40 @@
// idk how many of these are actually needed
import { NodeGlobalsPolyfillPlugin } from "@esbuild-plugins/node-globals-polyfill";
import { NodeModulesPolyfillPlugin } from "@esbuild-plugins/node-modules-polyfill";
import rollupNodePolyFill from "rollup-plugin-node-polyfills";
import { nodePolyfills } from "vite-plugin-node-polyfills";

/** @type {import('vite').UserConfig} */
export default {
resolve: {
preserveSymlinks: true,
alias: {
process: "rollup-plugin-node-polyfills/polyfills/process-es6",
buffer: "rollup-plugin-node-polyfills/polyfills/buffer-es6",
"wasi-js/dist/bindings/node": "wasi-js/dist/bindings/browser",
},
},
plugins: [nodePolyfills()],
worker: {
format: "es",
plugins: [nodePolyfills()],
rollupOptions: {
preserveSymlinks: true,
},
},
build: {
target: "esnext",
rollupOptions: {
plugins: [rollupNodePolyFill()],
target: "es2022",
commonjsOptions: {
// This is needed because winglang is symlinked
include: [/winglang/, /node_modules/],
},
},
server: {
fs: {
allow: [".."],
},
},
optimizeDeps: {
include: ["winglang"],
esbuildOptions: {
define: {
global: "globalThis",
},
plugins: [
NodeGlobalsPolyfillPlugin({
process: true,
buffer: true,
}),
NodeModulesPolyfillPlugin(),
],
target: "es2022",
preserveSymlinks: true,
},
force: true,
},
};
137 changes: 25 additions & 112 deletions apps/wing-playground/worker.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import wingcURL from "./wingc.wasm?url";
import { init, WASI } from "@wasmer/wasi";
import { env } from "process";

const WINGC_COMPILE = "wingc_compile";
import { load, invoke } from "winglang";
import { createFsFromVolume } from "@cowasm/memfs";
import wingcURL from "winglang/wingc.wasm?url";
import { Volume } from "@cowasm/memfs";

const wingsdkJSIIContent = await import("@winglang/sdk/.jsii?raw").then(
(i) => i.default
Expand All @@ -11,151 +10,65 @@ const wingsdkPackageJsonContent = await import(
"@winglang/sdk/package.json?raw"
).then((i) => i.default);

await init();
const fs = createFsFromVolume(
Volume.fromJSON({
"/wingsdk/package.json": wingsdkPackageJsonContent,
"/wingsdk/.jsii": wingsdkJSIIContent,
})
);

const wasm = await WebAssembly.compileStreaming(fetch(wingcURL));
let wasmFetchData = await fetch(wingcURL).then((d) => d.arrayBuffer());
const wingcWASMData = new Uint8Array(wasmFetchData);

let wasi = new WASI({
const wingc = await load({
env: {
...env,
WINGSDK_MANIFEST_ROOT: "/wingsdk",
RUST_BACKTRACE: "full",
},
fs: fs,
wingcWASMData,
wingsdkManifestRoot: "/wingsdk",
});

const instance = wasi.instantiate(wasm, {
env: {
// This function is used by the language server, which is not used in the playground
send_notification: () => {},
}
});

const defaultFilePerms = { read: true, write: true, create: true };
wasi.fs.createDir("/wingsdk");
let wingsdk_packagejson_file = wasi.fs.open(
"/wingsdk/package.json",
defaultFilePerms
);
wingsdk_packagejson_file.writeString(wingsdkPackageJsonContent);
let wingsdk_jsii_file = wasi.fs.open("/wingsdk/.jsii", defaultFilePerms);
wingsdk_jsii_file.writeString(wingsdkJSIIContent);

self.onmessage = async (event) => {
if (event.data === "") {
self.postMessage(undefined);
return;
}

try {
let file = wasi.fs.open("/code.w", defaultFilePerms);
file.writeString(event.data);
fs.writeFileSync("/code.w", event.data);

const compileResult = invoke(wingc, "wingc_compile", "/code.w");

const compileResult = wingcInvoke(instance, WINGC_COMPILE, "code.w");
const stderr = wasi.getStderrString();
if (stderr) {
throw stderr;
}
if (compileResult !== 0) {
throw compileResult;
}

const stdout = wasi.getStdoutString();
let intermediateJS = "";

const intermediatePath = "/code.w.out/preflight.js";
const intermediateFile = wasi.fs.open(intermediatePath, defaultFilePerms);
intermediateJS += intermediateFile.readString();
wasi.fs.removeFile(intermediatePath);
intermediateJS += fs.readFileSync("/code.w.out/preflight.js").toString();

let procRegex = /fromFile\(.+"(.+index\.js)"/g;
let procMatch;
while ((procMatch = procRegex.exec(intermediateJS))) {
const proc = procMatch[1];
const procPath = `/code.w.out/${proc}`;
let procFile = wasi.fs.open(procPath, defaultFilePerms);
intermediateJS += `\n\n// ${proc}\n// START\n${procFile.readString()}\n// END`;
wasi.fs.removeFile(procPath);
let procFile = fs.readFileSync(procPath);
intermediateJS += `\n\n// ${proc}\n// START\n${procFile}\n// END`;
}

self.postMessage({
stdout,
stderr: wasi.getStderrString(),
stdout: "",
stderr: "",
intermediateJS,
});
} catch (error) {
console.error(error);
self.postMessage({
stderr: error,
stdout: wasi.getStdoutString(),
stdout: "",
intermediateJS: null,
});
} finally {
try {
wasi.fs.removeFile("/code.w");
} catch (error) {}
}
};

self.postMessage("WORKER_READY");

// When WASM stuff returns a value, we need both a pointer and a length,
// We are using 32 bits for each, so we can combine them into a single 64 bit value.
// This is a bit mask to extract the low order 32 bits.
// https://stackoverflow.com/questions/5971645/extracting-high-and-low-order-bytes-of-a-64-bit-integer
const LOW_MASK = 2n ** 32n - 1n;
const HIGH_MASK = BigInt(32);

/**
* Runs the given WASM function in the Wing Compiler WASM instance.
*
* Assumptions:
* 1. The called WASM function is expecting a pointer and a length representing a string
* 2. The string will be UTF-8 encoded
* 3. The string will be less than 2^32 bytes long (4GB)
* 4. The WASI instance has already been initialized
*/
export function wingcInvoke(
instance,
func,
arg
) {
const exports = instance.exports;

const bytes = new TextEncoder().encode(arg);
const argPointer = exports.wingc_malloc(bytes.byteLength);

// track memory to free after the call
const toFree = [[argPointer, bytes.byteLength]];

try {
const argMemoryBuffer = new Uint8Array(
exports.memory.buffer,
argPointer,
bytes.byteLength
);
argMemoryBuffer.set(bytes);

const result = exports[func](argPointer, bytes.byteLength);

if (result === 0 || result === undefined || result === 0n) {
return 0;
} else {
const returnPtr = Number(result >> HIGH_MASK);
const returnLen = Number(result & LOW_MASK);

const entireMemoryBuffer = new Uint8Array(exports.memory.buffer);
const extractedBuffer = entireMemoryBuffer.slice(
returnPtr,
returnPtr + returnLen
);

toFree.push([returnPtr, returnLen]);

return new TextDecoder().decode(extractedBuffer) + "";
}
} finally {
toFree.forEach(([pointer, length]) => {
exports.wingc_free(pointer, length);
});
}
}
30 changes: 1 addition & 29 deletions apps/wing/bin/wing
Original file line number Diff line number Diff line change
@@ -1,31 +1,3 @@
#!/usr/bin/env node

// Only certain builds and versions of Node support WASI.
// We first assume this script is loaded in the correct runtime environment, and
// if not, we try to load it in a WASI-compatible runtime environment by running
// the same script (self) over a new Node process with the correct flags passed.
// package.json warns if user npm installs this for a non-compatible runtime.

try {
require.resolve("wasi");
} catch (err) {
if (err.code === "MODULE_NOT_FOUND") {
const { spawnSync } = require("child_process");
const { env, execPath, execArgv, argv } = process;
env.NODE_OPTIONS = [
...new Set((env.NODE_OPTIONS ?? "").split(" "))
.add("--experimental-wasi-unstable-preview1")
.add("--no-warnings"),
].join(" ");

const { status } = spawnSync(execPath, execArgv.concat(argv.slice(1)), {
env,
stdio: "inherit",
});
process.exit(status ?? 1);
} else {
console.error('Unable to load "wasi" module', err);
process.exit(1);
}
}
require("../dist/index.js");
require("../dist/cli.js");
Loading

0 comments on commit e8e8860

Please sign in to comment.