Skip to content

Commit

Permalink
feat(vm): support wasm module (#5131)
Browse files Browse the repository at this point in the history
  • Loading branch information
hi-ogawa authored Feb 8, 2024
1 parent 7a31a1a commit 5ed537f
Show file tree
Hide file tree
Showing 12 changed files with 205 additions and 13 deletions.
1 change: 1 addition & 0 deletions eslint.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export default antfu(
'**/bench.json',
'**/fixtures',
'test/core/src/self',
'test/core/src/wasm-bindgen-no-cyclic',
'test/workspaces/results.json',
'test/reporters/fixtures/with-syntax-error.test.js',
'test/network-imports/public/[email protected]',
Expand Down
13 changes: 10 additions & 3 deletions packages/vitest/src/runtime/external-executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ export interface ExternalModulesExecutorOptions {
}

interface ModuleInformation {
type: 'data' | 'builtin' | 'vite' | 'module' | 'commonjs'
type: 'data' | 'builtin' | 'vite' | 'wasm' | 'module' | 'commonjs'
url: string
path: string
}
Expand Down Expand Up @@ -165,7 +165,7 @@ export class ExternalModulesExecutor {
const pathUrl = isFileUrl ? fileURLToPath(identifier.split('?')[0]) : identifier
const fileUrl = isFileUrl ? identifier : pathToFileURL(pathUrl).toString()

let type: 'module' | 'commonjs' | 'vite'
let type: 'module' | 'commonjs' | 'vite' | 'wasm'
if (this.vite.canResolve(fileUrl)) {
type = 'vite'
}
Expand All @@ -175,6 +175,11 @@ export class ExternalModulesExecutor {
else if (extension === '.cjs') {
type = 'commonjs'
}
else if (extension === '.wasm') {
// still experimental on NodeJS --experimental-wasm-modules
// cf. ESM_FILE_FORMAT(url) in https://nodejs.org/docs/latest-v20.x/api/esm.html#resolution-algorithm
type = 'wasm'
}
else {
const pkgData = this.findNearestPackageData(normalize(pathUrl))
type = pkgData.type === 'module' ? 'module' : 'commonjs'
Expand All @@ -188,7 +193,7 @@ export class ExternalModulesExecutor {

// create ERR_MODULE_NOT_FOUND on our own since latest NodeJS's import.meta.resolve doesn't throw on non-existing namespace or path
// https://github.com/nodejs/node/pull/49038
if ((type === 'module' || type === 'commonjs') && !existsSync(path)) {
if ((type === 'module' || type === 'commonjs' || type === 'wasm') && !existsSync(path)) {
const error = new Error(`Cannot find module '${path}'`)
;(error as any).code = 'ERR_MODULE_NOT_FOUND'
throw error
Expand All @@ -203,6 +208,8 @@ export class ExternalModulesExecutor {
}
case 'vite':
return await this.vite.createViteModule(url)
case 'wasm':
return await this.esm.createWebAssemblyModule(url, this.fs.readBuffer(path))
case 'module':
return await this.esm.createEsModule(url, this.fs.readFile(path))
case 'commonjs': {
Expand Down
19 changes: 13 additions & 6 deletions packages/vitest/src/runtime/vm/esm-executor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,15 @@ export class EsmExecutor {
return m
}

public async createWebAssemblyModule(fileUrl: string, code: Buffer) {
const cached = this.moduleCache.get(fileUrl)
if (cached)
return cached
const m = this.loadWebAssemblyModule(code, fileUrl)
this.moduleCache.set(fileUrl, m)
return m
}

public async loadWebAssemblyModule(source: Buffer, identifier: string) {
const cached = this.moduleCache.get(identifier)
if (cached)
Expand All @@ -90,23 +99,21 @@ export class EsmExecutor {
const moduleLookup: Record<string, VMModule> = {}
for (const { module } of imports) {
if (moduleLookup[module] === undefined) {
const resolvedModule = await this.executor.resolveModule(
moduleLookup[module] = await this.executor.resolveModule(
module,
identifier,
)

moduleLookup[module] = await this.evaluateModule(resolvedModule)
}
}

const syntheticModule = new SyntheticModule(
exports.map(({ name }) => name),
() => {
async () => {
const importsObject: WebAssembly.Imports = {}
for (const { module, name } of imports) {
if (!importsObject[module])
importsObject[module] = {}

await this.evaluateModule(moduleLookup[module])
importsObject[module][name] = (moduleLookup[module].namespace as any)[name]
}
const wasmInstance = new WebAssembly.Instance(
Expand Down Expand Up @@ -150,7 +157,7 @@ export class EsmExecutor {
if (encoding !== 'base64')
throw new Error(`Invalid data URI encoding: ${encoding}`)

const module = await this.loadWebAssemblyModule(
const module = this.loadWebAssemblyModule(
Buffer.from(match.groups.code, 'base64'),
identifier,
)
Expand Down
15 changes: 15 additions & 0 deletions test/core/src/wasm-bindgen-no-cyclic/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
The recent version of the wasm-bindgen bundler output does not use cyclic imports between wasm and js.

For this non-cyclic version to work, both `index_bg.js` and `index_bg.wasm` need to be externalized
since otherwise a dual package hazard on `index_bg.js` would make it non-functional.

The code is copied from https://github.com/rustwasm/wasm-bindgen/tree/8198d2d25920e1f4fc593e9f8eb9d199e004d731/examples/hello_world

```sh
npm i
npm run build
# then
# 1. copy `examples/hello_world/pkg` to this directory
# 2. add { "type": "module" } to `package.json`
# (this will be automatically included after https://github.com/rustwasm/wasm-pack/pull/1061)
```
6 changes: 6 additions & 0 deletions test/core/src/wasm-bindgen-no-cyclic/index.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
/* tslint:disable */
/* eslint-disable */
/**
* @param {string} name
*/
export function greet(name: string): void;
4 changes: 4 additions & 0 deletions test/core/src/wasm-bindgen-no-cyclic/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import * as wasm from "./index_bg.wasm";
import { __wbg_set_wasm } from "./index_bg.js";
__wbg_set_wasm(wasm);
export * from "./index_bg.js";
117 changes: 117 additions & 0 deletions test/core/src/wasm-bindgen-no-cyclic/index_bg.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
let wasm;
export function __wbg_set_wasm(val) {
wasm = val;
}


const lTextDecoder = typeof TextDecoder === 'undefined' ? (0, module.require)('util').TextDecoder : TextDecoder;

let cachedTextDecoder = new lTextDecoder('utf-8', { ignoreBOM: true, fatal: true });

cachedTextDecoder.decode();

let cachedUint8Memory0 = null;

function getUint8Memory0() {
if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) {
cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8Memory0;
}

function getStringFromWasm0(ptr, len) {
ptr = ptr >>> 0;
return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
}

function logError(f, args) {
try {
return f.apply(this, args);
} catch (e) {
let error = (function () {
try {
return e instanceof Error ? `${e.message}\n\nStack:\n${e.stack}` : e.toString();
} catch(_) {
return "<failed to stringify thrown value>";
}
}());
console.error("wasm-bindgen: imported JS function that was not marked as `catch` threw an error:", error);
throw e;
}
}

let WASM_VECTOR_LEN = 0;

const lTextEncoder = typeof TextEncoder === 'undefined' ? (0, module.require)('util').TextEncoder : TextEncoder;

let cachedTextEncoder = new lTextEncoder('utf-8');

const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
? function (arg, view) {
return cachedTextEncoder.encodeInto(arg, view);
}
: function (arg, view) {
const buf = cachedTextEncoder.encode(arg);
view.set(buf);
return {
read: arg.length,
written: buf.length
};
});

function passStringToWasm0(arg, malloc, realloc) {

if (typeof(arg) !== 'string') throw new Error('expected a string argument');

if (realloc === undefined) {
const buf = cachedTextEncoder.encode(arg);
const ptr = malloc(buf.length, 1) >>> 0;
getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf);
WASM_VECTOR_LEN = buf.length;
return ptr;
}

let len = arg.length;
let ptr = malloc(len, 1) >>> 0;

const mem = getUint8Memory0();

let offset = 0;

for (; offset < len; offset++) {
const code = arg.charCodeAt(offset);
if (code > 0x7F) break;
mem[ptr + offset] = code;
}

if (offset !== len) {
if (offset !== 0) {
arg = arg.slice(offset);
}
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
const view = getUint8Memory0().subarray(ptr + offset, ptr + len);
const ret = encodeString(arg, view);
if (ret.read !== arg.length) throw new Error('failed to pass whole string');
offset += ret.written;
}

WASM_VECTOR_LEN = offset;
return ptr;
}
/**
* @param {string} name
*/
export function greet(name) {
const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.greet(ptr0, len0);
}

export function __wbg_alert_9ea5a791b0d4c7a3() { return logError(function (arg0, arg1) {
alert(getStringFromWasm0(arg0, arg1));
}, arguments) };

export function __wbindgen_throw(arg0, arg1) {
throw new Error(getStringFromWasm0(arg0, arg1));
};

Binary file not shown.
6 changes: 6 additions & 0 deletions test/core/src/wasm-bindgen-no-cyclic/index_bg.wasm.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export function greet(a: number, b: number): void;
export function __wbindgen_malloc(a: number, b: number): number;
export function __wbindgen_realloc(a: number, b: number, c: number, d: number): number;
20 changes: 20 additions & 0 deletions test/core/src/wasm-bindgen-no-cyclic/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"type": "module",
"name": "hello_world",
"collaborators": [
"The wasm-bindgen Developers"
],
"version": "0.1.0",
"files": [
"index_bg.wasm",
"index.js",
"index_bg.js",
"index.d.ts"
],
"module": "index.js",
"types": "index.d.ts",
"sideEffects": [
"./index.js",
"./snippets/*"
]
}
13 changes: 11 additions & 2 deletions test/core/test/vm-wasm.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { expect, test, vi } from 'vitest'
// @ts-expect-error wasm is not typed
import { add } from '../src/add.wasm'

const wasmFileBuffer = readFileSync(resolve(__dirname, './src/add.wasm'))
const wasmFileBuffer = readFileSync(resolve(__dirname, '../src/add.wasm'))

test('supports native wasm imports', () => {
expect(add(1, 2)).toBe(3)
Expand Down Expand Up @@ -54,7 +54,7 @@ test('imports from "data:application/wasm" URI with invalid encoding fail', asyn
).rejects.toThrow('Invalid data URI encoding: charset=utf-8')
})

test('supports wasm files that import js resources (wasm-bindgen)', async () => {
test('supports wasm/js cyclic import (old wasm-bindgen output)', async () => {
globalThis.alert = vi.fn()

// @ts-expect-error not typed
Expand All @@ -63,3 +63,12 @@ test('supports wasm files that import js resources (wasm-bindgen)', async () =>

expect(globalThis.alert).toHaveBeenCalledWith('Hello, World!')
})

test('supports wasm-bindgen', async () => {
globalThis.alert = vi.fn()

const { greet } = await import('../src/wasm-bindgen-no-cyclic/index.js')
greet('No Cyclic')

expect(globalThis.alert).toHaveBeenCalledWith('Hello, No Cyclic!')
})
4 changes: 2 additions & 2 deletions test/core/vite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export default defineConfig({
},
test: {
name: 'core',
exclude: ['**/fixtures/**', '**/vm-wasm.test.ts', ...defaultExclude],
exclude: ['**/fixtures/**', ...defaultExclude],
slowTestThreshold: 1000,
testTimeout: 2000,
setupFiles: [
Expand Down Expand Up @@ -75,7 +75,7 @@ export default defineConfig({
},
server: {
deps: {
external: ['tinyspy', /src\/external/, /esm\/esm/, /\.wasm$/],
external: ['tinyspy', /src\/external/, /esm\/esm/, /\.wasm$/, /\/wasm-bindgen-no-cyclic\/index_bg/],
inline: ['inline-lib'],
},
},
Expand Down

0 comments on commit 5ed537f

Please sign in to comment.