[WasmFS] Async proxied JS backend (#16229)

The key file src/library_wasmfs_fetch.js here shows a simple
async JS backend that depends on pthreads proxying. That is, the main code
looks sync as usual, and we proxy to a dedicated thread which does the async
operation, here, a network fetch().

This is a "hello world" backend, the minimal one I can think of that is async. But
it may still be useful - we used to have a LazyFile option in the old FS, which this
is very close to, that is, on the first read of the data we fetch it from the network,
and then it is cached like a normal JS file.

To implement this, add a new ProxiedAsyncJSImplFile in C++. This combines
the matters of proxying and the target being async. In theory we could add two
layers, here, first a C++ File that is async (perhaps using C++ futures?), but I
think that might be over-engineering, since we don't really want the async
aspect for C++ - it's very specific to JS. However, those are internal details, and
we could refactor the code later to add such laying if we wanted.

The JS side is the important part here. Basically each JS backend would define
a bunch of JS hooks that return Promises, and everything else is taken care of
automatically.

src/library.js

@@ -3648,7 +3648,6 @@ LibraryManager.library = {
     return x.indexOf('dynCall_') == 0 || unmangledSymbols.includes(x) ? x : '_' + x;
   },
 
-
   $asyncLoad__docs: '/** @param {boolean=} noRunDep */',
   $asyncLoad: function(url, onload, onerror, noRunDep) {
     var dep = !noRunDep ? getUniqueRunDependency('al ' + url) : '';

src/library_wasmfs.js

@@ -149,32 +149,36 @@ var WasmFSLibrary = {
   },
 
   // Backend support. wasmFS$backends will contain a mapping of backend IDs to
-  // the JS code that implements them. This is the JS side of the JSImpl class
+  // the JS code that implements them. This is the JS side of the JSImpl* class
   // in C++, together with the js_impl calls defined right after it.
   $wasmFS$backends: {},
 
+  // JSImpl
+
   _wasmfs_jsimpl_alloc_file: function(backend, file) {
 #if ASSERTIONS
     assert(wasmFS$backends[backend]);
 #endif
-    return wasmFS$backends[backend].alloc_file(file);
+    return wasmFS$backends[backend].allocFile(file);
   },
 
   _wasmfs_jsimpl_free_file: function(backend, file) {
 #if ASSERTIONS
     assert(wasmFS$backends[backend]);
 #endif
-    return wasmFS$backends[backend].free_file(file);
+    return wasmFS$backends[backend].freeFile(file);
   },
 
-  _wasmfs_jsimpl_write: function(backend, file, buffer, length, offset) {
+  _wasmfs_jsimpl_write: function(backend, file, buffer, length, {{{ defineI64Param('offset') }}}) {
+    {{{ receiveI64ParamAsDouble('offset') }}}
 #if ASSERTIONS
     assert(wasmFS$backends[backend]);
 #endif
     return wasmFS$backends[backend].write(file, buffer, length, offset);
   },
 
-  _wasmfs_jsimpl_read: function(backend, file, buffer, length, offset) {
+  _wasmfs_jsimpl_read: function(backend, file, buffer, length, {{{ defineI64Param('offset') }}}) {
+    {{{ receiveI64ParamAsDouble('offset') }}}
 #if ASSERTIONS
     assert(wasmFS$backends[backend]);
 #endif
@@ -187,6 +191,76 @@ var WasmFSLibrary = {
 #endif
     return wasmFS$backends[backend].getSize(file);
   },
+
+  // ProxiedAsyncJSImpl. Each function receives a function pointer and a
+  // parameter. We convert those into a convenient Promise API for the
+  // implementors of backends: the hooks we call should return Promises, which
+  // we then connect to the calling C++.
+
+  // TODO: arg is void*, which for MEMORY64 will be 64-bit. we need a way to
+  //       declare arg in the function signature here (like defineI64Param,
+  //       but that varies for wasm32/wasm64), and a way to do makeDynCall that
+  //       adds a 'p' signature type for pointer, or something like that
+  //       (however, dyncalls might also just work, given in MEMORY64 we assume
+  //       WASM_BIGINT so the pointer is just a single argument, just like in
+  //       wasm32).
+  _wasmfs_jsimpl_async_alloc_file: async function(backend, file, fptr, arg) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    await wasmFS$backends[backend].allocFile(file);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
+
+  _wasmfs_jsimpl_async_free_file: async function(backend, file, fptr, arg) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    await wasmFS$backends[backend].freeFile(file);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
+
+  _wasmfs_jsimpl_async_write: async function(backend, file, buffer, length, {{{ defineI64Param('offset') }}}, fptr, arg) {
+    {{{ receiveI64ParamAsDouble('offset') }}}
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    var size = await wasmFS$backends[backend].write(file, buffer, length, offset);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.result, '0', 'i32') }}};
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.offset, 'size', 'i64') }}};
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
+
+  _wasmfs_jsimpl_async_read: async function(backend, file, buffer, length, {{{ defineI64Param('offset') }}}, fptr, arg) {
+    {{{ receiveI64ParamAsDouble('offset') }}}
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    var size = await wasmFS$backends[backend].read(file, buffer, length, offset);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.result, '0', 'i32') }}};
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.offset, 'size', 'i64') }}};
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
+
+  _wasmfs_jsimpl_async_get_size: async function(backend, file, fptr, arg) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    {{{ runtimeKeepalivePush() }}}
+    var size = await wasmFS$backends[backend].getSize(file);
+    {{{ runtimeKeepalivePop() }}}
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.result, '0', 'i32') }}};
+    {{{ makeSetValue('arg', C_STRUCTS.CallbackState.offset, 'size', 'i64') }}};
+    {{{ makeDynCall('vi', 'fptr') }}}(arg);
+  },
 }
 
 mergeInto(LibraryManager.library, WasmFSLibrary);

src/library_wasmfs_fetch.js

@@ -0,0 +1,63 @@
+mergeInto(LibraryManager.library, {
+  // Fetch backend: On first access of the file (either a read or a getSize), it
+  // will fetch() the data from the network asynchronously. Otherwise, after
+  // that fetch it behaves just like JSFile (and it reuses the code from there).
+
+  _wasmfs_create_fetch_backend_js__deps: [
+    '$wasmFS$backends',
+    '$wasmFS$JSMemoryFiles',
+    '_wasmfs_create_js_file_backend_js',
+  ],
+  _wasmfs_create_fetch_backend_js: async function(backend) {
+    // Get a promise that fetches the data and stores it in JS memory (if it has
+    // not already been fetched).
+    async function getFile(file) {
+      if (wasmFS$JSMemoryFiles[file]) {
+        // The data is already here, so nothing to do before we continue on to
+        // the actual read below.
+        return Promise.resolve();
+      }
+
+      // This is the first time we want the file's data.
+      // TODO: real URL!
+      var url = 'data.dat';
+      var response = await fetch(url);
+      var buffer = await response['arrayBuffer']();
+      wasmFS$JSMemoryFiles[file] = new Uint8Array(buffer);
+    }
+
+    // Start with the normal JSFile operations. This sets
+    //   wasmFS$backends[backend]
+    // which we will then augment.
+    __wasmfs_create_js_file_backend_js(backend);
+
+    // Add the async operations on top.
+    var jsFileOps = wasmFS$backends[backend];
+    wasmFS$backends[backend] = {
+      // alloc/free operations are not actually async. Just forward to the
+      // parent class, but we must return a Promise as the caller expects.
+      allocFile: async (file) => {
+        jsFileOps.allocFile(file);
+        return Promise.resolve();
+      },
+      freeFile: async (file) => {
+        jsFileOps.freeFile(file);
+        return Promise.resolve();
+      },
+
+      write: async (file, buffer, length, offset) => {
+        abort("TODO: file writing in fetch backend? read-only for now");
+      },
+
+      // read/getSize fetch the data, then forward to the parent class.
+      read: async (file, buffer, length, offset) => {
+        await getFile(file);
+        return jsFileOps.read(file, buffer, length, offset);
+      },
+      getSize: async(file) => {
+        await getFile(file);
+        return jsFileOps.getSize(file);
+      },
+    };
+  },
+});

src/library_wasmfs_js_file.js

@@ -9,14 +9,14 @@ mergeInto(LibraryManager.library, {
   ],
   _wasmfs_create_js_file_backend_js: function(backend) {
     wasmFS$backends[backend] = {
-      alloc_file: function(file) {
+      allocFile: (file) => {
         // Do nothing: we allocate the typed array lazily, see write()
       },
-      free_file: function(file) {
+      freeFile: (file) => {
         // Release the memory, as it now has no references to it any more.
         wasmFS$JSMemoryFiles[file] = undefined;
       },
-      write: function(file, buffer, length, offset) {
+      write: (file, buffer, length, offset) => {
         try {
           if (!wasmFS$JSMemoryFiles[file]) {
             // Initialize typed array on first write operation.
@@ -37,15 +37,15 @@ mergeInto(LibraryManager.library, {
           return {{{ cDefine('EIO') }}};
         }
       },
-      read: function(file, buffer, length, offset) {
+      read: (file, buffer, length, offset) => {
         try {
           HEAPU8.set(wasmFS$JSMemoryFiles[file].subarray(offset, offset + length), buffer);
           return 0;
         } catch (err) {
           return {{{ cDefine('EIO') }}};
         }
       },
-      getSize: function(file) {
+      getSize: (file) => {
         return wasmFS$JSMemoryFiles[file] ? wasmFS$JSMemoryFiles[file].length : 0;
       },
     };

src/modules.js

@@ -96,6 +96,7 @@ global.LibraryManager = {
     } else if (WASMFS) {
       libraries.push('library_wasmfs.js');
       libraries.push('library_wasmfs_js_file.js');
+      libraries.push('library_wasmfs_fetch.js');
     }
 
     // Additional JS libraries (without AUTO_JS_LIBRARIES, link to these explicitly via -lxxx.js)

src/parseTools.js

@@ -1073,6 +1073,19 @@ function receiveI64ParamAsI32s(name) {
   return '';
 }
 
+// TODO: use this in library_wasi.js and other places. but we need to add an
+//       error-handling hook here.
+function receiveI64ParamAsDouble(name) {
+  if (WASM_BIGINT) {
+    // Just convert the bigint into a double.
+    return `${name} = Number(${name});`;
+  }
+
+  // Combine the i32 params. Use an unsigned operator on low and shift high by
+  // 32 bits.
+  return `${name} = ${name}_high * 0x100000000 + (${name}_low >>> 0);`;
+}
+
 function sendI64Argument(low, high) {
   if (WASM_BIGINT) {
     return 'BigInt(low) | (BigInt(high) << BigInt(32))';

src/struct_info_internal.json

@@ -47,5 +47,14 @@
               "name"
             ]
         }
+    },
+    {
+        "file": "async_callback.h",
+        "structs": {
+            "CallbackState": [
+                "result",
+                "offset"
+            ]
+        }
     }
 ]

system/include/emscripten/wasmfs.h

@@ -40,6 +40,8 @@ typedef backend_t (*backend_constructor_t)(void*);
 backend_t wasmfs_create_proxied_backend(backend_constructor_t create_backend,
                                         void* arg);
 
+backend_t wasmfs_create_fetch_backend(char* base_url);
+
 #ifdef __cplusplus
 }
 #endif

system/lib/wasmfs/async_callback.h

@@ -0,0 +1,27 @@
+// Copyright 2022 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+
+// This file defines the JS file backend and JS file of the new file system.
+// Current Status: Work in Progress.
+// See https://github.com/emscripten-core/emscripten/issues/15041.
+
+#pragma once
+
+#include "sys/types.h"
+#include "wasi/api.h"
+
+// Callbacks for the async API between C and JS. This is declared in a small
+// separate header for convenience of gen_struct_info.
+
+// Callbacks take a pointer to a CallbackState structure, which contains both
+// the function to call to resume execution, and storage for any out params.
+// Basically this stores the state during an async call.
+struct CallbackState {
+  // The result of the operation, either success or an error code.
+  __wasi_errno_t result;
+
+  // Some syscalls return an offset.
+  off_t offset;
+};

system/lib/wasmfs/fetch_backend.cpp

@@ -0,0 +1,27 @@
+// Copyright 2021 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+
+// This file defines the JS file backend and JS file of the new file system.
+// See https://github.com/emscripten-core/emscripten/issues/15041.
+
+#include "backend.h"
+#include "proxied_async_js_impl_backend.h"
+#include "wasmfs.h"
+
+// See library_wasmfs_fetch.js
+
+extern "C" {
+void _wasmfs_create_fetch_backend_js(wasmfs::backend_t);
+}
+
+namespace wasmfs {
+
+extern "C" backend_t wasmfs_create_fetch_backend(char* base_url) {
+  // TODO: use base url, cache on JS side
+  return wasmFS.addBackend(std::make_unique<ProxiedAsyncJSBackend>(
+    [](backend_t backend) { _wasmfs_create_fetch_backend_js(backend); }));
+}
+
+} // namespace wasmfs

system/lib/wasmfs/js_impl_backend.h

@@ -28,14 +28,16 @@
 // To write a new backend in JS, you basically do the following:
 //
 //  1. Add a declaration of the C function to create the backend in the
-//     "backend creation" section of emscripten/wasmfs.h.
+//     "backend creation" section of emscripten/wasmfs.h. (One line.)
 //  2. Add a cpp file for the new backend, and implement the C function from 1,
 //     which should create it on both the C++ (using JSImplBackend) and JS
 //     sides. (By convention, the C function should just call into C++ and JS
-//     which do the interesting work; the C is just a thin wrapper.)
+//     which do the interesting work; the C is just a thin wrapper.) (A few
+//     lines.)
 // 3. Write a new JS library, and add the implementation of the JS method just
 //    mentioned, which should set up the mapping from the C++ backend object's
-//    address to the JS code containing the hooks to read and write etc.
+//    address to the JS code containing the hooks to read and write etc. (99%
+//    of the work happens here.)
 //
 // For a simple example, see js_file_backend.cpp and library_wasmfs_js_file.js
 //
@@ -96,9 +98,7 @@ class JSImplFile : public DataFile {
     _wasmfs_jsimpl_alloc_file(getBackendIndex(), getFileIndex());
   }
 
-  ~JSImplFile() {
-    _wasmfs_jsimpl_free_file(getBackendIndex(), getFileIndex());
-  }
+  ~JSImplFile() { _wasmfs_jsimpl_free_file(getBackendIndex(), getFileIndex()); }
 };
 
 class JSImplBackend : public Backend {