From 88fc18991c939896eeb0d8489de1279dce469554 Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Sun, 6 Oct 2019 18:48:47 -0700 Subject: [PATCH 1/9] Undo assertion on STANDALONE_WASM not working with memory growth: While it isn't 100% standalone (there is an import to grow the memory), it is usable for people that implement that import, so don't break them. fixes #9587 --- emcc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/emcc.py b/emcc.py index d28b168c82fef..985c346e62966 100755 --- a/emcc.py +++ b/emcc.py @@ -1818,8 +1818,6 @@ def check_human_readable_list(items): exit_with_error('STANDALONE_WASM does not support pthreads yet') if shared.Settings.SIMD: exit_with_error('STANDALONE_WASM does not support simd yet') - if shared.Settings.ALLOW_MEMORY_GROWTH: - exit_with_error('STANDALONE_WASM does not support memory growth yet') # the wasm must be runnable without the JS, so there cannot be anything that # requires JS legalization shared.Settings.LEGALIZE_JS_FFI = 0 From daf219dce6b263c7e0c54d124934032b04e4f135 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 7 Oct 2019 13:19:19 -0700 Subject: [PATCH 2/9] wip [ci skip] --- src/library.js | 12 ++++++++++++ src/preamble.js | 3 ++- system/lib/standalone_wasm.c | 17 +++++++++++++---- tests/test_core.py | 22 ++++++++++++++++++++++ 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/library.js b/src/library.js index d9947563a9dda..3824a674d2e65 100644 --- a/src/library.js +++ b/src/library.js @@ -642,6 +642,18 @@ LibraryManager.library = { #endif // ALLOW_MEMORY_GROWTH }, + // Called after wasm grows memory. At that time we need to update the views. + // Without this notification, we'd need to check the buffer in JS every time + // we return from any wasm, which adds overhead. See + // https://github.com/WebAssembly/WASI/issues/82 + emscripten_notify_memory_growth: function(memoryIndex) { +#if ASSERTIONS + assert(memoryIndex == 0); +#endif +console.log("Gorw!"); + updateGlobalBufferAndViews(wasmMemory.buffer); + }, + system__deps: ['__setErrNo'], system: function(command) { // int system(const char *command); diff --git a/src/preamble.js b/src/preamble.js index fdd4c258df9ff..4985d327762fd 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -925,7 +925,8 @@ function createWasm() { // In pure wasm mode the memory is created in the wasm (not imported), and // then exported. // TODO: do not create a Memory earlier in JS - updateGlobalBufferAndViews(exports['memory'].buffer); + wasmMemory = exports['memory']; + updateGlobalBufferAndViews(wasmMemory.buffer); #if ASSERTIONS writeStackCookie(); #endif diff --git a/system/lib/standalone_wasm.c b/system/lib/standalone_wasm.c index d9dc659792875..ebf42824db788 100644 --- a/system/lib/standalone_wasm.c +++ b/system/lib/standalone_wasm.c @@ -5,6 +5,7 @@ * found in the LICENSE file. */ +#include #include #include #include @@ -70,9 +71,17 @@ void *emscripten_memcpy_big(void *restrict dest, const void *restrict src, size_ static const int WASM_PAGE_SIZE = 65536; -// Note that this does not support memory growth in JS because we don't update the JS -// heaps. Wasm and wasi lack a good API for that. +extern void emscripten_notify_memory_growth(size_t memory_index); + int emscripten_resize_heap(size_t size) { - size_t result = __builtin_wasm_memory_grow(0, (size + WASM_PAGE_SIZE - 1) / WASM_PAGE_SIZE); - return result != (size_t)-1; + size_t old_size = __builtin_wasm_memory_size(0) * WASM_PAGE_SIZE; + assert(old_size < size); + ssize_t diff = (size - old_size + WASM_PAGE_SIZE - 1) / WASM_PAGE_SIZE; + size_t result = __builtin_wasm_memory_grow(0, diff); + if (result != (size_t)-1) { + // Success, update JS (see https://github.com/WebAssembly/WASI/issues/82) + emscripten_notify_memory_growth(0); + return 1; + } + return 0; } diff --git a/tests/test_core.py b/tests/test_core.py index ec328dfda061f..cf30c3353f1a6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -127,6 +127,27 @@ def decorated(self): return decorated +# Similar to also_with_standalone_wasm, but suitable for tests that cannot +# run in a wasm VM yet, as they are not 100% standalone. We can still +# run them with the JS code though. +def also_with_impure_standalone_wasm(func): + def decorated(self): + func(self) + # Standalone mode is only supported in the wasm backend, and not in all + # modes there. + if self.is_wasm_backend() and self.get_setting('WASM') and not self.get_setting('SAFE_STACK'): + print('standalone') + self.set_setting('STANDALONE_WASM', 1) + wasm_engines = shared.WASM_ENGINES + try: + shared.WASM_ENGINES = [] + func(self) + finally: + shared.WASM_ENGINES = wasm_engines + + return decorated + + # A simple check whether the compiler arguments cause optimization. def is_optimizing(args): return '-O' in str(args) and '-O0' not in args @@ -1928,6 +1949,7 @@ def test_memorygrowth_3(self): self.emcc_args += ['-s', 'ALLOW_MEMORY_GROWTH=0', '-s', 'ABORTING_MALLOC=0', '-s', 'SAFE_HEAP'] self.do_run_in_out_file_test('tests', 'core', 'test_memorygrowth_3') + @also_with_impure_standalone_wasm def test_memorygrowth_wasm_mem_max(self): if self.has_changed_setting('ALLOW_MEMORY_GROWTH'): self.skipTest('test needs to modify memory growth') From d7a78dc784c75b8e6d15bdedd9dcb0a914a9f038 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 7 Oct 2019 13:27:29 -0700 Subject: [PATCH 3/9] more --- system/lib/standalone_wasm.c | 5 ++++- tools/system_libs.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/system/lib/standalone_wasm.c b/system/lib/standalone_wasm.c index ebf42824db788..e456d08dc5373 100644 --- a/system/lib/standalone_wasm.c +++ b/system/lib/standalone_wasm.c @@ -74,14 +74,17 @@ static const int WASM_PAGE_SIZE = 65536; extern void emscripten_notify_memory_growth(size_t memory_index); int emscripten_resize_heap(size_t size) { +#ifdef __EMSCRIPTEN_MEMORY_GROWTH__ size_t old_size = __builtin_wasm_memory_size(0) * WASM_PAGE_SIZE; assert(old_size < size); ssize_t diff = (size - old_size + WASM_PAGE_SIZE - 1) / WASM_PAGE_SIZE; size_t result = __builtin_wasm_memory_grow(0, diff); if (result != (size_t)-1) { - // Success, update JS (see https://github.com/WebAssembly/WASI/issues/82) + + // Success, update JS (see https://github.com/WebAssembly/WASI/issues/82) emscripten_notify_memory_growth(0); return 1; } +#endif return 0; } diff --git a/tools/system_libs.py b/tools/system_libs.py index b0e73289585fd..81d8028fcd58c 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1185,6 +1185,33 @@ class libstandalonewasm(MuslInternalLibrary): cflags = ['-Os'] src_dir = ['system', 'lib'] + def __init__(self, **kwargs): + self.is_mem_grow = kwargs.pop('is_mem_grow') + super(libstandalonewasm, self).__init__(**kwargs) + + def get_base_name(self): + name = super(libstandalonewasm, self).get_base_name() + if self.is_mem_grow: + name += '-mem_grow' + return name + + def get_cflags(self): + cflags = super(libstandalonewasm, self).get_cflags() + if self.is_mem_grow: + cflags += ['-D__EMSCRIPTEN_MEMORY_GROWTH__=1'] + return cflags + + @classmethod + def vary_on(cls): + return super(libstandalonewasm, cls).vary_on() + ['is_mem_grow'] + + @classmethod + def get_default_variation(cls, **kwargs): + return super(libstandalonewasm, cls).get_default_variation( + is_mem_grow=shared.Settings.ALLOW_MEMORY_GROWTH, + **kwargs + ) + def get_files(self): base_files = files_in_path( path_components=['system', 'lib'], From d14db1e3034cc1d224eb5a0de73b373bf183c330 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 7 Oct 2019 13:28:01 -0700 Subject: [PATCH 4/9] more --- src/library.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/library.js b/src/library.js index 3824a674d2e65..f86ebdf03a086 100644 --- a/src/library.js +++ b/src/library.js @@ -650,7 +650,6 @@ LibraryManager.library = { #if ASSERTIONS assert(memoryIndex == 0); #endif -console.log("Gorw!"); updateGlobalBufferAndViews(wasmMemory.buffer); }, From 2cbf2bc12f089b0b432df24dc4cdca465b08f9c0 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 7 Oct 2019 13:29:30 -0700 Subject: [PATCH 5/9] nicer --- tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index cf30c3353f1a6..737c2e5b2f3cd 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -136,7 +136,7 @@ def decorated(self): # Standalone mode is only supported in the wasm backend, and not in all # modes there. if self.is_wasm_backend() and self.get_setting('WASM') and not self.get_setting('SAFE_STACK'): - print('standalone') + print('standalone (impure; no wasm runtimes)') self.set_setting('STANDALONE_WASM', 1) wasm_engines = shared.WASM_ENGINES try: From 09979eb5fe0b9c04100348fec79c43fc07ba8da7 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 7 Oct 2019 14:17:07 -0700 Subject: [PATCH 6/9] test update --- tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent | 1 + tests/test_other.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent index 05e48cb7a5f53..9fb9f9870ac48 100644 --- a/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent +++ b/tests/other/metadce/hello_world_fastcomp_O3_MAIN_MODULE.sent @@ -911,6 +911,7 @@ _emscripten_log_js _emscripten_longjmp _emscripten_main_browser_thread_id _emscripten_memcpy_big +_emscripten_notify_memory_growth _emscripten_pause_main_loop _emscripten_pc_get_column _emscripten_pc_get_file diff --git a/tests/test_other.py b/tests/test_other.py index 2c161f2712663..eb43152ff66ff 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8108,7 +8108,7 @@ def test_binaryen_metadce_hello(self, *args): 4, [], [], 8, 0, 0, 0), # noqa; totally empty! # we don't metadce with linkable code! other modules may want stuff # don't compare the # of functions in a main module, which changes a lot - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1602, [], [], 226403, None, 107, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1603, [], [], 226403, None, 107, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 13, [], [], 10017, 13, 9, 20), # noqa }) @no_wasm_backend() From 9f0c1242c175747bd8c071fca6500b65db5fb5a5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Oct 2019 15:08:50 -0700 Subject: [PATCH 7/9] docs + metadata bump --- site/source/docs/api_reference/emscripten.h.rst | 17 +++++++++++++++++ tools/shared.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/site/source/docs/api_reference/emscripten.h.rst b/site/source/docs/api_reference/emscripten.h.rst index 9925137074ad3..9200b388ad1b7 100644 --- a/site/source/docs/api_reference/emscripten.h.rst +++ b/site/source/docs/api_reference/emscripten.h.rst @@ -1363,3 +1363,20 @@ Functions This function requires Asyncify - it relies on that option to spill the local state all the way up the stack. As a result, it will add overhead to your program. + +ABI functions +============= + +The following functions are not declared in ``emscripten.h``, but are used +internally in our system libraries. You may care about them if you replace the +Emscripten runtime JS code, or run Emscripten binaries in your own runtime. + + +.. c:function:: void emscripten_notify_memory_growth(i32 index) + + Called when memory has grown. In a JS runtime, this is used to know when + to update the JS views on the wasm memory, which otherwise we would need + to constantly check for after any wasm code runs. See + `this wasi discussion `_. + + :param i32 index: Which memory has grown. diff --git a/tools/shared.py b/tools/shared.py index 03eb93d347db9..67caee9c213ed 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -543,7 +543,7 @@ def get_emscripten_version(path): # change, increment EMSCRIPTEN_ABI_MINOR if EMSCRIPTEN_ABI_MAJOR == 0 # or the ABI change is backwards compatible, otherwise increment # EMSCRIPTEN_ABI_MAJOR and set EMSCRIPTEN_ABI_MINOR = 0. -(EMSCRIPTEN_ABI_MAJOR, EMSCRIPTEN_ABI_MINOR) = (0, 16) +(EMSCRIPTEN_ABI_MAJOR, EMSCRIPTEN_ABI_MINOR) = (0, 17) def generate_sanity(): From bf43407491c630037134560be7226f7b8f3d8c96 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Oct 2019 15:11:15 -0700 Subject: [PATCH 8/9] test update --- tests/test_other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_other.py b/tests/test_other.py index 4839e9609fda9..f835e32c2e6be 100644 --- a/tests/test_other.py +++ b/tests/test_other.py @@ -8104,7 +8104,7 @@ def test_binaryen_metadce_hello(self, *args): 4, [], [], 8, 0, 0, 0), # noqa; totally empty! # we don't metadce with linkable code! other modules may want stuff # don't compare the # of functions in a main module, which changes a lot - 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1603, [], [], 226403, None, 107, None), # noqa + 'main_module_1': (['-O3', '-s', 'MAIN_MODULE=1'], 1604, [], [], 226403, None, 107, None), # noqa 'main_module_2': (['-O3', '-s', 'MAIN_MODULE=2'], 13, [], [], 10017, 13, 9, 20), # noqa }) @no_wasm_backend() From d145fa0c4cd8783724e00e8ef32bc96de3f71926 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 9 Oct 2019 15:32:05 -0700 Subject: [PATCH 9/9] rename --- tools/system_libs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 81d8028fcd58c..1cf795e148d96 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -1192,7 +1192,7 @@ def __init__(self, **kwargs): def get_base_name(self): name = super(libstandalonewasm, self).get_base_name() if self.is_mem_grow: - name += '-mem_grow' + name += '-memgrow' return name def get_cflags(self):