From 161416832fd4ea3eb49f670f6a06d47d63f0bb8e Mon Sep 17 00:00:00 2001 From: Peter Hoddie Date: Sun, 20 Aug 2023 14:12:20 -0700 Subject: [PATCH] worker configures VM using manifest's creation object - consistent and more powerful #1195 --- build/devices/esp/main.cpp | 6 +- build/devices/esp32/xsProj-esp32/main/main.c | 4 +- .../devices/esp32/xsProj-esp32c3/main/main.c | 2 +- .../devices/esp32/xsProj-esp32s2/main/main.c | 2 +- .../devices/esp32/xsProj-esp32s3/main/main.c | 2 +- build/devices/nrf52/base/xsmain.c | 4 +- build/devices/pico/base/xsmain.c | 2 +- documentation/base/worker.md | 36 +++++--- examples/base/worker/main.js | 19 ++++- modules/base/worker/modWorker.c | 82 +++++++++++++++---- tests/modules/base/worker/constructor.js | 20 +++-- .../base/worker/constructor_deprecated.js | 26 ++++++ tests/modules/base/worker/messages.js | 17 +++- tests/modules/base/worker/terminate.js | 17 +++- xs/platforms/mc/xsHosts.c | 29 ++----- xs/platforms/mc/xsHosts.h | 4 +- 16 files changed, 197 insertions(+), 75 deletions(-) create mode 100644 tests/modules/base/worker/constructor_deprecated.js diff --git a/build/devices/esp/main.cpp b/build/devices/esp/main.cpp index 7161492024..a48acd2c43 100644 --- a/build/devices/esp/main.cpp +++ b/build/devices/esp/main.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Moddable Tech, Inc. + * Copyright (c) 2016-2023 Moddable Tech, Inc. * * This file is part of the Moddable SDK Runtime. * @@ -81,7 +81,7 @@ void setup() wifi_set_opmode_current(NULL_MODE); #ifdef mxDebug - gThe = modCloneMachine(0, 0, 0, 0, NULL); + gThe = modCloneMachine(NULL, NULL); if (!gThe) { modLog("can't clone: no memory?"); while (true) @@ -90,7 +90,7 @@ void setup() modRunMachineSetup(gThe); #else - modRunMachineSetup(modCloneMachine(0, 0, 0, 0, NULL)); + modRunMachineSetup(modCloneMachine(NULL, NULL)); #endif } diff --git a/build/devices/esp32/xsProj-esp32/main/main.c b/build/devices/esp32/xsProj-esp32/main/main.c index 7bc925f2c6..7def8b1647 100644 --- a/build/devices/esp32/xsProj-esp32/main/main.c +++ b/build/devices/esp32/xsProj-esp32/main/main.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Moddable Tech, Inc. + * Copyright (c) 2016-2023 Moddable Tech, Inc. * * This file is part of the Moddable SDK Runtime. * @@ -123,7 +123,7 @@ void loop_task(void *pvParameter) #endif while (true) { - gThe = modCloneMachine(0, 0, 0, 0, NULL); + gThe = modCloneMachine(NULL, NULL); modRunMachineSetup(gThe); diff --git a/build/devices/esp32/xsProj-esp32c3/main/main.c b/build/devices/esp32/xsProj-esp32c3/main/main.c index 4482e8b61b..db5820ce7c 100644 --- a/build/devices/esp32/xsProj-esp32c3/main/main.c +++ b/build/devices/esp32/xsProj-esp32c3/main/main.c @@ -142,7 +142,7 @@ void loop_task(void *pvParameter) #endif while (true) { - gThe = modCloneMachine(0, 0, 0, 0, NULL); + gThe = modCloneMachine(NULL, NULL); modRunMachineSetup(gThe); diff --git a/build/devices/esp32/xsProj-esp32s2/main/main.c b/build/devices/esp32/xsProj-esp32s2/main/main.c index dca31f8a4f..afe3c34309 100644 --- a/build/devices/esp32/xsProj-esp32s2/main/main.c +++ b/build/devices/esp32/xsProj-esp32s2/main/main.c @@ -250,7 +250,7 @@ void loop_task(void *pvParameter) #endif while (true) { - gThe = modCloneMachine(0, 0, 0, 0, NULL); + gThe = modCloneMachine(NULL, NULL); modRunMachineSetup(gThe); diff --git a/build/devices/esp32/xsProj-esp32s3/main/main.c b/build/devices/esp32/xsProj-esp32s3/main/main.c index e617b8f185..2b5f671b75 100644 --- a/build/devices/esp32/xsProj-esp32s3/main/main.c +++ b/build/devices/esp32/xsProj-esp32s3/main/main.c @@ -227,7 +227,7 @@ void loop_task(void *pvParameter) #endif while (true) { - gThe = modCloneMachine(0, 0, 0, 0, NULL); + gThe = modCloneMachine(NULL, NULL); modRunMachineSetup(gThe); diff --git a/build/devices/nrf52/base/xsmain.c b/build/devices/nrf52/base/xsmain.c index c77382084a..13724c7a16 100644 --- a/build/devices/nrf52/base/xsmain.c +++ b/build/devices/nrf52/base/xsmain.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Moddable Tech, Inc. + * Copyright (c) 2016-2023 Moddable Tech, Inc. * * This file is part of the Moddable SDK Runtime. * @@ -68,7 +68,7 @@ void loop_task(void *pvParameter) #if MODDEF_XS_TEST while (true) { - gThe = modCloneMachine(0, 0, 0, 0, NULL); + gThe = modCloneMachine(NULL, NULL); modRunMachineSetup(gThe); diff --git a/build/devices/pico/base/xsmain.c b/build/devices/pico/base/xsmain.c index cc98a6c960..1e650f7011 100644 --- a/build/devices/pico/base/xsmain.c +++ b/build/devices/pico/base/xsmain.c @@ -50,7 +50,7 @@ void xs_setup(void) #endif while (true) { - gThe = modCloneMachine(0, 0, 0, 0, NULL); + gThe = modCloneMachine(NULL, NULL); modRunMachineSetup(gThe); diff --git a/documentation/base/worker.md b/documentation/base/worker.md index 59c7b6c228..64bfb85dc2 100644 --- a/documentation/base/worker.md +++ b/documentation/base/worker.md @@ -1,6 +1,6 @@ # Worker Copyright 2018-2023 Moddable Tech, Inc.
-Revised: July 6, 2023 +Revised: August 20, 2023 The Moddable runtime integrates with XS to allow a multiple virtual machines to co-exist on a single microcontroller. The majority of projects use only a single virtual machine. However, there are situations where the several independent runtime contexts provided by having several virtual machines is advantageous. This isolation is useful to fully separate a particular set of scripts, for example user installed modules, from the core project functionality for security, privacy, and reliability reasons. Another useful situation is to allow scripts to perform blocking operations in one virtual machine while scripts in another virtual machine remain fully responsive. On microcontrollers with multiple CPU cores, workers may execute in parallel to take full advantage of the available CPU power. @@ -33,10 +33,18 @@ To launch a worker, create an instance of the `Worker` class, passing the name o The call to the `Worker` constructor returns only after execution of the specified module completes. If the worker module generates an exception during this step, an exception is propagated so that the call to `new Worker` throws an exception. This behavior means that the invoking virtual machine blocks until the new worker virtual machine has completed initialization. Consequently, any operations performed in a newly instantiated virtual machine should be relatively brief. ### Launching a worker with memory configuration -The previous example launches the worker with the default memory configuration. This may not be large enough for the worker, or may allocate more RAM than needed by the worker. An optional configuration object allows the script instantiating a new virtual machine to set the memory use. - - let aWorker = new Worker("simpleworker", - {allocation: 8192, stackCount: 64, slotCount: 64}); +The previous example launches the worker with the default memory creation configuration used for the main virtual machine. This may not be large enough for the worker, or may allocate more RAM than needed by the worker. An optional configuration object allows the script instantiating a new virtual machine to set the memory use. + +```js +let aWorker = new Worker("simpleworker", { + static: 8192, + stack: 64, + heap: { + initial: 64, + incremental: 32 + } +}); +``` ### Sending a message to a worker Messages to workers are JavaScript objects and binary data. @@ -98,12 +106,20 @@ The `Worker` constructor takes a path to the module used to initialize the new w let aWorker = new Worker("simpleworker"); -An optional dictionary contains configuration properties for the new worker. If the dictionary is not provided, the default parameters are used. These defaults vary by host runtime, so it is recommended to always provide a memory configuration. - - let aWorker = new Worker("simpleworker", - {allocation: 8192, stackCount: 64, slotCount: 64}); +An optional dictionary contains creation properties for the new worker. If the dictionary is not provided, the default parameters are used. These defaults vary by host runtime, so it is recommended to always provide a memory configuration. The creation properties are the same as the `creation` section of a manifest. See the [manifest documentation](../tools/manifest.md#creation) for details. + +```js +let aWorker = new Worker("simpleworker", { + static: 8192, + stack: 64, + heap: { + initial: 64, + incremental: 32 + } +}); +``` -The `allocation` property is the total memory shared by the new virtual machine for its chunk heap, slot heap, and stack. It is allocated when the virtual machine is created. The `stackCount` property is the number of slot entries on the virtual machine's stack. The `slotCount` property is the initial number of slots in the virtual machine's slot heap. +> **Note**: An earlier implementation of `Worker` used different properties to configure the memory creation. These have been deprecated and are no longer included in the documentation. It is recommended to update scripts to use the new format. If an error occurs or an exception is thrown during execution of the module, the `Worker` constructor also throws an error. diff --git a/examples/base/worker/main.js b/examples/base/worker/main.js index 8291fca463..bc08361dab 100644 --- a/examples/base/worker/main.js +++ b/examples/base/worker/main.js @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2017 Moddable Tech, Inc. + * Copyright (c) 2016-2023 Moddable Tech, Inc. * * This file is part of the Moddable SDK. * @@ -19,15 +19,26 @@ trace("hello\n"); let index = 0 function start() { - // note: allocation and stackCount are very small - most real workers will require a larger allocation and stackCount - let aWorker = new Worker("simpleworker", {allocation: 6 * 1024, stackCount: 64, slotCount: 32}); + // note: chunk, heap (slots), and stack are very small - most real-world workers will require different settings + const aWorker = new Worker("simpleworker", { + static: 6 * 1024, + chunk: { + initial: 1536, + incremental: 256 + }, + heap: { + initial: 64, + incremental: 32 + }, + stack: 64 + }); aWorker.postMessage({hello: "world", index: ++index}); aWorker.postMessage("hello, again"); aWorker.postMessage([1, 2, 3]); aWorker.onmessage = function(message) { - if (3 == message.counter) { + if (3 === message.counter) { trace(`start worker ${index}\n`); aWorker.terminate(); start(); diff --git a/modules/base/worker/modWorker.c b/modules/base/worker/modWorker.c index f770909ce1..4709d04235 100644 --- a/modules/base/worker/modWorker.c +++ b/modules/base/worker/modWorker.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Moddable Tech, Inc. + * Copyright (c) 2016-2023 Moddable Tech, Inc. * * This file is part of the Moddable SDK Runtime. * @@ -51,10 +51,7 @@ struct modWorkerRecord { xsSlot owner; xsSlot ownerPort; xsSlot workerPort; - uint32_t allocation; - uint32_t stackCount; - uint32_t slotCount; - uint32_t keyCount; + xsCreation creation; xsBooleanValue closing; xsBooleanValue shared; #ifdef INC_FREERTOS_H @@ -136,6 +133,16 @@ void xs_worker_destructor(void *data) } } +static void getIntegerProperty(xsMachine *the, xsSlot *slot, xsIdentifier id, xsIntegerValue *value) +{ + if (!xsmcHas(*slot, id)) + return; + + xsSlot tmp; + xsmcGet(tmp, *slot, id); + *value = xsmcToInteger(tmp); +} + static void workerConstructor(xsMachine *the, xsBooleanValue shared) { modCriticalSectionDeclare; @@ -190,18 +197,65 @@ static void workerConstructor(xsMachine *the, xsBooleanValue shared) } } + xsCreation *creationP; + (void *)xsPreparationAndCreation(&creationP); + worker->creation = *creationP; if (xsmcArgc > 1) { - xsmcGet(xsVar(0), xsArg(1), xsID_allocation); - worker->allocation = xsmcToInteger(xsVar(0)); + if (xsmcHas(xsArg(1), xsID_allocation) || + xsmcHas(xsArg(1), xsID_stackCount) || + xsmcHas(xsArg(1), xsID_slotCount) || + xsmcHas(xsArg(1), xsID_keyCount)) { +#ifdef mxDebug + xsTrace("deprecated worker creation parameters detected. update recommended."); +#endif + xsIntegerValue allocation, stackCount, slotCount, keyCount; + xsmcGet(xsVar(0), xsArg(1), xsID_allocation); + allocation = xsmcToInteger(xsVar(0)); + + xsmcGet(xsVar(0), xsArg(1), xsID_stackCount); + stackCount = xsmcToInteger(xsVar(0)); + + xsmcGet(xsVar(0), xsArg(1), xsID_slotCount); + slotCount = xsmcToInteger(xsVar(0)); - xsmcGet(xsVar(0), xsArg(1), xsID_stackCount); - worker->stackCount = xsmcToInteger(xsVar(0)); + xsmcGet(xsVar(0), xsArg(1), xsID_keyCount); + keyCount = xsmcToInteger(xsVar(0)); - xsmcGet(xsVar(0), xsArg(1), xsID_slotCount); - worker->slotCount = xsmcToInteger(xsVar(0)); + if (allocation) + worker->creation.staticSize = allocation; - xsmcGet(xsVar(0), xsArg(1), xsID_keyCount); - worker->keyCount = xsmcToInteger(xsVar(0)); + if (stackCount) + worker->creation.stackCount = stackCount; + + if (slotCount) + worker->creation.initialHeapCount = slotCount; + + if (keyCount) + worker->creation.initialKeyCount = keyCount; + } + else { + xsIntegerValue value; + + getIntegerProperty(the, &xsArg(1), xsID_static, &worker->creation.staticSize); + xsmcGet(xsVar(0), xsArg(1), xsID_chunk); + if (xsmcTest(xsVar(0))) { + getIntegerProperty(the, &xsVar(0), xsID_initial, &worker->creation.initialChunkSize); + getIntegerProperty(the, &xsVar(0), xsID_incremental, &worker->creation.incrementalChunkSize); + } + xsmcGet(xsVar(0), xsArg(1), xsID_heap); + if (xsmcTest(xsVar(0))) { + getIntegerProperty(the, &xsVar(0), xsID_initial, &worker->creation.initialHeapCount); + getIntegerProperty(the, &xsVar(0), xsID_incremental, &worker->creation.incrementalHeapCount); + } + getIntegerProperty(the, &xsArg(1), xsID_stack, &worker->creation.stackCount); + xsmcGet(xsVar(0), xsArg(1), xsID_keys); + if (xsmcTest(xsVar(0))) { + getIntegerProperty(the, &xsVar(0), xsID_initial, &worker->creation.initialKeyCount); + getIntegerProperty(the, &xsVar(0), xsID_incremental, &worker->creation.incrementalKeyCount); + getIntegerProperty(the, &xsVar(0), xsID_name, &worker->creation.nameModulo); + getIntegerProperty(the, &xsVar(0), xsID_symbol, &worker->creation.symbolModulo); + } + } } #ifdef INC_FREERTOS_H @@ -367,7 +421,7 @@ int workerStart(modWorker worker) xsMachine *the; int result = 0; - the = modCloneMachine(worker->allocation, worker->stackCount, worker->slotCount, worker->keyCount, worker->module); + the = modCloneMachine(&worker->creation, worker->module); if (!the) return -1; diff --git a/tests/modules/base/worker/constructor.js b/tests/modules/base/worker/constructor.js index 184403772c..2e9273415d 100644 --- a/tests/modules/base/worker/constructor.js +++ b/tests/modules/base/worker/constructor.js @@ -6,10 +6,19 @@ flags: [module] import Worker from "worker"; const minimumOptions = { - allocation: 8192, - stackCount: 64, - slotCount: 64, - keyCount: 7 + static: 8192, + heap: { + initial: 64, + incremental: 64 + }, + chunk: { + initial: 1024, + incremental: 512 + }, + stack: 64, + keys: { + initial: 1 + } }; assert.throws(SyntaxError, () => new Worker, "Worker constructor requires 1 argument"); @@ -19,7 +28,8 @@ assert.throws(TypeError, () => Worker("testworker"), "Worker constructor called assert.throws(Error, () => new Worker("invalid module specifier", minimumOptions), "Worker constructor with invalid module specifier"); assert.throws(Error, () => new Worker(12, minimumOptions), "Worker constructor with invalid module specifier"); -assert.throws(Error, () => new Worker("testworker", {...minimumOptions, allocation: 1024 * 1024 * 1024}), "Worker with 1 GB alocation should fail on microcontroller"); +assert.throws(Error, () => new Worker("testworker", {allocation: 1024 * 1024 * 1024, stackCount: 64, slotCount: 64, keyCount: 7}), "Worker with 1 GB alocation should fail on microcontroller (deprecated property names)"); +assert.throws(Error, () => new Worker("testworker", {...minimumOptions, static: 1024 * 1024 * 1024}), "Worker with 1 GB alocation should fail on microcontroller"); assert.throws(Error, () => new Worker("testthrowworker", minimumOptions), "Worker module throws"); new Worker("testworker", minimumOptions) diff --git a/tests/modules/base/worker/constructor_deprecated.js b/tests/modules/base/worker/constructor_deprecated.js new file mode 100644 index 0000000000..184403772c --- /dev/null +++ b/tests/modules/base/worker/constructor_deprecated.js @@ -0,0 +1,26 @@ +/*--- +description: +flags: [module] +---*/ + +import Worker from "worker"; + +const minimumOptions = { + allocation: 8192, + stackCount: 64, + slotCount: 64, + keyCount: 7 +}; + +assert.throws(SyntaxError, () => new Worker, "Worker constructor requires 1 argument"); +assert.throws(TypeError, () => new Worker(Symbol()), "Worker constructor rejects Symbol"); +assert.throws(TypeError, () => Worker("testworker"), "Worker constructor called as function"); + +assert.throws(Error, () => new Worker("invalid module specifier", minimumOptions), "Worker constructor with invalid module specifier"); +assert.throws(Error, () => new Worker(12, minimumOptions), "Worker constructor with invalid module specifier"); + +assert.throws(Error, () => new Worker("testworker", {...minimumOptions, allocation: 1024 * 1024 * 1024}), "Worker with 1 GB alocation should fail on microcontroller"); +assert.throws(Error, () => new Worker("testthrowworker", minimumOptions), "Worker module throws"); + +new Worker("testworker", minimumOptions) + diff --git a/tests/modules/base/worker/messages.js b/tests/modules/base/worker/messages.js index b4922be0c8..6f3bb109d7 100644 --- a/tests/modules/base/worker/messages.js +++ b/tests/modules/base/worker/messages.js @@ -6,10 +6,19 @@ flags: [module, async] import Worker from "worker"; const minimumOptions = { - allocation: 8192, - stackCount: 64, - slotCount: 64, - keyCount: 7 + static: 8192, + heap: { + initial: 64, + incremental: 64 + }, + chunk: { + initial: 1024, + incremental: 512 + }, + stack: 64, + keys: { + initial: 1 + } }; const messages = [ diff --git a/tests/modules/base/worker/terminate.js b/tests/modules/base/worker/terminate.js index 2c492495df..8d7cc4d2ff 100644 --- a/tests/modules/base/worker/terminate.js +++ b/tests/modules/base/worker/terminate.js @@ -6,10 +6,19 @@ flags: [module] import Worker from "worker"; const minimumOptions = { - allocation: 8192, - stackCount: 64, - slotCount: 64, - keyCount: 7 + static: 8192, + heap: { + initial: 64, + incremental: 64 + }, + chunk: { + initial: 1024, + incremental: 512 + }, + stack: 64, + keys: { + initial: 1 + } }; let worker = new Worker("testworker", minimumOptions); diff --git a/xs/platforms/mc/xsHosts.c b/xs/platforms/mc/xsHosts.c index 48f9adf635..880b737fcd 100644 --- a/xs/platforms/mc/xsHosts.c +++ b/xs/platforms/mc/xsHosts.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Moddable Tech, Inc. + * Copyright (c) 2016-2023 Moddable Tech, Inc. * * This file is part of the Moddable SDK Runtime. * @@ -450,11 +450,11 @@ char *findNthAtom(uint32_t atomTypeIn, int index, const uint8_t *xsb, int xsbSiz create VM */ -txMachine *modCloneMachine(uint32_t allocation, uint32_t stackCount, uint32_t slotCount, uint32_t keyCount, const char *name) +txMachine *modCloneMachine(xsCreation *creationIn, const char *name) { txMachine *the; - xsCreation *creationP; - void *preparation = xsPreparationAndCreation(&creationP); + xsCreation *creation = creationIn; + void *preparation = xsPreparationAndCreation(creation ? NULL : &creation); #if MODDEF_XS_MODS uint8_t modStatus = 0; @@ -467,30 +467,17 @@ txMachine *modCloneMachine(uint32_t allocation, uint32_t stackCount, uint32_t sl if (!name) name = ((txPreparation *)preparation)->main; - if (0 == allocation) - allocation = creationP->staticSize; - - if (allocation) { - xsCreation creation = *creationP; + if (creation->staticSize) { uint8_t *context[2]; - if (stackCount) - creation.stackCount = stackCount; - - if (slotCount) - creation.initialHeapCount = slotCount; - - if (keyCount) - creation.initialKeyCount = keyCount; - - context[0] = c_malloc(allocation); + context[0] = c_malloc(creation->staticSize); if (NULL == context[0]) { modLog("failed to allocate xs block"); return NULL; } - context[1] = context[0] + allocation; + context[1] = context[0] + creation->staticSize; - the = xsPrepareMachine(&creation, preparation, (char *)name, context, archive); + the = xsPrepareMachine(creation, preparation, (char *)name, context, archive); if (NULL == the) { if (context[0]) c_free(context[0]); diff --git a/xs/platforms/mc/xsHosts.h b/xs/platforms/mc/xsHosts.h index 259a305496..4a12233c86 100644 --- a/xs/platforms/mc/xsHosts.h +++ b/xs/platforms/mc/xsHosts.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2022 Moddable Tech, Inc. + * Copyright (c) 2016-2023 Moddable Tech, Inc. * * This file is part of the Moddable SDK Runtime. * @@ -29,7 +29,7 @@ extern "C" { #include "modTimer.h" #endif -extern xsMachine *modCloneMachine(uint32_t allocation, uint32_t stack, uint32_t slotCount, uint32_t keyCount, const char *name); +extern xsMachine *modCloneMachine(xsCreation *creation, const char *name); extern void modRunMachineSetup(xsMachine *the); extern char *modGetModAtom(xsMachine *the, uint32_t atomTypeIn, int *atomSizeOut);