Profiler/instrumentation support and promise jobs

xsnap/documentation/Compartments.md

@@ -119,9 +119,7 @@ The `moduleMap` parameter is an object that initializes the **module map** of th
 
 The `Compartment` constructor copies the `moduleMap` properties using the same behavior as `Object.assign`.
 
-A compartment cannot acces module descriptors or module specifiers provided by
-the module map. A compartment can only access module namespaces that will be
-loaded and initialized based on such module descriptors or module specifiers.
+A compartment cannot access module records or module specifiers provided by the module map. A compartment can only access module namespaces that will be loaded and initialized based on such module records or module specifiers.
 
 #### module namespaces
 

xsnap/documentation/xsnap-worker.md

@@ -22,7 +22,11 @@ Once started, the process listens on file descriptor 3, and will write to file d
 * `?` (command): feed the body (as a JS `String`) to the registered `handleCommand(body)` handler
   * for both `e` and `?`, execution continues until both the `setImmediate` and the ready-promise-callback queues are empty (the vat is "quiescent")
   * if evaluation/`handleCommand()` throws an error (and the evaluated code does not catch it), the worker writes `!${toString(err)}` to fd4
-  * if successful, the result should be an ArrayBuffer, or an object with a `.result` property that is an ArrayBuffer
+  * while running, if the application calls `globalThis.issueCommand(query)` (where `query` is an ArrayBuffer), the worker will write a netstring to file descriptor 4, whose payload is a single `?` character followed by contents of `query`
+    * the application will then do a blocking read on file descriptor 3 until a complete netstring is received
+    * the payload of this response must start with a single `/` character
+    * the remainder of the payload will be returned (as an ArrayBuffer) to the caller of `issueCommand()`
+  * if successful, the evaluation/`handleCommand()` result should be an ArrayBuffer, or an object with a `.result` property that is an ArrayBuffer
     * anything else will yield an empty response string
   * the worker writes a netstring with the following body to fd4:
     * `.${meterObj}\1${result}`
@@ -44,11 +48,16 @@ Once started, the process listens on file descriptor 3, and will write to file d
   * for both `s` and `m`, an error writes a terse `!` to fd4, and success writes `.${meterObj}\1` (the same success response as for `e`/`?` but with an empty message: just the metering data)
   * both `s` and `m` are holdovers from `xsnap.c`, and should be considered deprecated in `xsnap-worker.c`
 * `w`: the body is treated as a filename. A GC collection is triggered, and then the JS engine state snapshot (the entire virtual machine state: heap, stack, symbol table, etc) is written to the given filename. Then execution continues normally. The response is `!` or `.${meterObj}\1` as with `s`/`m`
-* all other command characters cause the worker to exit
+* `q`: causes the worker to exit gently, with an exit code of `E_SUCCESS` (0)
+* all other command characters cause the worker to exit noisily, with a messge to stderr about the unrecognized command, and an exit code of `E_IO_ERROR` (2)
 
 If at any point the computation exceeds one of the following limits, the process will exit with a non-zero (and non-negative) exit code:
 
 * `E_NOT_ENOUGH_MEMORY` (11): when memory allocation uses more than `allocationLimit` bytes (hard-coded to 2GiB in `xsnapPlatform.c`/`fxCreateMachinePlatform()`)
 * `E_STACK_OVERFLOW` (12): when the JS stack exceeds the configured limit (hard-coded in `xsnap-worker.c` as `stackCount` to 4096). Also, at least for now, when the native stack exceeds a limit.
 * `E_NO_MORE_KEYS` (16): when the number of "keys" (unique property names) exceeds the limit (hard-coded in `xsnap-worker.c` as `keyCount` to 32000)
 * `E_TOO_MUCH_COMPUTATION` (17): when the computation exceeds the `-l` computron limit
+
+The other possible exit codes are:
+* `E_SUCCESS` (0): when a `q` command is received
+* `E_IO_ERROR` (2): when an unrecognized command is received

xsnap/makefiles/lin/xsnap-worker.mk

@@ -31,6 +31,7 @@ C_OPTIONS = \
 	-DmxLockdown=1 \
 	-DmxMetering=1 \
 	-DmxDebug=1 \
+	-UmxInstrument \
 	-DmxNoConsole=1 \
 	-DmxBoundsCheck=1 \
 	-DmxParse=1 \

xsnap/makefiles/lin/xsnap.mk

@@ -25,6 +25,7 @@ C_OPTIONS = \
 	-DmxLockdown=1 \
 	-DmxMetering=1 \
 	-DmxParse=1 \
+	-DmxProfile=1 \
 	-DmxRun=1 \
 	-DmxSloppy=1 \
 	-DmxSnapshot=1 \

xsnap/makefiles/mac/xsnap-worker.mk

@@ -36,6 +36,7 @@ C_OPTIONS = \
 	-DmxLockdown=1 \
 	-DmxMetering=1 \
 	-DmxDebug=1 \
+	-UmxInstrument \
 	-DmxNoConsole=1 \
 	-DmxBoundsCheck=1 \
 	-DmxParse=1 \

xsnap/makefiles/mac/xsnap.mk

@@ -30,6 +30,7 @@ C_OPTIONS = \
 	-DmxLockdown=1 \
 	-DmxMetering=1 \
 	-DmxParse=1 \
+	-DmxProfile=1 \
 	-DmxRun=1 \
 	-DmxSloppy=1 \
 	-DmxSnapshot=1 \

xsnap/sources/xsnap-worker.c

@@ -102,6 +102,19 @@ static int fxSnapshotWrite(void* stream, void* address, size_t size)
 	return (fwrite(address, size, 1, stream) == 1) ? 0 : errno;
 }
 
+#if mxInstrument
+#define xsnapInstrumentCount 1
+static xsStringValue xsnapInstrumentNames[xsnapInstrumentCount] = {
+	"Metering",
+};
+static xsStringValue xsnapInstrumentUnits[xsnapInstrumentCount] = {
+	" times",
+};
+static xsIntegerValue xsnapInstrumentValues[xsnapInstrumentCount] = {
+	0,
+};
+#endif
+
 #if mxMetering
 #define xsBeginCrank(_THE, _LIMIT) \
 	(xsSetCurrentMeter(_THE, 0), \
@@ -340,10 +353,30 @@ int main(int argc, char* argv[])
 		fprintf(stderr, "fdopen(4) to parent failed\n");
 		c_exit(E_IO_ERROR);
 	}
+#if mxInstrument
+	xsDescribeInstrumentation(machine, xsnapInstrumentCount, xsnapInstrumentNames, xsnapInstrumentUnits);
+#endif
 	xsBeginMetering(machine, fxMeteringCallback, interval);
 	{
+		fd_set rfds;
 		char done = 0;
 		while (!done) {
+			#if mxInstrument
+			FD_ZERO(&rfds);
+			FD_SET(3, &rfds);
+			FD_SET(5, &rfds);
+			if (select(6, &rfds, NULL, NULL, NULL) >= 0) {
+				if (FD_ISSET(5, &rfds))
+					xsRunDebugger(machine);
+				if (!FD_ISSET(3, &rfds))
+					continue;
+			}
+			else {
+				fprintf(stderr, "select failed: %s\n", strerror(errno));
+				error = E_IO_ERROR;
+				break;
+			}
+			#endif
 			// By default, use the infinite meter.
 			gxCurrentMeter = 0;
 
@@ -372,6 +405,8 @@ int main(int argc, char* argv[])
 			case '?':
 			case 'e':
 				xsBeginCrank(machine, gxCrankMeteringLimit);
+				char* response = NULL;
+				xsIntegerValue responseLength = 0;
 				error = 0;
 				xsBeginHost(machine);
 				{
@@ -405,12 +440,9 @@ int main(int argc, char* argv[])
 				meterIndex = xsEndCrank(machine);
 				{
 					if (error) {
-						xsStringValue message = xsToString(xsVar(1));
-						writeError = fxWriteNetString(toParent, "!", message, strlen(message));
-						// fprintf(stderr, "error: %d, writeError: %d %s\n", error, writeError, message);
+						response = xsToString(xsVar(1));
+						responseLength = strlen(response);
 					} else {
-						char* response = NULL;
-						xsIntegerValue responseLength = 0;
 						// fprintf(stderr, "report: %d %s\n", xsTypeOf(report), xsToString(report));
 						xsTry {
 							if (xsTypeOf(xsVar(1)) == xsReferenceType && xsHas(xsVar(1), xsID("result"))) {
@@ -433,11 +465,16 @@ int main(int argc, char* argv[])
 								xsException = xsUndefined;
 							}
 						}
-						// fprintf(stderr, "response of %d bytes\n", responseLength);
-						writeError = fxWriteOkay(toParent, meterIndex, the, response, responseLength);
 					}
 				}
 				xsEndHost(machine);
+				if (error) {
+						writeError = fxWriteNetString(toParent, "!", response, responseLength);
+						// fprintf(stderr, "error: %d, writeError: %d %s\n", error, writeError, response);
+				} else {
+						// fprintf(stderr, "response of %d bytes\n", responseLength);
+						writeError = fxWriteOkay(toParent, meterIndex, machine, response, responseLength);
+				}
 				if (writeError != 0) {
 					fprintf(stderr, "%s\n", fxWriteNetStringError(writeError));
 					c_exit(E_IO_ERROR);
@@ -516,11 +553,27 @@ int main(int argc, char* argv[])
 					}
 				}
 				break;
-			default:
+			case 'q':
 				done = 1;
 				break;
+
+			// We reserve some prefix characters to avoid/detect/debug confusion,
+			// all of which are explicitly rejected, just like unknown commands. Do not
+			// reuse these for new commands.
+			case '/': // downstream response to upstream issueCommand()
+			case '.': // upstream good response to downstream execute/eval
+			case '!': // upstream error response to downstream execute/eval
+			default:
+				// note: the nsbuf we receive from fxReadNetString is null-terminated
+				fprintf(stderr, "Unexpected prefix '%c' in command '%s'\n", command, nsbuf);
+				c_exit(E_IO_ERROR);
+				break;
 			}
 			free(nsbuf);
+#if mxInstrument
+			xsnapInstrumentValues[0] = (xsIntegerValue)meterIndex;
+			xsSampleInstrumentation(machine, xsnapInstrumentCount, xsnapInstrumentValues);
+#endif
 		}
 		xsBeginHost(machine);
 		{
@@ -814,8 +867,12 @@ static void xs_issueCommand(xsMachine *the)
 #if XSNAP_TEST_RECORD
 	fxTestRecord(mxTestRecordJSON | mxTestRecordReply, buf, len);
 #endif
-	// TODO: can we avoid a copy?
-	xsResult = xsArrayBuffer(buf, len);
+	char command = *buf;
+	if (len == 0 || command != '/') {
+		xsUnknownError("Received unexpected command reply.");
+	}
+
+	xsResult = xsArrayBuffer(buf + 1, len - 1);
 	free(buf);
 }
 

xsnap/sources/xsnap.c

@@ -97,12 +97,14 @@ int main(int argc, char* argv[])
 {
 	int argi;
 	int argd = 0;
+	int argp = 0;
 	int argr = 0;
 	int argw = 0;
 	int error = 0;
 	int interval = 0;
 	int option = 0;
 	int parserBufferSize = 8192 * 1024;
+	int profiling = 0;
 	xsCreation _creation = {
 		32 * 1024 * 1024,	/* initialChunkSize */
 		4 * 1024 * 1024,	/* incrementalChunkSize */
@@ -176,7 +178,15 @@ int main(int argc, char* argv[])
 		}
 		else if (!strcmp(argv[argi], "-m"))
 			option = 2;
-		else if (!strcmp(argv[argi], "-p"))
+		else if (!strcmp(argv[argi], "-p")) {
+			profiling = 1;
+			argi++;
+			if ((argi < argc) && (argv[argi][0] != '-'))
+				argp = argi;
+			else
+				argi--;
+		}
+		else if (!strcmp(argv[argi], "-q"))
 			gxMeteringPrint = 1;
 		else if (!strcmp(argv[argi], "-r")) {
 			argi++;
@@ -231,6 +241,8 @@ int main(int argc, char* argv[])
 		machine = xsCreateMachine(creation, "xsnap", NULL);
 		xsBuildAgent(machine);
 	}
+	if (profiling)
+		fxStartProfiling(machine);
 	xsBeginMetering(machine, xsMeteringCallback, interval);
 	{
 		if (option == 5) {
@@ -265,6 +277,8 @@ int main(int argc, char* argv[])
 					}
 					if (argv[argi][0] == '-')
 						continue;
+					if (argi == argp)
+						continue;
 					if (argi == argr)
 						continue;
 					if (argi == argw)
@@ -301,6 +315,17 @@ int main(int argc, char* argv[])
 		}
 	}
 	xsEndMetering(machine);
+	if (profiling) {
+		if (argp) {
+			FILE* stream = fopen(argv[argp], "w");
+			if (stream)
+				fxStopProfiling(machine, stream);
+			else
+				fprintf(stderr, "cannot write profile %s: %s\n", argv[argp], strerror(errno));
+		}
+		else
+			fxStopProfiling(machine, C_NULL);
+	}
 	if (machine->abortStatus)
 		error = machine->abortStatus;
 	xsDeleteMachine(machine);

xsnap/sources/xsnap.h

@@ -72,6 +72,8 @@ struct xsSnapshotRecord {
 #define xsWriteSnapshot(_THE, _SNAPSHOT) \
 	fxWriteSnapshot(_THE, _SNAPSHOT)
 
+#define xsRunDebugger(_THE) \
+	fxRunDebugger(_THE)
 #define xsRunModuleFile(_PATH) \
 	fxRunModuleFile(the, _PATH)
 #define xsRunProgramFile(_PATH) \
@@ -87,6 +89,13 @@ struct xsSnapshotRecord {
 #define xsVersion(_BUFFER, _SIZE) \
 	fxVersion(_BUFFER, _SIZE)
 
+#ifdef mxInstrument	
+#define xsDescribeInstrumentation(_THE,_COUNT,_NAMES,_UNITS) \
+	fxDescribeInstrumentation(_THE,_COUNT,_NAMES,_UNITS)
+#define xsSampleInstrumentation(_THE,_COUNT,_VALUES) \
+	fxSampleInstrumentation(_THE,_COUNT,_VALUES)
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -106,13 +115,19 @@ mxImport void fxSetCurrentMeter(xsMachine* the, xsUnsignedValue value);
 mxImport xsMachine* fxReadSnapshot(xsSnapshot* snapshot, xsStringValue theName, void* theContext);
 mxImport int fxWriteSnapshot(xsMachine* the, xsSnapshot* snapshot);
 
+mxImport void fxRunDebugger(xsMachine* the);
 mxImport void fxRunModuleFile(xsMachine* the, xsStringValue path);
 mxImport void fxRunProgramFile(xsMachine* the, xsStringValue path);
 mxImport void fxRunLoop(xsMachine* the);
 
 mxImport void fxClearTimer(xsMachine* the);
 mxImport void fxSetTimer(xsMachine* the, xsNumberValue interval, xsBooleanValue repeat);
 
+#ifdef mxInstrument	
+mxImport void fxDescribeInstrumentation(xsMachine* the, xsIntegerValue count, xsStringValue* names, xsStringValue* units);
+mxImport void fxSampleInstrumentation(xsMachine* the, xsIntegerValue count, xsIntegerValue* values);
+#endif
+
 mxImport void fxVersion(xsStringValue theBuffer, xsUnsignedValue theSize);
 
 mxImport void fx_lockdown(xsMachine* the);