feat: add support for capturing data for any variable at runtime (#355)

Fixes #105

packages/build/src/build/impl/gen-model.ts

@@ -165,13 +165,18 @@ async function generateC(context: BuildContext, sdeDir: string, sdeCmdPath: stri
 
   // Use SDE to generate a C version of the model
   const command = sdeCmdPath
-  const args = ['generate', '--genc', '--spec', 'spec.json', 'processed']
-  await context.spawnChild(prepDir, command, args, {
+  const gencArgs = ['generate', '--genc', '--spec', 'spec.json', 'processed']
+  await context.spawnChild(prepDir, command, gencArgs, {
     // By default, ignore lines that start with "WARNING: Data for" since these are often harmless
     // TODO: Don't filter by default, but make it configurable
     // ignoredMessageFilter: 'WARNING: Data for'
   })
 
+  // Use SDE to generate a JSON list of all model dimensions and variables
+  // TODO: Allow --genc and --list in same command so that we only need to process once
+  const listArgs = ['generate', '--list', '--spec', 'spec.json', 'processed']
+  await context.spawnChild(prepDir, command, listArgs, {})
+
   // Copy SDE's supporting C files into the build directory
   const buildDir = joinPath(prepDir, 'build')
   const sdeCDir = joinPath(sdeDir, 'src', 'c')

packages/cli/src/c/model.c

@@ -8,6 +8,14 @@
 struct timespec startTime, finishTime;
 #endif
 
+// For each output variable specified in the indices buffer, there
+// are 4 index values:
+//   varIndex
+//   subIndex0
+//   subIndex1
+//   subIndex2
+#define INDICES_PER_OUTPUT 4
+
 // The special _time variable is not included in .mdl files.
 double _time;
 
@@ -17,6 +25,7 @@ size_t outputIndex = 0;
 
 // Output data buffer used by `runModelWithBuffers`
 double* outputBuffer = NULL;
+int32_t* outputIndexBuffer = NULL;
 size_t outputVarIndex = 0;
 size_t numSavePoints = 0;
 size_t savePointIndex = 0;
@@ -66,6 +75,13 @@ double getSaveper() {
   return _saveper;
 }
 
+/**
+ * Return the constant `maxOutputIndices` value.
+ */
+int getMaxOutputIndices() {
+  return maxOutputIndices;
+}
+
 char* run_model(const char* inputs) {
   // run_model does everything necessary to run the model with the given inputs.
   // It may be called multiple times. Call finish() after all runs are complete.
@@ -102,13 +118,15 @@ char* run_model(const char* inputs) {
  * (where tN is the last time in the range), the second variable outputs will begin,
  * and so on.
  */
-void runModelWithBuffers(double* inputs, double* outputs) {
+void runModelWithBuffers(double* inputs, double* outputs, int32_t* outputIndices) {
   outputBuffer = outputs;
+  outputIndexBuffer = outputIndices;
   initConstants();
   setInputsFromBuffer(inputs);
   initLevels();
   run();
   outputBuffer = NULL;
+  outputIndexBuffer = NULL;
 }
 
 void run() {
@@ -137,7 +155,25 @@ void run() {
         numSavePoints = (size_t)(round((_final_time - _initial_time) / _saveper)) + 1;
       }
       outputVarIndex = 0;
-      storeOutputData();
+      if (outputIndexBuffer != NULL) {
+        // Store the outputs as specified in the current output index buffer
+        for (size_t i = 0; i < maxOutputIndices; i++) {
+          size_t indexBufferOffset = i * INDICES_PER_OUTPUT;
+          size_t varIndex = (size_t)outputIndexBuffer[indexBufferOffset];
+          if (varIndex > 0) {
+            size_t subIndex0 = (size_t)outputIndexBuffer[indexBufferOffset + 1];
+            size_t subIndex1 = (size_t)outputIndexBuffer[indexBufferOffset + 2];
+            size_t subIndex2 = (size_t)outputIndexBuffer[indexBufferOffset + 3];
+            storeOutput(varIndex, subIndex0, subIndex1, subIndex2);
+          } else {
+            // Stop when we reach the first zero index
+            break;
+          }
+        }
+      } else {
+        // Store the normal outputs
+        storeOutputData();
+      }
       savePointIndex++;
     }
     if (step == lastStep) break;

packages/cli/src/c/sde.h

@@ -42,6 +42,7 @@ EXTERN double _epsilon;
 
 // Internal variables
 EXTERN const int numOutputs;
+EXTERN const int maxOutputIndices;
 
 // Standard simulation control parameters
 EXTERN double _time;
@@ -55,7 +56,7 @@ double getInitialTime(void);
 double getFinalTime(void);
 double getSaveper(void);
 char* run_model(const char* inputs);
-void runModelWithBuffers(double* inputs, double* outputs);
+void runModelWithBuffers(double* inputs, double* outputs, int32_t* outputIndices);
 void run(void);
 void startOutput(void);
 void outputVar(double value);
@@ -69,6 +70,7 @@ void setInputsFromBuffer(double *inputData);
 void evalAux(void);
 void evalLevels(void);
 void storeOutputData(void);
+void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t subIndex2);
 const char* getHeader(void);
 
 #ifdef __cplusplus

packages/compile/src/generate/code-gen.js

@@ -163,7 +163,17 @@ const char* getHeader() {
 }
 
 void storeOutputData() {
-${outputSection(outputVars)}
+${specOutputSection(outputVars)}
+}
+
+void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t subIndex2) {
+#if SDE_USE_OUTPUT_INDICES
+  switch (varIndex) {
+${fullOutputSection(Model.varIndexInfo())}
+    default:
+      break;
+  }
+#endif
 }
 `
   }
@@ -250,11 +260,16 @@ ${postStep}
   }
   function internalVarsSection() {
     // Declare internal variables to run the model.
+    let decls
     if (outputAllVars) {
-      return `const int numOutputs = ${expandedVarNames().length};`
+      decls = `const int numOutputs = ${expandedVarNames().length};`
     } else {
-      return `const int numOutputs = ${spec.outputVars.length};`
+      decls = `const int numOutputs = ${spec.outputVars.length};`
     }
+    decls += `\n#define SDE_USE_OUTPUT_INDICES 0`
+    decls += `\n#define SDE_MAX_OUTPUT_INDICES 1000`
+    decls += `\nconst int maxOutputIndices = SDE_USE_OUTPUT_INDICES ? SDE_MAX_OUTPUT_INDICES : 0;`
+    return decls
   }
   function arrayDimensionsSection() {
     // Emit a declaration for each array dimension's index numbers.
@@ -312,12 +327,34 @@ ${postStep}
   //
   // Input/output section helpers
   //
-  function outputSection(varNames) {
+  function specOutputSection(varNames) {
     // Emit output calls using varNames in C format.
     let code = R.map(varName => `  outputVar(${varName});`)
     let section = R.pipe(code, lines)
     return section(varNames)
   }
+  function fullOutputSection(varIndexInfo) {
+    // Emit output calls for all variables.
+    const code = R.map(info => {
+      let varAccess = info.varName
+      if (info.subscriptCount > 0) {
+        varAccess += '[subIndex0]'
+      }
+      if (info.subscriptCount > 1) {
+        varAccess += '[subIndex1]'
+      }
+      if (info.subscriptCount > 2) {
+        varAccess += '[subIndex2]'
+      }
+      let c = ''
+      c += `    case ${info.varIndex}:\n`
+      c += `      outputVar(${varAccess});\n`
+      c += `      break;`
+      return c
+    })
+    const section = R.pipe(code, lines)
+    return section(varIndexInfo)
+  }
   function inputsFromStringImpl() {
     // If there was an I/O spec file, then emit code to parse input variables.
     // The user can replace this with a parser for a different serialization format.

packages/compile/src/model/model.js

@@ -330,8 +330,14 @@ function removeUnusedVariables(spec) {
       if (!referencedRefIds.has(refId)) {
         referencedRefIds.add(refId)
         const refVar = varWithRefId(refId)
-        recordUsedVariable(refVar)
-        recordRefsOfVariable(refVar)
+        if (refVar) {
+          recordUsedVariable(refVar)
+          recordRefsOfVariable(refVar)
+        } else {
+          console.error(`No var found for ${refId}`)
+          console.error(v)
+          process.exit(1)
+        }
       }
     }
   }
@@ -1010,6 +1016,105 @@ function printDepsGraph(graph, varType) {
     console.error(`${dep[0]} → ${dep[1]}`)
   }
 }
+
+function allListedVars() {
+  // Put variables into the order that they are evaluated by SDE in the generated model
+  let vars = []
+  vars.push(...constVars())
+  vars.push(...lookupVars())
+  vars.push(...dataVars())
+  vars.push(varWithName('_time'))
+  vars.push(...initVars())
+  vars.push(...auxVars())
+  // TODO: Also levelVars not covered by initVars?
+
+  // Filter out data/lookup variables and variables that are generated/used internally
+  const isInternal = v => {
+    return v.refId.startsWith('__level') || v.refId.startsWith('__aux')
+  }
+
+  return R.filter(v => !isInternal(v), vars)
+}
+
+function filteredListedVars() {
+  // Extract a subset of the available info for each variable and sort all variables
+  // according to the order that they are evaluated by SDE in the generated model
+  return R.map(v => filterVar(v), allListedVars())
+}
+
+function varIndexInfoMap() {
+  // Return a map containing information for each listed variable:
+  //   varName
+  //   varIndex
+  //   subscriptCount
+
+  // Get the filtered variables in the order that they are evaluated by SDE in the
+  // generated model
+  const sortedVars = filteredListedVars()
+
+  // Get the set of unique variable names, and assign a 1-based index
+  // to each; this matches the index number used in `storeOutput()`
+  // in the generated C code
+  const infoMap = new Map()
+  let varIndex = 1
+  for (const v of sortedVars) {
+    if (v.varType === 'data' || v.varType === 'lookup') {
+      // Omit the index for data and lookup variables; at this time, the data for these
+      // cannot be output like for other types of variables
+      continue
+    }
+    const varName = v.varName
+    if (!infoMap.get(varName)) {
+      infoMap.set(varName, {
+        varName,
+        varIndex,
+        subscriptCount: v.families ? v.families.length : 0
+      })
+      varIndex++
+    }
+  }
+
+  return infoMap
+}
+
+function varIndexInfo() {
+  // Return an array, sorted by `varName`, containing information for each
+  // listed variable:
+  //   varName
+  //   varIndex
+  //   subscriptCount
+  return Array.from(varIndexInfoMap().values())
+}
+
+function jsonList() {
+  // Return a stringified JSON object containing variable and subscript information
+  // for the model.
+
+  // Get the set of available subscripts
+  const allDims = [...allDimensions()]
+  const sortedDims = allDims.sort((a, b) => a.name.localeCompare(b.name))
+
+  // Extract a subset of the available info for each variable and put them in eval order
+  const sortedVars = filteredListedVars()
+
+  // Assign a 1-based index for each variable that has data that can be accessed.
+  // This matches the index number used in `storeOutput()` in the generated C code.
+  const infoMap = varIndexInfoMap()
+  for (const v of sortedVars) {
+    const varInfo = infoMap.get(v.varName)
+    if (varInfo) {
+      v.varIndex = varInfo.varIndex
+    }
+  }
+
+  // Convert to JSON
+  const obj = {
+    dimensions: sortedDims,
+    variables: sortedVars
+  }
+  return JSON.stringify(obj, null, 2)
+}
+
 export default {
   addConstantExpr,
   addEquation,
@@ -1026,6 +1131,7 @@ export default {
   initVars,
   isInputVar,
   isNonAtoAName,
+  jsonList,
   levelVars,
   lookupVars,
   printRefGraph,
@@ -1036,6 +1142,7 @@ export default {
   refIdsWithName,
   splitRefId,
   variables,
+  varIndexInfo,
   varNames,
   varsWithName,
   varWithName,

packages/compile/src/parse-and-generate.js

@@ -17,7 +17,7 @@ import { generateCode } from './generate/code-gen.js'
  *
  * - If `operation` is 'generateC', the generated C code will be written to `buildDir`.
  * - If `operation` is 'printVarList', variables and subscripts will be written to
- *   txt and yaml files under `buildDir`.
+ *   txt, yaml, and json files under `buildDir`.
  * - If `operation` is 'printRefIdTest', reference identifiers will be printed to the console.
  * - If `operation` is 'convertNames', no output will be generated, but the results of model
  *   analysis will be available.
@@ -85,6 +85,8 @@ export async function parseAndGenerate(input, spec, operation, modelDirname, mod
     writeOutput(`${modelName}_vars.yaml`, Model.yamlVarList())
     // Write subscripts to a YAML file.
     writeOutput(`${modelName}_subs.yaml`, yamlSubsList())
+    // Write variables and subscripts to a JSON file.
+    writeOutput(`${modelName}.json`, Model.jsonList())
   }
 
   return code

packages/plugin-wasm/src/plugin.ts

@@ -107,7 +107,9 @@ async function buildWasm(
   addFlag('SINGLE_FILE=1')
   addFlag('EXPORT_ES6=1')
   addFlag('USE_ES6_IMPORT_META=0')
-  addFlag(`EXPORTED_FUNCTIONS=['_malloc','_getInitialTime','_getFinalTime','_getSaveper','_runModelWithBuffers']`)
+  addFlag(
+    `EXPORTED_FUNCTIONS=['_malloc','_getMaxOutputIndices','_getInitialTime','_getFinalTime','_getSaveper','_runModelWithBuffers']`
+  )
   addFlag(`EXPORTED_RUNTIME_METHODS=['cwrap']`)
 
   await context.spawnChild(prepDir, command, args, {