feat: add sde command for flattening parent+submodels

src/Preprocessor.js

@@ -3,7 +3,7 @@ const R = require('ramda')
 const B = require('bufx')
 const { splitEquations } = require('./Helpers')
 
-let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false) => {
+let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false, outDecls = []) => {
   const MACROS_FILENAME = 'macros.txt'
   const REMOVALS_FILENAME = 'removals.txt'
   const INSERTIONS_FILENAME = 'mdl-edits.txt'
@@ -128,7 +128,7 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false)
 
   // Extract the LHS variable name for each equation, which we will use to sort
   // the equations alphabetically
-  const unsorted = []
+  const unsorted = outDecls
   for (let eqn of eqns) {
     // Ignore the encoding
     eqn = eqn.replace('{UTF-8}', '')
@@ -139,26 +139,36 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false)
     if (eqn.length > 0) {
       // Split on newlines so that we look only at the first line of each declaration
       let line = eqn.split(/\n/)[0].trim()
-      // If the line contains an '=', treat this as an equation, otherwise it is a
-      // basic declaration
       let kind
       let key = line
+      // Strip the ":INTERPOLATE:"; it should not be included in the key
+      key = key.replace(/:INTERPOLATE:/g, '')
       if (key.includes('=')) {
+        // The line contains an '='; treat this as an equation
         kind = 'eqn'
         key = key.split('=')[0].trim()
+      } else if (key.includes(':')) {
+        // The line contains a ':'; treat this as an subscript declaration
+        kind = 'sub'
+        key = key.split(':')[0].trim()
       } else {
+        // Treat this as a general declaration
         kind = 'decl'
       }
       // Ignore double quotes
       key = key.replace(/\"/g, '')
+      // Ignore the comments if this is a one-line declaration
+      key = key.split('~')[0]
+      // Ignore the lookup data if it starts on the first line
+      key = key.split('(')[0]
       // Ignore any whitespace that remains
       key = key.trim()
       // Ignore case
       key = key.toLowerCase()
       unsorted.push({
         key,
         kind,
-        eqn
+        originalDecl: eqn
       })
     }
   }
@@ -170,9 +180,11 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false)
 
   // Emit formula lines without comment contents.
   for (const elem of sorted) {
-    const eqn = elem.eqn
+    const eqn = elem.originalDecl
+    let processedDecl = eqn
     let iComment = eqn.indexOf('~')
     if (iComment >= 0) {
+      processedDecl = ''
       let formula = B.lines(eqn.substr(0, iComment))
       for (let i = 0; i < formula.length; i++) {
         let line = formula[i]
@@ -181,25 +193,33 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false)
         if (i === 0) {
           if (line !== ENCODING) {
             emitPP(line)
+            processedDecl += line
           }
         } else {
           if (opts.joinFormulaLines) {
             // Remove any leading tabs
-            emitPP(line.replace(/^\t+/, ''))
+            const lineWithoutLeadingTabs = line.replace(/^\t+/, '')
+            emitPP(lineWithoutLeadingTabs)
+            processedDecl += lineWithoutLeadingTabs
           } else {
             // Only emit the line if it has non-whitespace characters
             if (line.length > 0) {
               emitPP(`\n${line}`)
+              processedDecl += `\n${line}`
             }
           }
         }
       }
+      // Emit the last line
       if (opts.emitCommentMarkers) {
-        B.emitLine('\n\t~~|\n', 'pp')
+        const declEnd = '\n\t~~|'
+        B.emitLine(`${declEnd}\n`, 'pp')
+        processedDecl += declEnd
       } else {
         B.emitLine('', 'pp')
       }
     }
+    elem.processedDecl = processedDecl
   }
   getMdlFromPPBuf()
 

src/sde-flatten.js

@@ -0,0 +1,192 @@
+const B = require('bufx')
+const path = require('path')
+const { modelPathProps, buildDir } = require('./Helpers')
+const { preprocessModel } = require('./Preprocessor')
+
+const command = 'flatten [options] <outmodel>'
+
+// TODO: Set to false for now to keep it hidden from the help menu, since
+// this is an experimental feature
+//const describe = 'flatten submodels into a single preprocessed mdl file'
+const describe = false
+
+const builder = {
+  builddir: {
+    describe: 'build directory',
+    type: 'string',
+    alias: 'b'
+  },
+  inputs: {
+    describe: 'input mdl files',
+    demand: true,
+    type: 'array'
+  }
+}
+
+const handler = argv => {
+  flatten(argv.outmodel, argv.inputs, argv)
+}
+
+const flatten = async (outFile, inFiles, opts) => {
+  // Ensure the build directory exists
+  // TODO: Since the input mdl files can technically exist in more than
+  // one directory, we don't want to guess, so just use the current directory
+  // if one isn't provided on the command line
+  const buildDirname = buildDir(opts.builddir, '.')
+
+  // Extract the equations and declarations from each parent or submodel file
+  // and store them into a map keyed by source file
+  const fileDecls = {}
+  for (const inFile of inFiles) {
+    // Get the path and short name of the input mdl file
+    const inModelProps = modelPathProps(inFile)
+
+    // Preprocess the mdl file and extract the equations
+    const decls = []
+    preprocessModel(inModelProps.modelPathname, undefined, 'genc', false, decls)
+
+    // Associate each declaration with the name of the model from which it came
+    for (const decl of decls) {
+      decl.sourceModelName = inModelProps.modelName
+    }
+
+    fileDecls[inModelProps.modelName] = decls
+  }
+
+  // Collapse so that we have only one equation or declaration per key.
+  // Equations that are defined in a different submodel will appear like
+  // a data variable in the model file where it is used (i.e., no equals
+  // sign).  If we see an equation, that takes priority over a "data"
+  // declaration.
+  const collapsed = {}
+  let hasErrors = false
+  for (const modelName of Object.keys(fileDecls)) {
+    const decls = fileDecls[modelName]
+    for (const decl of decls) {
+      const existingDecl = collapsed[decl.key]
+      if (existingDecl) {
+        // We've already seen a declaration with this key (from another file)
+        if (decl.kind === existingDecl.kind) {
+          if (decl.kind === 'sub') {
+            // The subscript keys are the same, so see if the contents are the same
+            if (decl.processedDecl !== existingDecl.processedDecl) {
+              // TODO: If we see two subscript declarations that differ, it's
+              // probably because a submodel uses a simplified set of dimension
+              // mappings, while the parent includes a superset of mappings.
+              // We should add more checks here, like see if the mappings in one
+              // are a subset of the other, but for now, we will emit a warning
+              // and just take the longer of the two.
+              let longerDecl
+              if (decl.processedDecl.length > existingDecl.processedDecl.length) {
+                longerDecl = decl
+              } else {
+                longerDecl = existingDecl
+              }
+              collapsed[decl.key] = longerDecl
+              let warnMsg = 'Subscript declarations are different; '
+              warnMsg += `will use the one from '${longerDecl.sourceModelName}.mdl' because it is longer`
+              console.warn('----------')
+              console.warn(`\nWARNING: ${warnMsg}:`)
+              console.warn(`\nIn ${existingDecl.sourceModelName}.mdl:`)
+              console.warn(`${existingDecl.processedDecl}`)
+              console.warn(`\nIn ${decl.sourceModelName}.mdl:`)
+              console.warn(`${decl.processedDecl}\n`)
+            }
+          } else if (decl.kind === 'eqn') {
+            // The equation keys are the same, so see if the contents are the same
+            if (decl.processedDecl !== existingDecl.processedDecl) {
+              hasErrors = true
+              console.error('----------')
+              console.error(`\nERROR: Differing equations:`)
+              console.error(`\nIn ${existingDecl.sourceModelName}.mdl:`)
+              console.error(`${existingDecl.processedDecl}`)
+              console.error(`\nIn ${decl.sourceModelName}.mdl:`)
+              console.error(`${decl.processedDecl}\n`)
+            }
+          }
+        } else if (decl.kind === 'eqn' && existingDecl.kind === 'decl') {
+          // Replace the declaration with this equation
+          collapsed[decl.key] = decl
+        }
+      } else {
+        // We haven't seen this key yet, so save the declaration
+        collapsed[decl.key] = decl
+      }
+    }
+  }
+
+  // XXX: For any subscripted (non-equation) declarations that remain, see if
+  // there are any corresponding equations.  This can happen in the case of
+  // subscripted variables that are defined in multiple parts in one submodel,
+  // but are declared using the full set in the consuming model.
+  // For example, in the submodel:
+  //   Variable[SubscriptA_Subset1] = ... ~~|
+  //   Variable[SubscriptA_Subset2] = ... ~~|
+  // In the consuming model:
+  //   Variable[SubscriptA_All] ~~|
+  // In this case, if we see Variable[SubscriptA_All], we can remove it and
+  // use the other `Variable` definitions.
+  for (const key of Object.keys(collapsed)) {
+    const decl = collapsed[key]
+    if (decl.kind === 'decl') {
+      const declKeyWithoutSubscript = key.split('[')[0]
+
+      // See if there is another equation that has the same key (minus subscript)
+      const otherDeclsThatMatch = []
+      for (const otherDecl of Object.values(collapsed)) {
+        if (otherDecl.kind === 'eqn') {
+          const otherDeclKeyWithoutSubscript = otherDecl.key.split('[')[0]
+          if (declKeyWithoutSubscript === otherDeclKeyWithoutSubscript) {
+            otherDeclsThatMatch.push(otherDecl)
+          }
+        }
+      }
+
+      if (otherDeclsThatMatch.length > 0) {
+        // Remove this declaration and emit a warning
+        delete collapsed[key]
+        console.warn('----------')
+        console.warn(`\nWARNING: Skipping declaration:`)
+        console.warn(`\nIn ${decl.sourceModelName}.mdl:`)
+        console.warn(`${decl.processedDecl}`)
+        console.warn(`\nThe following equations are assumed to provide the necessary data:`)
+        for (const otherDecl of otherDeclsThatMatch) {
+          console.warn(`\nIn ${otherDecl.sourceModelName}.mdl:`)
+          console.warn(`${otherDecl.processedDecl}`)
+        }
+        console.warn()
+      }
+    }
+  }
+
+  // Exit with a non-zero error code if there are any conflicting declarations
+  if (hasErrors) {
+    process.exit(1)
+  }
+
+  // Sort the declarations alphabetically by LHS variable name
+  const sorted = Object.values(collapsed).sort((a, b) => {
+    return (a.key < b.key) ? -1 : (a.key > b.key) ? 1 : 0;
+  })
+
+  // Build a single buffer containing the sorted declarations
+  B.open('pp')
+  const ENCODING = '{UTF-8}'
+  B.emitLine(ENCODING, 'pp')
+  B.emitLine('', 'pp')
+  for (const decl of sorted) {
+    B.emitLine(`${decl.processedDecl}\n`, 'pp')
+  }
+
+  // Write the flattened mdl file to the build directory
+  const outModelProps = modelPathProps(outFile)
+  let outputPathname = path.join(buildDirname, `${outModelProps.modelName}.mdl`)
+  B.writeBuf(outputPathname, 'pp')
+}
+
+module.exports = {
+  command,
+  describe,
+  builder,
+  handler
+}

src/sde.js

@@ -22,6 +22,7 @@ let exitCode = require('yargs')
   .strict()
   .usage('usage: $0 <command>')
   .command(require('./sde-generate'))
+  .command(require('./sde-flatten'))
   .command(require('./sde-compile'))
   .command(require('./sde-exec'))
   .command(require('./sde-log'))