diff --git a/src/Preprocessor.js b/src/Preprocessor.js index 5307b1ed..4c46941d 100644 --- a/src/Preprocessor.js +++ b/src/Preprocessor.js @@ -3,7 +3,7 @@ const R = require('ramda') const B = require('bufx') const { splitEquations } = require('./Helpers') -let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false) => { +let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false, outDecls = []) => { const MACROS_FILENAME = 'macros.txt' const REMOVALS_FILENAME = 'removals.txt' const INSERTIONS_FILENAME = 'mdl-edits.txt' @@ -128,7 +128,7 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false) // Extract the LHS variable name for each equation, which we will use to sort // the equations alphabetically - const unsorted = [] + const unsorted = outDecls for (let eqn of eqns) { // Ignore the encoding eqn = eqn.replace('{UTF-8}', '') @@ -139,18 +139,28 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false) if (eqn.length > 0) { // Split on newlines so that we look only at the first line of each declaration let line = eqn.split(/\n/)[0].trim() - // If the line contains an '=', treat this as an equation, otherwise it is a - // basic declaration let kind let key = line + // Strip the ":INTERPOLATE:"; it should not be included in the key + key = key.replace(/:INTERPOLATE:/g, '') if (key.includes('=')) { + // The line contains an '='; treat this as an equation kind = 'eqn' key = key.split('=')[0].trim() + } else if (key.includes(':')) { + // The line contains a ':'; treat this as an subscript declaration + kind = 'sub' + key = key.split(':')[0].trim() } else { + // Treat this as a general declaration kind = 'decl' } // Ignore double quotes key = key.replace(/\"/g, '') + // Ignore the comments if this is a one-line declaration + key = key.split('~')[0] + // Ignore the lookup data if it starts on the first line + key = key.split('(')[0] // Ignore any whitespace that remains key = key.trim() // Ignore case @@ -158,7 +168,7 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false) unsorted.push({ key, kind, - eqn + originalDecl: eqn }) } } @@ -170,9 +180,11 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false) // Emit formula lines without comment contents. for (const elem of sorted) { - const eqn = elem.eqn + const eqn = elem.originalDecl + let processedDecl = eqn let iComment = eqn.indexOf('~') if (iComment >= 0) { + processedDecl = '' let formula = B.lines(eqn.substr(0, iComment)) for (let i = 0; i < formula.length; i++) { let line = formula[i] @@ -181,25 +193,33 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false) if (i === 0) { if (line !== ENCODING) { emitPP(line) + processedDecl += line } } else { if (opts.joinFormulaLines) { // Remove any leading tabs - emitPP(line.replace(/^\t+/, '')) + const lineWithoutLeadingTabs = line.replace(/^\t+/, '') + emitPP(lineWithoutLeadingTabs) + processedDecl += lineWithoutLeadingTabs } else { // Only emit the line if it has non-whitespace characters if (line.length > 0) { emitPP(`\n${line}`) + processedDecl += `\n${line}` } } } } + // Emit the last line if (opts.emitCommentMarkers) { - B.emitLine('\n\t~~|\n', 'pp') + const declEnd = '\n\t~~|' + B.emitLine(`${declEnd}\n`, 'pp') + processedDecl += declEnd } else { B.emitLine('', 'pp') } } + elem.processedDecl = processedDecl } getMdlFromPPBuf() diff --git a/src/sde-flatten.js b/src/sde-flatten.js new file mode 100644 index 00000000..f8adbc93 --- /dev/null +++ b/src/sde-flatten.js @@ -0,0 +1,192 @@ +const B = require('bufx') +const path = require('path') +const { modelPathProps, buildDir } = require('./Helpers') +const { preprocessModel } = require('./Preprocessor') + +const command = 'flatten [options] ' + +// TODO: Set to false for now to keep it hidden from the help menu, since +// this is an experimental feature +//const describe = 'flatten submodels into a single preprocessed mdl file' +const describe = false + +const builder = { + builddir: { + describe: 'build directory', + type: 'string', + alias: 'b' + }, + inputs: { + describe: 'input mdl files', + demand: true, + type: 'array' + } +} + +const handler = argv => { + flatten(argv.outmodel, argv.inputs, argv) +} + +const flatten = async (outFile, inFiles, opts) => { + // Ensure the build directory exists + // TODO: Since the input mdl files can technically exist in more than + // one directory, we don't want to guess, so just use the current directory + // if one isn't provided on the command line + const buildDirname = buildDir(opts.builddir, '.') + + // Extract the equations and declarations from each parent or submodel file + // and store them into a map keyed by source file + const fileDecls = {} + for (const inFile of inFiles) { + // Get the path and short name of the input mdl file + const inModelProps = modelPathProps(inFile) + + // Preprocess the mdl file and extract the equations + const decls = [] + preprocessModel(inModelProps.modelPathname, undefined, 'genc', false, decls) + + // Associate each declaration with the name of the model from which it came + for (const decl of decls) { + decl.sourceModelName = inModelProps.modelName + } + + fileDecls[inModelProps.modelName] = decls + } + + // Collapse so that we have only one equation or declaration per key. + // Equations that are defined in a different submodel will appear like + // a data variable in the model file where it is used (i.e., no equals + // sign). If we see an equation, that takes priority over a "data" + // declaration. + const collapsed = {} + let hasErrors = false + for (const modelName of Object.keys(fileDecls)) { + const decls = fileDecls[modelName] + for (const decl of decls) { + const existingDecl = collapsed[decl.key] + if (existingDecl) { + // We've already seen a declaration with this key (from another file) + if (decl.kind === existingDecl.kind) { + if (decl.kind === 'sub') { + // The subscript keys are the same, so see if the contents are the same + if (decl.processedDecl !== existingDecl.processedDecl) { + // TODO: If we see two subscript declarations that differ, it's + // probably because a submodel uses a simplified set of dimension + // mappings, while the parent includes a superset of mappings. + // We should add more checks here, like see if the mappings in one + // are a subset of the other, but for now, we will emit a warning + // and just take the longer of the two. + let longerDecl + if (decl.processedDecl.length > existingDecl.processedDecl.length) { + longerDecl = decl + } else { + longerDecl = existingDecl + } + collapsed[decl.key] = longerDecl + let warnMsg = 'Subscript declarations are different; ' + warnMsg += `will use the one from '${longerDecl.sourceModelName}.mdl' because it is longer` + console.warn('----------') + console.warn(`\nWARNING: ${warnMsg}:`) + console.warn(`\nIn ${existingDecl.sourceModelName}.mdl:`) + console.warn(`${existingDecl.processedDecl}`) + console.warn(`\nIn ${decl.sourceModelName}.mdl:`) + console.warn(`${decl.processedDecl}\n`) + } + } else if (decl.kind === 'eqn') { + // The equation keys are the same, so see if the contents are the same + if (decl.processedDecl !== existingDecl.processedDecl) { + hasErrors = true + console.error('----------') + console.error(`\nERROR: Differing equations:`) + console.error(`\nIn ${existingDecl.sourceModelName}.mdl:`) + console.error(`${existingDecl.processedDecl}`) + console.error(`\nIn ${decl.sourceModelName}.mdl:`) + console.error(`${decl.processedDecl}\n`) + } + } + } else if (decl.kind === 'eqn' && existingDecl.kind === 'decl') { + // Replace the declaration with this equation + collapsed[decl.key] = decl + } + } else { + // We haven't seen this key yet, so save the declaration + collapsed[decl.key] = decl + } + } + } + + // XXX: For any subscripted (non-equation) declarations that remain, see if + // there are any corresponding equations. This can happen in the case of + // subscripted variables that are defined in multiple parts in one submodel, + // but are declared using the full set in the consuming model. + // For example, in the submodel: + // Variable[SubscriptA_Subset1] = ... ~~| + // Variable[SubscriptA_Subset2] = ... ~~| + // In the consuming model: + // Variable[SubscriptA_All] ~~| + // In this case, if we see Variable[SubscriptA_All], we can remove it and + // use the other `Variable` definitions. + for (const key of Object.keys(collapsed)) { + const decl = collapsed[key] + if (decl.kind === 'decl') { + const declKeyWithoutSubscript = key.split('[')[0] + + // See if there is another equation that has the same key (minus subscript) + const otherDeclsThatMatch = [] + for (const otherDecl of Object.values(collapsed)) { + if (otherDecl.kind === 'eqn') { + const otherDeclKeyWithoutSubscript = otherDecl.key.split('[')[0] + if (declKeyWithoutSubscript === otherDeclKeyWithoutSubscript) { + otherDeclsThatMatch.push(otherDecl) + } + } + } + + if (otherDeclsThatMatch.length > 0) { + // Remove this declaration and emit a warning + delete collapsed[key] + console.warn('----------') + console.warn(`\nWARNING: Skipping declaration:`) + console.warn(`\nIn ${decl.sourceModelName}.mdl:`) + console.warn(`${decl.processedDecl}`) + console.warn(`\nThe following equations are assumed to provide the necessary data:`) + for (const otherDecl of otherDeclsThatMatch) { + console.warn(`\nIn ${otherDecl.sourceModelName}.mdl:`) + console.warn(`${otherDecl.processedDecl}`) + } + console.warn() + } + } + } + + // Exit with a non-zero error code if there are any conflicting declarations + if (hasErrors) { + process.exit(1) + } + + // Sort the declarations alphabetically by LHS variable name + const sorted = Object.values(collapsed).sort((a, b) => { + return (a.key < b.key) ? -1 : (a.key > b.key) ? 1 : 0; + }) + + // Build a single buffer containing the sorted declarations + B.open('pp') + const ENCODING = '{UTF-8}' + B.emitLine(ENCODING, 'pp') + B.emitLine('', 'pp') + for (const decl of sorted) { + B.emitLine(`${decl.processedDecl}\n`, 'pp') + } + + // Write the flattened mdl file to the build directory + const outModelProps = modelPathProps(outFile) + let outputPathname = path.join(buildDirname, `${outModelProps.modelName}.mdl`) + B.writeBuf(outputPathname, 'pp') +} + +module.exports = { + command, + describe, + builder, + handler +} diff --git a/src/sde.js b/src/sde.js index 3de9d888..866e2541 100644 --- a/src/sde.js +++ b/src/sde.js @@ -22,6 +22,7 @@ let exitCode = require('yargs') .strict() .usage('usage: $0 ') .command(require('./sde-generate')) + .command(require('./sde-flatten')) .command(require('./sde-compile')) .command(require('./sde-exec')) .command(require('./sde-log'))