From 2985751cb948edb7324be0504d31751798e93d17 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 11:40:24 -0700 Subject: [PATCH 01/15] fix: delete equation-gen.js --- packages/compile/src/generate/code-gen.js | 7 +- packages/compile/src/generate/equation-gen.js | 1268 ----------------- .../src/generate/gen-equation-c.spec.ts | 14 +- 3 files changed, 2 insertions(+), 1287 deletions(-) delete mode 100644 packages/compile/src/generate/equation-gen.js diff --git a/packages/compile/src/generate/code-gen.js b/packages/compile/src/generate/code-gen.js index c7d30548..aa2dcf51 100644 --- a/packages/compile/src/generate/code-gen.js +++ b/packages/compile/src/generate/code-gen.js @@ -5,7 +5,6 @@ import { sub, allDimensions, allMappings, subscriptFamilies } from '../_shared/s import Model from '../model/model.js' import { generateEquation } from './gen-equation.js' -import EquationGen from './equation-gen.js' import { expandVarNames } from './expand-var-names.js' export function generateCode(parsedModel, opts) { @@ -20,11 +19,7 @@ let codeGenerator = (parsedModel, opts) => { let outputAllVars = spec.outputVarNames === undefined || spec.outputVarNames.length === 0 // Function to generate a section of the code let generateSection = R.map(v => { - if (parsedModel.kind === 'vensim-legacy') { - return new EquationGen(v, extData, directData, mode, modelDirname).generate() - } else { - return generateEquation(v, mode, extData, directData, modelDirname) - } + return generateEquation(v, mode, extData, directData, modelDirname) }) let section = R.pipe(generateSection, R.flatten, lines) function generate() { diff --git a/packages/compile/src/generate/equation-gen.js b/packages/compile/src/generate/equation-gen.js deleted file mode 100644 index 3a182a33..00000000 --- a/packages/compile/src/generate/equation-gen.js +++ /dev/null @@ -1,1268 +0,0 @@ -import path from 'path' -import { ModelLexer, ModelParser } from 'antlr4-vensim' -import * as R from 'ramda' -import XLSX from 'xlsx' - -import { - canonicalName, - cartesianProductOf, - cdbl, - cFunctionName, - isArrayFunction, - isDelayFunction, - isSmoothFunction, - isTrendFunction, - isNpvFunction, - listConcat, - newTmpVarName, - permutationsOf, - readCsv, - readXlsx, - strToConst, - vlog -} from '../_shared/helpers.js' -import { - dimensionNames, - extractMarkedDims, - hasMapping, - isDimension, - isIndex, - isTrivialDimension, - indexInSepDim, - normalizeSubscripts, - separatedVariableIndex, - sub -} from '../_shared/subscript.js' -import ModelReader from '../parse/model-reader.js' -import Model from '../model/model.js' - -import LoopIndexVars from './loop-index-vars.js' -import ModelLHSReader from './model-lhs-reader.js' - -export default class EquationGen extends ModelReader { - constructor(variable, extData, directData, mode, modelDirname) { - super() - // the variable we are generating code for - this.var = variable - // external data map from DAT files - this.extData = extData - // direct data workbooks from Excel files - this.directData = directData - // set to 'decl', 'init-lookups', 'eval', etc depending on the section being generated - this.mode = mode - // The model directory is required when reading data files for GET DIRECT DATA. - this.modelDirname = modelDirname - // Maps of LHS subscript families to loop index vars for lookup on the RHS - this.loopIndexVars = new LoopIndexVars(['i', 'j', 'k', 'l', 'm']) - this.arrayIndexVars = new LoopIndexVars(['u', 'v', 'w', 's', 't', 'f', 'g', 'h', 'o', 'p', 'q', 'r']) - // The LHS for array variables includes subscripts in normal form. - this.lhs = this.var.varName + this.lhsSubscriptGen(this.var.subscripts) - // formula expression channel - this.exprCode = '' - // comments channel - this.comments = [] - // temporary variable channel - this.tmpVarCode = [] - // subscript loop opening channel - this.subscriptLoopOpeningCode = [] - // subscript loop closing channel - this.subscriptLoopClosingCode = [] - // the name of the current array function (might differ from `currentFunctionName` - // in the case where an expression is passed to an array function such as `SUM`) - this.currentArrayFunctionName = '' - // array function code buffer - this.arrayFunctionCode = '' - // the marked dimensions for an array function - this.markedDims = [] - // stack of var names inside an expr - this.varNames = [] - // components extracted from arguments to VECTOR ELM MAP - this.vemVarName = '' - this.vemSubscripts = [] - this.vemIndexDim = '' - this.vemIndexBase = 0 - this.vemOffset = '' - // components extracted from arguments to VECTOR SORT ORDER - this.vsoVarName = '' - this.vsoOrder = '' - this.vsoTmpName = '' - this.vsoTmpDimName = '' - // components extracted from arguments to VECTOR SELECT - this.vsSelectionArray = '' - this.vsNullValue = '' - this.vsAction = 0 - this.vsError = '' - // components extracted from arguments to ALLOCATE AVAILABLE - this.aaRequestArray = '' - this.aaPriorityArray = '' - this.aaAvailableResource = '' - this.aaTmpName = '' - this.aaTmpDimName = '' - } - generate() { - // Generate code for the variable in either init or eval mode. - if (this.var.isData()) { - // If the data var was converted from a const, it will have lookup points. - // Otherwise, read a data file to get lookup data. - if (R.isEmpty(this.var.points)) { - if (this.var.directDataArgs) { - return this.generateDirectDataInit() - } else { - return this.generateExternalDataInit() - } - } else if (this.mode === 'decl') { - return - } - } - if (this.var.isLookup()) { - return this.generateLookup() - } - // Show the model var as a comment for reference. - this.comments.push(` // ${this.var.modelLHS} = ${this.var.modelFormula.replace(/\n/g, '')}`) - // Emit direct constants individually without separating them first. - if (this.var.directConstArgs) { - return this.generateDirectConstInit() - } - // Initialize array variables with dimensions in a loop for each dimension. - let dimNames = dimensionNames(this.var.subscripts) - // Turn each dimension name into a loop with a loop index variable. - // If the variable has no subscripts, nothing will be emitted here. - this.subscriptLoopOpeningCode = R.concat( - this.subscriptLoopOpeningCode, - R.map(dimName => { - let i = this.loopIndexVars.index(dimName) - return ` for (size_t ${i} = 0; ${i} < ${sub(dimName).size}; ${i}++) {` - }, dimNames) - ) - // Walk the parse tree to generate code into all channels. - // Use this to examine code generation for a particular variable. - // if (this.var.refId === '') { - // debugger - // } - this.visitEquation(this.var.eqnCtx) - // Either emit constant list code or a regular var assignment. - let formula = ` ${this.lhs} = ${this.exprCode};` - // Close the assignment loops. - this.subscriptLoopClosingCode = R.concat( - this.subscriptLoopClosingCode, - R.map(() => ` }`, dimNames) - ) - // Assemble code from each channel into final var code output. - return this.comments.concat(this.subscriptLoopOpeningCode, this.tmpVarCode, formula, this.subscriptLoopClosingCode) - } - // - // Helpers - // - currentVarName() { - let n = this.varNames.length - return n > 0 ? this.varNames[n - 1] : undefined - } - lookupName() { - // Convert a call name into a lookup name. - return canonicalName(this.currentFunctionName()).slice(1) - } - emit(text) { - if (this.currentArrayFunctionName) { - // Emit code to the array function code buffer if we are in an array function. - this.arrayFunctionCode += text - } else if (this.argIndexForFunctionName('_VECTOR_ELM_MAP') === 1) { - // Emit expression code in the second argument of VECTOR ELM MAP to vemOffset. - this.vemOffset += text - } else { - // Otherwise emit code to the expression code channel. - this.exprCode += text - } - } - cVarOrConst(expr) { - // Get either a constant or a var name in C format from a parse tree expression. - let value = expr.getText().trim() - if (value === ':NA:') { - return '_NA_' - } else { - let v = Model.varWithName(canonicalName(value)) - if (v) { - return v.varName - } else { - let d = parseFloat(value) - if (Number.isNaN(d)) { - d = 0 - } - return cdbl(d) - } - } - } - constValue(c) { - // Get a numeric value from a constant var name in model form. - // Return 0 if the value is not a numeric string or const variable. - let value = parseFloat(c) - if (!Number.isNaN(value)) { - return value - } - // Look up the value as a symbol name and return the const value. - value = 0 - let v = Model.varWithName(canonicalName(c)) - if (v && v.isConst()) { - value = parseFloat(v.modelFormula) - if (Number.isNaN(value)) { - value = 0 - } - } - return value - } - handleExcelWorkbook(fileOrTag, workbook, tab, dataKind, dataSource) { - // Return a `getCellValue` function for the given Excel workbook parsed from an XLS[X] file. - if (workbook) { - let sheet = workbook.Sheets[tab] - if (sheet) { - return (c, r) => { - let cell = sheet[XLSX.utils.encode_cell({ c, r })] - return cell != null ? cdbl(cell.v) : null - } - } else { - throw new Error(`Direct ${dataKind} worksheet ${tab} in ${dataSource} ${fileOrTag} not found`) - } - } else { - throw new Error(`Direct ${dataKind} workbook ${dataSource} ${fileOrTag} not found`) - } - } - handleCsvFile(file, dataPathname, tab, dataKind) { - // Return a `getCellValue` function for the given CSV file. - let data = readCsv(dataPathname, tab) - if (data) { - return (c, r) => { - let value = '0.0' - try { - value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null - } catch (error) { - console.error(`${error.message} in ${dataPathname}`) - } - return value - } - } else { - throw new Error(`Direct ${dataKind} file ${file} could not be read`) - } - } - handleExcelOrCsvFile(fileOrTag, tab, dataKind) { - // Return a `getCellValue` function that reads the CSV or XLS[X] content. - if (fileOrTag.startsWith('?')) { - // The file is a tag for an Excel file with data in the directData map. - let workbook = this.directData.get(fileOrTag) - return this.handleExcelWorkbook(fileOrTag, workbook, tab, dataKind, 'tagged') - } else { - // The file is a CSV or XLS[X] pathname. Read it now. - let dataPathname = path.resolve(this.modelDirname, fileOrTag) - if (dataPathname.toLowerCase().endsWith('csv')) { - return this.handleCsvFile(fileOrTag, dataPathname, tab, dataKind) - } else { - let workbook = readXlsx(dataPathname) - return this.handleExcelWorkbook(fileOrTag, workbook, tab, dataKind, 'file') - } - } - } - lookupDataNameGen(subscripts) { - // Construct a name for the static data array associated with a lookup variable. - return R.map(subscript => { - if (isDimension(subscript)) { - let i = this.loopIndexVars.index(subscript) - if (isTrivialDimension(subscript)) { - // When the dimension is trivial, we can simply emit e.g. `[i]` instead of `[_dim[i]]` - return `_${i}_` - } else { - return `_${subscript}_${i}_` - } - } else { - return `_${sub(subscript).value}_` - } - }, subscripts).join('') - } - lhsSubscriptGen(subscripts) { - // Construct C array subscripts from subscript names in the variable's normal order. - return R.map(subscript => { - if (isDimension(subscript)) { - let i = this.loopIndexVars.index(subscript) - if (isTrivialDimension(subscript)) { - // When the dimension is trivial, we can simply emit e.g. `[i]` instead of `[_dim[i]]` - return `[${i}]` - } else { - return `[${subscript}[${i}]]` - } - } else { - return `[${sub(subscript).value}]` - } - }, subscripts).join('') - } - rhsSubscriptGen(subscripts) { - // Normalize RHS subscripts. - try { - subscripts = normalizeSubscripts(subscripts) - } catch (e) { - console.error('ERROR: normalizeSubscripts failed in rhsSubscriptGen') - vlog('this.var.refId', this.var.refId) - vlog('this.currentVarName()', this.currentVarName()) - vlog('subscripts', subscripts) - throw e - } - // Get the loop index var name source. - let cSubscripts = R.map(rhsSub => { - if (isIndex(rhsSub)) { - // Return the index number for an index subscript. - return `[${sub(rhsSub).value}]` - } else { - // The subscript is a dimension. - // Get the loop index variable, matching the previously emitted for loop variable. - let i - if (this.markedDims.includes(rhsSub)) { - i = this.arrayIndexVars.index(rhsSub) - } else { - // Use the single index name for a separated variable if it exists. - let separatedIndexName = separatedVariableIndex(rhsSub, this.var, subscripts) - if (separatedIndexName) { - return `[${sub(separatedIndexName).value}]` - } - // See if we need to apply a mapping because the RHS dim is not found on the LHS. - let found = this.var.subscripts.findIndex(lhsSub => sub(lhsSub).family === sub(rhsSub).family) - if (found < 0) { - // Find the mapping from the RHS subscript to a LHS subscript. - for (let lhsSub of this.var.subscripts) { - if (hasMapping(rhsSub, lhsSub)) { - // console.error(`${this.var.refId} hasMapping ${rhsSub} → ${lhsSub}`); - i = this.loopIndexVars.index(lhsSub) - return `[__map${rhsSub}${lhsSub}[${i}]]` - } - } - } - // There is no mapping, so use the loop index for this dim family on the LHS. - i = this.loopIndexVars.index(rhsSub) - } - // Return the dimension and loop index for a dimension subscript. - if (isTrivialDimension(rhsSub)) { - // When the dimension is trivial, we can simply emit e.g. `[i]` instead of `[_dim[i]]` - return `[${i}]` - } else { - return `[${rhsSub}[${i}]]` - } - } - }, subscripts).join('') - return cSubscripts - } - vemSubscriptGen() { - // VECTOR ELM MAP replaces one subscript with a calculated vemOffset from a base index. - let subscripts = normalizeSubscripts(this.vemSubscripts) - let cSubscripts = R.map(rhsSub => { - if (isIndex(rhsSub)) { - // Emit the index vemOffset from VECTOR ELM MAP for the index subscript. - return `[${this.vemIndexDim}[(size_t)(${this.vemIndexBase} + ${this.vemOffset})]]` - } else { - let i = this.loopIndexVars.index(rhsSub) - return `[${rhsSub}[${i}]]` - } - }, subscripts).join('') - return cSubscripts - } - vsoSubscriptGen(subscripts) { - // _VECTOR_SORT_ORDER will iterate over the last subscript in its first arg. - let i = this.loopIndexVars.index(subscripts[0]) - if (subscripts.length > 1) { - this.vsoVarName += `[${subscripts[0]}[${i}]]` - i = this.loopIndexVars.index(subscripts[1]) - this.vsoTmpDimName = subscripts[1] - } else { - this.vsoTmpDimName = subscripts[0] - } - // Emit the tmp var subscript just after emitting the tmp var elsewhere. - this.emit(`[${this.vsoTmpDimName}[${i}]]`) - } - aaSubscriptGen(subscripts) { - // _ALLOCATE_AVAILABLE will iterate over the subscript in its first arg. - let i = this.loopIndexVars.index(subscripts[0]) - this.aaTmpDimName = subscripts[0] - // Emit the tmp var subscript just after emitting the tmp var elsewhere. - this.emit(`[${this.aaTmpDimName}[${i}]]`) - } - functionIsLookup() { - // See if the function name in the current call is actually a lookup. - // console.error(`isLookup ${this.lookupName()}`); - let v = Model.varWithName(this.lookupName()) - return v && v.isLookup() - } - generateLookup() { - // Construct the name of the data array, which is based on the associated lookup var name, - // with any subscripts tacked on the end. - const dataName = this.var.varName + '_data_' + this.lookupDataNameGen(this.var.subscripts) - if (this.mode === 'decl') { - // In decl mode, declare a static data array that will be used to create the associated `Lookup` - // at init time. Using static arrays is better for code size, helps us avoid creating a copy of - // the data in memory, and seems to perform much better when compiled to wasm when compared to the - // previous approach that used varargs + copying, especially on constrained (e.g. iOS) devices. - let data = R.reduce((a, p) => listConcat(a, `${cdbl(p[0])}, ${cdbl(p[1])}`, true), '', this.var.points) - return [`double ${dataName}[${this.var.points.length * 2}] = { ${data} };`] - } else if (this.mode === 'init-lookups') { - // In init mode, create the `Lookup`, passing in a pointer to the static data array declared earlier. - // TODO: Make use of the lookup range - if (this.var.points.length < 1) { - throw new Error(`ERROR: lookup size = ${this.var.points.length} in ${this.lhs}`) - } - return [` ${this.lhs} = __new_lookup(${this.var.points.length}, /*copy=*/false, ${dataName});`] - } else { - return [] - } - } - generateDirectDataInit() { - // If direct data exists for this variable, copy it from the workbook into one or more lookups. - let result = [] - if (this.mode === 'init-lookups') { - let { file, tab, timeRowOrCol, startCell } = this.var.directDataArgs - - // Create a function that reads the CSV or XLS[X] content - let getCellValue = this.handleExcelOrCsvFile(file, tab, 'data') - - // If the data was found, convert it to a lookup. - if (getCellValue) { - let indexNum = 0 - if (!R.isEmpty(this.var.separationDims)) { - // Generate a lookup for a separated index in the variable's dimension. - if (this.var.separationDims.length > 1) { - console.error(`WARNING: direct data variable ${this.var.varName} separated on more than one dimension`) - } - let dimName = this.var.separationDims[0] - for (let subscript of this.var.subscripts) { - if (sub(subscript).family === dimName) { - // Use the index value in the subscript family when that is the separation dimension. - indexNum = sub(subscript).value - break - } - if (sub(dimName).value.includes(subscript)) { - // Look up the index when the separation dimension is a subdimension. - indexNum = sub(dimName).value.indexOf(subscript) - break - } - } - } - result.push(this.generateDirectDataLookup(getCellValue, timeRowOrCol, startCell, indexNum)) - } - } - return result - } - generateDirectDataLookup(getCellValue, timeRowOrCol, startCell, indexNum) { - // Read a row or column of data as (time, value) pairs from the worksheet. - // The cell(c,r) function wraps data access by column and row. - let dataCol, dataRow, dataValue, timeCol, timeRow, timeValue, nextCell - let lookupData = '' - let lookupSize = 0 - let dataAddress = XLSX.utils.decode_cell(startCell.toUpperCase()) - dataCol = dataAddress.c - dataRow = dataAddress.r - if (dataCol < 0 || dataRow < 0) { - throw new Error( - `Failed to parse 'cell' argument for GET DIRECT {DATA,LOOKUPS} call for ${this.lhs}: ${startCell}` - ) - } - if (isNaN(parseInt(timeRowOrCol))) { - // Time values are in a column. - timeCol = XLSX.utils.decode_col(timeRowOrCol.toUpperCase()) - timeRow = dataRow - dataCol += indexNum - nextCell = () => { - dataRow++ - timeRow++ - } - } else { - // Time values are in a row. - timeCol = dataCol - timeRow = XLSX.utils.decode_row(timeRowOrCol) - dataRow += indexNum - nextCell = () => { - dataCol++ - timeCol++ - } - } - timeValue = getCellValue(timeCol, timeRow) - dataValue = getCellValue(dataCol, dataRow) - while (timeValue != null && dataValue != null) { - lookupData = listConcat(lookupData, `${timeValue}, ${dataValue}`, true) - lookupSize++ - nextCell() - dataValue = getCellValue(dataCol, dataRow) - timeValue = getCellValue(timeCol, timeRow) - } - if (lookupSize < 1) { - throw new Error(`ERROR: lookup size = ${lookupSize} in ${this.lhs}`) - } - return [` ${this.lhs} = __new_lookup(${lookupSize}, /*copy=*/true, (double[]){ ${lookupData} });`] - } - generateDirectConstInit() { - // Map zero, one, or two subscripts on the LHS in model order to a table of numbers in a CSV file. - // The subscripts may be indices to pick out a subset of the data. - let result = this.comments - let { file, tab, startCell } = this.var.directConstArgs - - // Create a function that reads the CSV or XLS[X] content - let getCellValue = this.handleExcelOrCsvFile(file, tab, 'constants') - if (getCellValue) { - // Get C subscripts in text form for the LHS in normal order. - let modelLHSReader = new ModelLHSReader() - modelLHSReader.read(this.var.modelLHS) - let modelDimNames = modelLHSReader.modelSubscripts.filter(s => isDimension(s)) - // Generate offsets from the start cell in the table corresponding to LHS indices. - let cellOffsets = [] - let cSubscripts = this.var.subscripts.map(s => (isDimension(s) ? sub(s).value : [s])) - let lhsIndexSubscripts = cartesianProductOf(cSubscripts) - // Find the table cell offset for each LHS index tuple. - for (let indexSubscripts of lhsIndexSubscripts) { - let entry = [null, null] - for (let i = 0; i < this.var.subscripts.length; i++) { - // LHS dimensions or indices in a separated dimension map to table cells. - let lhsSubscript = this.var.subscripts[i] - if (isDimension(lhsSubscript) || indexInSepDim(lhsSubscript, this.var)) { - // Consider the LHS index subscript at this position. - let indexSubscript = indexSubscripts[i] - let ind = sub(indexSubscript) - // Find the model subscript position corresponding to the LHS index subscript. - for (let iModelDim = 0; iModelDim < modelDimNames.length; iModelDim++) { - // Only fill an entry position once. - if (entry[iModelDim] === null) { - let modelDim = sub(modelDimNames[iModelDim]) - if (modelDim.family === ind.family) { - // Set the numeric index for the model dimension in the cell offset entry. - // Use the position within the dimension to map subdimensions onto cell offsets. - let pos = modelDim.value.indexOf(indexSubscript) - // Vectors use a 2D cell offset that maps to columns in the first row. - // Tables use a 2D cell offset with the row or column matching the model dimension. - let entryRowOrCol = modelDimNames.length > 1 ? iModelDim : 1 - entry[entryRowOrCol] = pos - break - } - } - } - } - } - // Replace unfilled entry positions with zero. - entry = entry.map(x => (x === null ? 0 : x)) - // Read values by column first when the start cell ends with an asterisk. - // Ref: https://www.vensim.com/documentation/fn_get_direct_constants.html - if (startCell.endsWith('*')) { - entry.reverse() - } - cellOffsets.push(entry) - } - // Read tabular data into an indexed variable for each cell. - let numericSubscripts = lhsIndexSubscripts.map(idx => idx.map(s => sub(s).value)) - let lhsSubscripts = numericSubscripts.map(s => s.reduce((a, v) => a.concat(`[${v}]`), '')) - let dataAddress = XLSX.utils.decode_cell(startCell.toUpperCase()) - let startCol = dataAddress.c - let startRow = dataAddress.r - if (startCol < 0 || startRow < 0) { - throw new Error(`Failed to parse 'cell' argument for GET DIRECT CONSTANTS call for ${this.lhs}: ${startCell}`) - } - for (let i = 0; i < cellOffsets.length; i++) { - let rowOffset = cellOffsets[i][0] ? cellOffsets[i][0] : 0 - let colOffset = cellOffsets[i][1] ? cellOffsets[i][1] : 0 - let dataValue = getCellValue(startCol + colOffset, startRow + rowOffset) - let lhs = `${this.var.varName}${lhsSubscripts[i] || ''}` - result.push(` ${lhs} = ${dataValue};`) - } - } - return result - } - generateExternalDataInit() { - // If there is external data for this variable, copy it from an external file to a lookup. - // Just like in generateLookup(), we declare static arrays to hold the data points in the first pass - // ("decl" mode), then initialize each `Lookup` using that data in the second pass ("init" mode). - const mode = this.mode - - const newLookup = (name, lhs, data, subscriptIndexes) => { - if (!data) { - throw new Error(`ERROR: Data for ${name} not found in external data sources`) - } - - const dataName = this.var.varName + '_data_' + R.map(i => `_${i}_`, subscriptIndexes).join('') - if (mode === 'decl') { - // In decl mode, declare a static data array that will be used to create the associated `Lookup` - // at init time. See `generateLookup` for more details. - const points = R.reduce( - (a, p) => listConcat(a, `${cdbl(p[0])}, ${cdbl(p[1])}`, true), - '', - Array.from(data.entries()) - ) - return `double ${dataName}[${data.size * 2}] = { ${points} };` - } else if (mode === 'init-lookups') { - // In init mode, create the `Lookup`, passing in a pointer to the static data array declared in decl mode. - if (data.size < 1) { - throw new Error(`ERROR: lookup size = ${data.size} in ${lhs}`) - } - return ` ${lhs} = __new_lookup(${data.size}, /*copy=*/false, ${dataName});` - } else { - return undefined - } - } - - // There are three common cases that we handle: - // - variable has no subscripts (C variable _thing = _thing from dat file) - // - variable has subscript(s) (C variable with index _thing[0] = _thing[_subscript] from dat file) - // - variable has dimension(s) (C variable in for loop, _thing[i] = _thing[_subscript_i] from dat file) - - if (!this.var.subscripts || this.var.subscripts.length === 0) { - // No subscripts - const data = this.extData.get(this.var.varName) - return [newLookup(this.var.varName, this.lhs, data, [])] - } - - if (this.var.subscripts.length === 1 && !isDimension(this.var.subscripts[0])) { - // There is exactly one subscript - const subscript = this.var.subscripts[0] - const nameInDat = `${this.var.varName}[${subscript}]` - const data = this.extData.get(nameInDat) - const subIndex = sub(subscript).value - return [newLookup(nameInDat, this.lhs, data, [subIndex])] - } - - if (!R.all(s => isDimension(s), this.var.subscripts)) { - // We don't yet handle the case where there are more than one subscript or a mix of - // subscripts and dimensions - // TODO: Remove this restriction - throw new Error(`ERROR: Data variable ${this.var.varName} has >= 2 subscripts; not yet handled`) - } - - // At this point, we know that we have one or more dimensions; compute all combinations - // of the dimensions that we will iterate over - const result = [] - const allDims = R.map(s => sub(s).value, this.var.subscripts) - const dimTuples = cartesianProductOf(allDims) - for (const dims of dimTuples) { - // Note: It appears that the dat file can have the subscripts in a different order - // than what SDE uses when declaring the C array. If we don't find data for one - // order, we try the other possible permutations. - const dimNamePermutations = permutationsOf(dims) - let nameInDat, data - for (const dimNames of dimNamePermutations) { - nameInDat = `${this.var.varName}[${dimNames.join(',')}]` - data = this.extData.get(nameInDat) - if (data) { - break - } - } - if (!data) { - // We currently treat this as a warning, not an error, since there can sometimes be - // datasets that are a sparse matrix, i.e., data is not defined for certain dimensions. - // For these cases, the lookup will not be initialized (the Lookup pointer will remain - // NULL, and any calls to `LOOKUP` will return `:NA:`. - if (mode === 'decl') { - console.error(`WARNING: Data for ${nameInDat} not found in external data sources`) - } - continue - } - - const subscriptIndexes = R.map(dim => sub(dim).value, dims) - const varSubscripts = R.map(index => `[${index}]`, subscriptIndexes).join('') - const lhs = `${this.var.varName}${varSubscripts}` - const lookup = newLookup(nameInDat, lhs, data, subscriptIndexes) - if (lookup) { - result.push(lookup) - } - } - return result - } - // - // Visitor callbacks - // - visitEquation(ctx) { - if (this.var.isData() && !R.isEmpty(this.var.points)) { - if (this.mode === 'init-lookups') { - // If the var already has lookup data points, use those instead of reading them from a file. - if (this.var.points.length < 1) { - throw new Error(`ERROR: lookup size = ${this.var.points.length} in ${this.var.refId}`) - } - let lookupData = R.reduce((a, p) => listConcat(a, `${cdbl(p[0])}, ${cdbl(p[1])}`, true), '', this.var.points) - this.emit(`__new_lookup(${this.var.points.length}, /*copy=*/true, (double[]){ ${lookupData} })`) - } - } else { - super.visitEquation(ctx) - } - } - visitCall(ctx) { - // Convert the function name from Vensim to C format and push it onto the function name stack. - // This maintains the name of the current function as its arguments are visited. - this.callStack.push({ fn: cFunctionName(ctx.Id().getText()) }) - let fn = this.currentFunctionName() - // Do not emit the function calls in init mode, only the init expression. - // Do emit function calls inside an init expression (with call stack length > 1). - if (this.var.hasInitValue && this.mode.startsWith('init') && this.callStack.length <= 1) { - super.visitCall(ctx) - this.callStack.pop() - } else if (fn === '_ELMCOUNT') { - // Replace the function with the value of its argument, emitted in visitVar. - super.visitCall(ctx) - this.callStack.pop() - } else if (isArrayFunction(fn)) { - // Capture the name of this array function (e.g. `SUM`). This should be used - // to determine if a subscripted variable is used inside of an expression - // passed to an array function, e.g.: - // SUM ( Variable[Dim] ) - // or - // SUM ( IF THEN ELSE ( Variable[Dim], ... ) ) - // In the first example, when `Variable` is evaluated, both `currentFunctionName` - // and `currentArrayFunctionName` will be `SUM`. But in the second case, when - // `Variable` is evaluated, `currentFunctionName` will be `IF THEN ELSE` but - // `currentArrayFunctionName` will be `SUM`. A non-empty `currentArrayFunctionName` - // is an indication that a loop needs to be generated. - this.currentArrayFunctionName = fn - // Generate a loop that evaluates array functions inline. - // Collect information and generate the argument expression into the array function code buffer. - super.visitCall(ctx) - // Start a temporary variable to hold the result of the array function. - let condVar - let initValue = '0.0' - if (fn === '_VECTOR_SELECT') { - initValue = this.vsAction === 3 ? '-DBL_MAX' : '0.0' - condVar = newTmpVarName() - this.tmpVarCode.push(` bool ${condVar} = false;`) - } else if (fn === '_VMIN') { - initValue = 'DBL_MAX' - } else if (fn === '_VMAX') { - initValue = '-DBL_MAX' - } - let tmpVar = newTmpVarName() - this.tmpVarCode.push(` double ${tmpVar} = ${initValue};`) - // Emit the array function loop opening into the tmp var channel. - for (let markedDim of this.markedDims) { - let n - try { - n = sub(markedDim).size - } catch (e) { - console.error(`ERROR: marked dimension "${markedDim}" not found in var ${this.var.refId}`) - throw e - } - let i = this.arrayIndexVars.index(markedDim) - this.tmpVarCode.push(` for (size_t ${i} = 0; ${i} < ${n}; ${i}++) {`) - } - // Emit the body of the array function loop. - if (fn === '_VECTOR_SELECT') { - this.tmpVarCode.push(` if (bool_cond(${this.vsSelectionArray})) {`) - } - if (fn === '_SUM' || (fn === '_VECTOR_SELECT' && this.vsAction === 0)) { - this.tmpVarCode.push(` ${tmpVar} += ${this.arrayFunctionCode};`) - } else if (fn === '_VMIN') { - this.tmpVarCode.push(` ${tmpVar} = fmin(${tmpVar}, ${this.arrayFunctionCode});`) - } else if (fn === '_VMAX' || (fn === '_VECTOR_SELECT' && this.vsAction === 3)) { - this.tmpVarCode.push(` ${tmpVar} = fmax(${tmpVar}, ${this.arrayFunctionCode});`) - } - if (fn === '_VECTOR_SELECT') { - this.tmpVarCode.push(` ${condVar} = true;`) - this.tmpVarCode.push(' }') - } - // Close the array function loops. - for (let i = 0; i < this.markedDims.length; i++) { - this.tmpVarCode.push(` }`) - } - this.callStack.pop() - // Reset state variables that were set down in the parse tree. - this.markedDims = [] - this.arrayFunctionCode = '' - this.currentArrayFunctionName = '' - // Emit the temporary variable into the formula expression in place of the SUM call. - if (fn === '_VECTOR_SELECT') { - this.emit(`${condVar} ? ${tmpVar} : ${this.vsNullValue}`) - } else { - this.emit(tmpVar) - } - } else if (fn === '_VECTOR_ELM_MAP') { - super.visitCall(ctx) - this.callStack.pop() - this.emit(`${this.vemVarName}${this.vemSubscriptGen()}`) - this.vemVarName = '' - this.vemSubscripts = [] - this.vemIndexDim = '' - this.vemIndexBase = 0 - this.vemOffset = '' - } else if (fn === '_VECTOR_SORT_ORDER') { - super.visitCall(ctx) - let dimSize = sub(this.vsoTmpDimName).size - let vso = ` double* ${this.vsoTmpName} = _VECTOR_SORT_ORDER(${this.vsoVarName}, ${dimSize}, ${this.vsoOrder});` - // Inject the VSO call into the loop opening code that was aleady emitted into that channel. - this.subscriptLoopOpeningCode.splice(this.var.subscripts.length - 1, 0, vso) - this.callStack.pop() - this.vsoVarName = '' - this.vsoOrder = '' - this.vsoTmpName = '' - this.vsoTmpDimName = '' - } else if (fn === '_ALLOCATE_AVAILABLE') { - super.visitCall(ctx) - let dimSize = sub(this.aaTmpDimName).size - let aa = ` double* ${this.aaTmpName} = _ALLOCATE_AVAILABLE(${this.aaRequestArray}, (double*)${this.aaPriorityArray}, ${this.aaAvailableResource}, ${dimSize});` - // Inject the AA call into the loop opening code that was aleady emitted into that channel. - this.subscriptLoopOpeningCode.splice(this.var.subscripts.length - 1, 0, aa) - this.callStack.pop() - this.aaRequestArray = '' - this.aaPriorityArray = '' - this.aaAvailableResource = '' - this.aaTmpName = '' - this.aaTmpDimName = '' - } else if (fn === '_GET_DATA_BETWEEN_TIMES') { - this.emit('_GET_DATA_BETWEEN_TIMES(') - super.visitCall(ctx) - this.emit(')') - this.callStack.pop() - } else if (this.functionIsLookup() || this.var.isData()) { - // A lookup has function syntax but lookup semantics. Convert the function call into a lookup call. - this.emit(`_LOOKUP(${this.lookupName()}, `) - super.visitCall(ctx) - this.emit(')') - this.callStack.pop() - } else if (fn === '_ACTIVE_INITIAL') { - // Only emit the eval-time initialization without the function call for ACTIVE INITIAL. - super.visitCall(ctx) - } else if (fn === '_IF_THEN_ELSE') { - // Conditional expressions are handled specially in `visitExprList`. - super.visitCall(ctx) - this.callStack.pop() - } else if (isSmoothFunction(fn)) { - // For smooth functions, replace the entire call with the expansion variable generated earlier. - let smoothVar = Model.varWithRefId(this.var.smoothVarRefId) - this.emit(smoothVar.varName) - this.emit(this.rhsSubscriptGen(smoothVar.subscripts)) - } else if (isTrendFunction(fn)) { - // For trend functions, replace the entire call with the expansion variable generated earlier. - let trendVar = Model.varWithRefId(this.var.trendVarName) - let rhsSubs = this.rhsSubscriptGen(trendVar.subscripts) - this.emit(`${this.var.trendVarName}${rhsSubs}`) - } else if (isNpvFunction(fn)) { - // For NPV functions, replace the entire call with the expansion variable generated earlier. - let npvVar = Model.varWithRefId(this.var.npvVarName) - let rhsSubs = this.rhsSubscriptGen(npvVar.subscripts) - this.emit(`${this.var.npvVarName}${rhsSubs}`) - } else if (isDelayFunction(fn)) { - // For delay functions, replace the entire call with the expansion variable generated earlier. - let delayVar = Model.varWithRefId(this.var.delayVarRefId) - let rhsSubs = this.rhsSubscriptGen(delayVar.subscripts) - this.emit(`(${delayVar.varName}${rhsSubs} / ${this.var.delayTimeVarName}${rhsSubs})`) - } else { - // Generate code for ordinary function calls here. - this.emit(fn) - this.emit('(') - super.visitCall(ctx) - this.emit(')') - this.callStack.pop() - } - } - visitExprList(ctx) { - let exprs = ctx.expr() - let fn = this.currentFunctionName() - // Split level functions into init and eval expressions. - if ( - fn === '_INTEG' || - fn === '_SAMPLE_IF_TRUE' || - fn === '_ACTIVE_INITIAL' || - fn === '_DELAY_FIXED' || - fn === '_DEPRECIATE_STRAIGHTLINE' - ) { - if (this.mode.startsWith('init')) { - // Get the index of the argument holding the initial value. - let i = 0 - if (fn === '_INTEG' || fn === '_ACTIVE_INITIAL') { - i = 1 - } else if (fn === '_SAMPLE_IF_TRUE' || fn === '_DELAY_FIXED') { - i = 2 - } else if (fn === '_DEPRECIATE_STRAIGHTLINE') { - i = 3 - } - this.setArgIndex(i) - exprs[i].accept(this) - // For DELAY FIXED and DEPRECIATE STRAIGHTLINE, also initialize the support struct - // out of band, as they are not Vensim vars. - if (fn === '_DELAY_FIXED') { - let fixedDelay = `${this.var.fixedDelayVarName}${this.lhsSubscriptGen(this.var.subscripts)}` - this.emit(`;\n ${fixedDelay} = __new_fixed_delay(${fixedDelay}, `) - this.setArgIndex(1) - exprs[1].accept(this) - this.emit(', ') - this.setArgIndex(2) - exprs[2].accept(this) - this.emit(')') - } else if (fn === '_DEPRECIATE_STRAIGHTLINE') { - let depreciation = `${this.var.depreciationVarName}${this.lhsSubscriptGen(this.var.subscripts)}` - this.emit(`;\n ${depreciation} = __new_depreciation(${depreciation}, `) - this.setArgIndex(1) - exprs[1].accept(this) - this.emit(', ') - this.setArgIndex(2) - exprs[3].accept(this) - this.emit(')') - } - } else { - // We are in eval mode, not init mode. - if (fn === '_ACTIVE_INITIAL') { - // For ACTIVE INITIAL, emit the first arg without a function call. - this.setArgIndex(0) - exprs[0].accept(this) - } else if (fn === '_DELAY_FIXED') { - // For DELAY FIXED, emit the first arg followed by the FixedDelay support var. - this.setArgIndex(0) - exprs[0].accept(this) - this.emit(', ') - this.emit(`${this.var.fixedDelayVarName}${this.lhsSubscriptGen(this.var.subscripts)}`) - } else if (fn === '_DEPRECIATE_STRAIGHTLINE') { - // For DEPRECIATE STRAIGHTLINE, emit the first arg followed by the Depreciation support var. - this.setArgIndex(0) - exprs[0].accept(this) - this.emit(', ') - this.emit(`${this.var.depreciationVarName}${this.lhsSubscriptGen(this.var.subscripts)}`) - } else { - // Emit the variable LHS as the first arg at eval time, giving the current value for the level. - this.emit(this.lhs) - this.emit(', ') - // Emit the remaining arguments by visiting each expression in the list. - this.setArgIndex(0) - exprs[0].accept(this) - if (fn === '_SAMPLE_IF_TRUE') { - this.emit(', ') - this.setArgIndex(1) - exprs[1].accept(this) - } - } - } - } else if (fn === '_VECTOR_SELECT') { - this.setArgIndex(0) - exprs[0].accept(this) - this.setArgIndex(1) - exprs[1].accept(this) - this.setArgIndex(2) - this.vsNullValue = this.cVarOrConst(exprs[2]) - // TODO implement other actions besides just sum and max - this.setArgIndex(3) - this.vsAction = this.constValue(exprs[3].getText().trim()) - // TODO obey the error handling instruction here - this.setArgIndex(4) - this.vsError = this.cVarOrConst(exprs[4]) - } else if (fn === '_VECTOR_ELM_MAP') { - this.setArgIndex(0) - exprs[0].accept(this) - this.setArgIndex(1) - exprs[1].accept(this) - } else if (fn === '_VECTOR_SORT_ORDER') { - this.setArgIndex(0) - exprs[0].accept(this) - this.setArgIndex(1) - this.vsoOrder = this.cVarOrConst(exprs[1]) - } else if (fn === '_ALLOCATE_AVAILABLE') { - this.setArgIndex(0) - exprs[0].accept(this) - this.setArgIndex(1) - exprs[1].accept(this) - this.setArgIndex(2) - this.aaAvailableResource = this.cVarOrConst(exprs[2]) - } else if (fn === '_IF_THEN_ELSE') { - // See if the condition expression was previously determined to resolve to a - // compile-time constant. If so, we only need to emit code for one branch. - const condText = ctx.expr(0).getText() - const condValue = Model.getConstantExprValue(condText) - if (condValue !== undefined) { - this.emit('(') - if (condValue !== 0) { - // Emit only the "if true" branch - this.setArgIndex(1) - ctx.expr(1).accept(this) - } else { - // Emit only the "if false" branch - this.setArgIndex(2) - ctx.expr(2).accept(this) - } - this.emit(')') - } else { - // Emit a normal if/else with both branches - this.emit(fn) - this.emit('(') - for (let i = 0; i < exprs.length; i++) { - if (i > 0) this.emit(', ') - this.setArgIndex(i) - exprs[i].accept(this) - } - this.emit(')') - } - } else { - // Ordinary expression lists are completely emitted with comma delimiters. - for (let i = 0; i < exprs.length; i++) { - if (i > 0) this.emit(', ') - this.setArgIndex(i) - exprs[i].accept(this) - } - } - } - visitVar(ctx) { - // Helper function that emits a lookup call if the variable is a data variable, - // otherwise emits a normal variable. - const emitVar = () => { - let v = Model.varWithName(this.currentVarName()) - if (v && v.varType === 'data') { - this.emit(`_LOOKUP(${this.currentVarName()}`) - super.visitVar(ctx) - this.emit(', _time)') - } else { - this.emit(this.currentVarName()) - super.visitVar(ctx) - } - } - - // Push the var name on the stack and then emit it. - let id = ctx.Id().getText() - let varName = canonicalName(id) - let functionName = this.currentFunctionName() - if (isDimension(varName)) { - if (functionName === '_ELMCOUNT') { - // Emit the size of the dimension in place of the dimension name. - this.emit(`${sub(varName).size}`) - } else { - // A dimension masquerading as a variable (i.e., in expression position) takes the - // value of the loop index var plus one (since Vensim indices are one-based). - let s = this.rhsSubscriptGen([varName]) - // Remove the brackets around the C subscript expression. - s = s.slice(1, s.length - 1) - this.emit(`(${s} + 1)`) - } - } else if (isIndex(varName)) { - // A subscript masquerading as a variable (i.e., in expression position) takes the - // numeric index value plus one (since Vensim indices are one-based). - const index = sub(varName).value - this.emit(`${index + 1}`) - } else { - this.varNames.push(varName) - if (functionName === '_VECTOR_SELECT') { - let argIndex = this.argIndexForFunctionName(functionName) - if (argIndex === 0) { - this.vsSelectionArray = this.currentVarName() - super.visitVar(ctx) - } else if (argIndex === 1) { - emitVar() - } else { - super.visitVar(ctx) - } - } else if (functionName === '_VECTOR_ELM_MAP') { - if (this.argIndexForFunctionName(functionName) === 1) { - this.vemOffset = this.currentVarName() - } - super.visitVar(ctx) - } else if (functionName === '_VECTOR_SORT_ORDER') { - if (this.argIndexForFunctionName(functionName) === 0) { - this.vsoVarName = this.currentVarName() - this.vsoTmpName = newTmpVarName() - this.emit(this.vsoTmpName) - } - super.visitVar(ctx) - } else if (functionName === '_ALLOCATE_AVAILABLE') { - if (this.argIndexForFunctionName(functionName) === 0) { - this.aaRequestArray = this.currentVarName() - this.aaTmpName = newTmpVarName() - this.emit(this.aaTmpName) - } else if (this.argIndexForFunctionName(functionName) === 1) { - this.aaPriorityArray = this.currentVarName() - } - super.visitVar(ctx) - } else if (functionName === '_GET_DATA_BETWEEN_TIMES') { - this.emit(this.currentVarName()) - super.visitVar(ctx) - } else if ( - functionName === '_LOOKUP_FORWARD' || - functionName === '_LOOKUP_BACKWARD' || - functionName === '_LOOKUP_INVERT' - ) { - let argIndex = this.argIndexForFunctionName(functionName) - if (argIndex === 0) { - this.emit(this.currentVarName()) - super.visitVar(ctx) - } else { - emitVar() - } - } else { - emitVar() - } - this.varNames.pop() - } - } - visitLookupArg() { - // Substitute the previously generated lookup arg var name into the expression. - if (this.var.lookupArgVarName) { - this.emit(this.var.lookupArgVarName) - } - } - visitLookupCall(ctx) { - // Make a lookup argument into a _LOOKUP function call. - let id = ctx.Id().getText() - this.varNames.push(canonicalName(id)) - this.emit(`_LOOKUP(${canonicalName(id)}`) - // Emit subscripts after the var name, if any. - super.visitLookupCall(ctx) - this.emit(', ') - ctx.expr().accept(this) - this.emit(')') - this.varNames.pop() - } - visitSubscriptList(ctx) { - // Emit subscripts for a variable occurring on the RHS. - if (ctx.parentCtx.ruleIndex === ModelParser.RULE_expr) { - let subscripts = R.map(id => canonicalName(id.getText()), ctx.Id()) - let mergeMarkedDims = () => { - // Extract all marked dimensions and update subscripts. - let dims = extractMarkedDims(subscripts) - // Merge marked dims that were found into the list for this call. - this.markedDims = R.uniq(R.concat(this.markedDims, dims)) - } - let fn = this.currentFunctionName() - let arrayFn = this.currentArrayFunctionName - if (arrayFn === '_SUM' || arrayFn === '_VMIN' || arrayFn === '_VMAX') { - mergeMarkedDims() - this.emit(this.rhsSubscriptGen(subscripts)) - } else if (arrayFn === '_VECTOR_SELECT') { - let argIndex = this.argIndexForFunctionName('_VECTOR_SELECT') - if (argIndex === 0) { - mergeMarkedDims() - this.vsSelectionArray += this.rhsSubscriptGen(subscripts) - } else if (argIndex === 1) { - mergeMarkedDims() - this.emit(this.rhsSubscriptGen(subscripts)) - } - } else if (fn === '_VECTOR_ELM_MAP') { - if (this.argIndexForFunctionName('_VECTOR_ELM_MAP') === 0) { - this.vemVarName = this.currentVarName() - // Gather information from the argument to generate code later. - // The marked dim is an index in the vector argument. - this.vemSubscripts = subscripts - for (let subscript of subscripts) { - if (isIndex(subscript)) { - let ind = sub(subscript) - this.vemIndexDim = ind.family - this.vemIndexBase = ind.value - break - } - } - } else { - // Add subscripts to the offset argument. - this.vemOffset += this.rhsSubscriptGen(subscripts) - } - } else if (fn === '_VECTOR_SORT_ORDER') { - if (this.argIndexForFunctionName('_VECTOR_SORT_ORDER') === 0) { - this.vsoSubscriptGen(subscripts) - } - } else if (fn === '_ALLOCATE_AVAILABLE') { - if (this.argIndexForFunctionName('_ALLOCATE_AVAILABLE') === 0) { - this.aaSubscriptGen(subscripts) - } - } else { - // Add C subscripts to the variable name that was already emitted. - this.emit(this.rhsSubscriptGen(subscripts)) - } - } - } - visitConstList(ctx) { - let emitConstAtPos = i => { - this.emit(strToConst(exprs[i].getText())) - } - let exprs = ctx.expr() - // console.error(`visitConstList ${this.var.refId} ${exprs.length} exprs`) - if (exprs.length === 1) { - // Emit a single constant into the expression code. - emitConstAtPos(0) - } else { - // All const lists with > 1 value are separated on dimensions in the LHS. - // The LHS of a separated variable here will contain only index subscripts in normal order. - // Calculate an index into a flattened array by converting the indices to numeric form and looking them up - // in a C name array listed in the same Vensim order as the constant array in the model. - let modelLHSReader = new ModelLHSReader() - modelLHSReader.read(this.var.modelLHS) - let cNames = modelLHSReader.names().map(Model.cName) - let cVarName = this.var.varName + R.map(indName => `[${sub(indName).value}]`, this.var.subscripts).join('') - // Find the position of the constant in Vensim order from the expanded LHS var list. - let constPos = R.indexOf(cVarName, cNames) - if (constPos >= 0) { - emitConstAtPos(constPos) - // console.error(`${this.var.refId} position = ${constPos}`) - } else { - console.error(`ERROR: const list element ${this.var.refId} → ${cVarName} not found in C names`) - } - } - } - // - // Operators, etc. - // - visitNegative(ctx) { - this.emit('-') - super.visitNegative(ctx) - } - visitNot(ctx) { - this.emit('!') - super.visitNot(ctx) - } - visitPower(ctx) { - this.emit('pow(') - ctx.expr(0).accept(this) - this.emit(', ') - ctx.expr(1).accept(this) - this.emit(')') - } - visitMulDiv(ctx) { - ctx.expr(0).accept(this) - if (ctx.op.type === ModelLexer.Star) { - this.emit(' * ') - } else { - this.emit(' / ') - } - ctx.expr(1).accept(this) - } - visitAddSub(ctx) { - ctx.expr(0).accept(this) - if (ctx.op.type === ModelLexer.Plus) { - this.emit(' + ') - } else { - this.emit(' - ') - } - ctx.expr(1).accept(this) - } - visitRelational(ctx) { - ctx.expr(0).accept(this) - if (ctx.op.type === ModelLexer.Less) { - this.emit(' < ') - } else if (ctx.op.type === ModelLexer.Greater) { - this.emit(' > ') - } else if (ctx.op.type === ModelLexer.LessEqual) { - this.emit(' <= ') - } else { - this.emit(' >= ') - } - ctx.expr(1).accept(this) - } - visitEquality(ctx) { - ctx.expr(0).accept(this) - if (ctx.op.type === ModelLexer.Equal) { - this.emit(' == ') - } else { - this.emit(' != ') - } - ctx.expr(1).accept(this) - } - visitAnd(ctx) { - ctx.expr(0).accept(this) - this.emit(' && ') - ctx.expr(1).accept(this) - } - visitOr(ctx) { - ctx.expr(0).accept(this) - this.emit(' || ') - ctx.expr(1).accept(this) - } - visitKeyword(ctx) { - var keyword = ctx.Keyword().getText() - if (keyword === ':NA:') { - keyword = '_NA_' - } else if (keyword === ':INTERPOLATE:') { - keyword = '' - } - this.emit(keyword) - } - visitConst(ctx) { - let c = ctx.Const().getText() - this.emit(strToConst(c)) - } - visitParens(ctx) { - this.emit('(') - super.visitParens(ctx) - this.emit(')') - } -} diff --git a/packages/compile/src/generate/gen-equation-c.spec.ts b/packages/compile/src/generate/gen-equation-c.spec.ts index 44c0b158..4ba697f0 100644 --- a/packages/compile/src/generate/gen-equation-c.spec.ts +++ b/packages/compile/src/generate/gen-equation-c.spec.ts @@ -7,7 +7,6 @@ import { resetSubscriptsAndDimensions } from '../_shared/subscript' import Model from '../model/model' // import { default as VariableImpl } from '../model/variable' -import EquationGen from './equation-gen' import { parseInlineVensimModel, sampleModelDir, type Variable } from '../_tests/test-support' import { generateEquation } from './gen-equation' @@ -74,18 +73,7 @@ function genC( } } - let lines: string[] - if (process.env.SDE_NONPUBLIC_USE_NEW_PARSE !== '0') { - lines = generateEquation(variable, mode, opts?.extData, directData, opts?.modelDir) - } else { - // TODO: The `flat` call is only needed because the legacy EquationGen adds a nested array unnecessarily - // in `generateDirectDataInit` - lines = new EquationGen(variable, opts?.extData, directData, mode, opts?.modelDir).generate().flat() - // XXX: The legacy EquationGen sometimes appends code with a line break instead of returning a separate - // string for each line, so for now, split on newlines and treat them as separate lines for better - // compatibility with the new `generateEquation` - lines = lines.map(line => line.split('\n')).flat() - } + const lines = generateEquation(variable, mode, opts?.extData, directData, opts?.modelDir) // Strip the first comment line (containing the Vensim equation) if (lines.length > 0 && lines[0].trim().startsWith('//')) { From 6944361e76434ec6b2464f50a7974466318e16e1 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 11:41:53 -0700 Subject: [PATCH 02/15] fix: delete model-lhs-reader.js --- .../compile/src/generate/expand-var-names.js | 10 --- .../compile/src/generate/model-lhs-reader.js | 88 ------------------- 2 files changed, 98 deletions(-) delete mode 100644 packages/compile/src/generate/model-lhs-reader.js diff --git a/packages/compile/src/generate/expand-var-names.js b/packages/compile/src/generate/expand-var-names.js index 0f26841f..81ac4277 100644 --- a/packages/compile/src/generate/expand-var-names.js +++ b/packages/compile/src/generate/expand-var-names.js @@ -5,8 +5,6 @@ import { sub, isDimension } from '../_shared/subscript.js' import Model from '../model/model.js' -import ModelLHSReader from './model-lhs-reader.js' - /** * Return an array of names for all variable in the model, sorted alphabetically and expanded to * include the full set of subscripted variants for variables that include subscripts. @@ -43,14 +41,6 @@ export function expandVarNames(canonical) { * @returns {string[]} An array of expanded names for the given variable. */ function namesForVar(v) { - if (process.env.SDE_NONPUBLIC_USE_NEW_PARSE === '0') { - // TODO: When the old parsing code is active, use the old ModelLHSReader. This code path - // will be removed when the old parsing code is removed. - let modelLHSReader = new ModelLHSReader() - modelLHSReader.read(v.modelLHS) - return modelLHSReader.names() - } - if (v.parsedEqn === undefined) { // XXX: The special `Time` variable does not have a `parsedEqn`, so use the raw LHS return [v.modelLHS] diff --git a/packages/compile/src/generate/model-lhs-reader.js b/packages/compile/src/generate/model-lhs-reader.js deleted file mode 100644 index e91eccbc..00000000 --- a/packages/compile/src/generate/model-lhs-reader.js +++ /dev/null @@ -1,88 +0,0 @@ -import * as R from 'ramda' - -import { canonicalName } from '../_shared/helpers.js' -import { sub, isDimension } from '../_shared/subscript.js' -import ModelReader from '../parse/model-reader.js' -import { createParser } from '../parse/parser.js' - -// -// ModelLHSReader parses the LHS of a var in Vensim format and -// constructs a list of var names with indices for subscripted vars. -// -export default class ModelLHSReader extends ModelReader { - constructor() { - super() - this.varName = '' - this.modelLHSList = [] - this.modelSubscripts = [] - } - read(modelLHS) { - // Parse a model LHS and return the var name without subscripts. - // The names function may be called on this object to retrieve expanded subscript names. - let parser = createParser(modelLHS) - let tree = parser.lhs() - this.visitLhs(tree) - return this.varName - } - names() { - // Expand dimensions in a subscripted var into individual vars with indices. - if (this.modelLHSList.length > 0) { - return this.modelLHSList - } else { - return [this.varName] - } - } - visitLhs(ctx) { - let varName = ctx.Id().getText() - this.varName = varName - super.visitLhs(ctx) - } - visitSubscriptList(ctx) { - // Construct the modelLHSList array with the LHS expanded into an entry for each index - // in the same format as Vensim log files. - let subscripts = R.map(id => id.getText(), ctx.Id()) - this.modelSubscripts = subscripts.map(s => canonicalName(s)) - let modelLHSInds = dim => { - // Construct the model indices for a dimension. - // If the subscript range contains a dimension, expand it into index names in place. - let indNames = R.map(subscriptModelName => { - let subscript = canonicalName(subscriptModelName) - if (isDimension(subscript)) { - return sub(subscript).modelValue - } else { - return subscriptModelName - } - }, dim.modelValue) - return R.flatten(indNames) - } - let expandLHSDims = (a, subscripts) => { - // Recursively emit an LHS with Vensim names for each index in LHS dimensions. - // Accumulate expanded subscripts in the "a" variable. - if (subscripts.length === 0) { - this.modelLHSList.push(`${this.varName}[${a.join(',')}]`) - } else { - // Expand the first subscript into the accumulator. - let firstSub = canonicalName(R.head(subscripts)) - if (isDimension(firstSub)) { - // Emit each index in a dimension subscript. - for (let subscriptModelName of sub(firstSub).modelValue) { - if (isDimension(canonicalName(subscriptModelName))) { - // Expand a subdimension found in a dimension subscript value. - for (let ind of modelLHSInds(sub(canonicalName(subscriptModelName)))) { - expandLHSDims(a.concat(ind), R.tail(subscripts)) - } - } else { - // Expand an index subscript in a dimension directly. - expandLHSDims(a.concat(subscriptModelName), R.tail(subscripts)) - } - } - } else { - // Emit an index subscript directly. - expandLHSDims(a.concat(R.head(subscripts)), R.tail(subscripts)) - } - } - } - expandLHSDims([], subscripts) - return this.modelLHSList - } -} From 387c1cb8d2b92563ff400d30a4a88e8bfd3e8d7b Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 11:43:01 -0700 Subject: [PATCH 03/15] fix: delete var-name-reader.js --- packages/compile/src/model/model.js | 7 +--- packages/compile/src/model/var-name-reader.js | 40 ------------------- 2 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 packages/compile/src/model/var-name-reader.js diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index 2cffa03e..19a9a907 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -23,7 +23,6 @@ import { readVariables as readVariables2 } from './read-variables.js' import { reduceVariables } from './reduce-variables.js' import SubscriptRangeReader from './subscript-range-reader.js' import toposort from './toposort.js' -import VarNameReader from './var-name-reader.js' import Variable from './variable.js' import VariableReader from './variable-reader.js' @@ -799,11 +798,7 @@ function vensimName(cVarName) { function cName(vensimVarName) { // Convert a Vensim variable name to a C name. // This function requires model analysis to be completed first when the variable has subscripts. - if (process.env.SDE_NONPUBLIC_USE_NEW_PARSE === '0') { - // TODO: For now we use the legacy VarNameReader when the old parser is active; this - // code will be removed once the old parser is removed - return new VarNameReader().read(vensimVarName) - } + // Split the variable name from the subscripts let matches = vensimVarName.match(/([^[]+)(?:\[([^\]]+)\])?/) if (!matches) { diff --git a/packages/compile/src/model/var-name-reader.js b/packages/compile/src/model/var-name-reader.js deleted file mode 100644 index 574fbe55..00000000 --- a/packages/compile/src/model/var-name-reader.js +++ /dev/null @@ -1,40 +0,0 @@ -import * as R from 'ramda' - -import { canonicalName } from '../_shared/helpers.js' -import { sub, isIndex, normalizeSubscripts } from '../_shared/subscript.js' -import ModelReader from '../parse/model-reader.js' -import { createParser } from '../parse/parser.js' - -// -// VarNameReader reads a model var name using the parser to get the var name in C format. -// This is used to generate a variable output in the output section. -// -export default class VarNameReader extends ModelReader { - constructor() { - super() - this.varName = '' - } - read(modelVarName) { - // Parse an individual model var name and convert it into a a canonical C var name. - // Parse a single var name, which may include subscripts. - let parser = createParser(modelVarName) - let tree = parser.lhs() - // Generate and return the canonical name. - this.visitLhs(tree) - return this.varName - } - visitLhs(ctx) { - let varName = ctx.Id().getText() - this.varName = canonicalName(varName) - super.visitLhs(ctx) - } - visitSubscriptList(ctx) { - // Get the canonical form of subscripts found in the var name. - let subscripts = R.map(id => canonicalName(id.getText()), ctx.Id()) - subscripts = normalizeSubscripts(subscripts) - // If a subscript is an index, convert it to an index number to match Vensim data exports. - this.varName += R.map(subName => { - return isIndex(subName) ? `[${sub(subName).value}]` : `[${subName}]` - }, subscripts).join('') - } -} From cb8376d56b25c7bb09fb554794204b68cef4494f Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 11:46:40 -0700 Subject: [PATCH 04/15] fix: delete equation-reader.js and expr-reader.js --- packages/compile/src/model/equation-reader.js | 723 ------------------ packages/compile/src/model/expr-reader.js | 202 ----- packages/compile/src/model/model.js | 42 +- 3 files changed, 14 insertions(+), 953 deletions(-) delete mode 100644 packages/compile/src/model/equation-reader.js delete mode 100644 packages/compile/src/model/expr-reader.js diff --git a/packages/compile/src/model/equation-reader.js b/packages/compile/src/model/equation-reader.js deleted file mode 100644 index c27156d8..00000000 --- a/packages/compile/src/model/equation-reader.js +++ /dev/null @@ -1,723 +0,0 @@ -import { ModelParser } from 'antlr4-vensim' -import * as R from 'ramda' - -import { - canonicalName, - canonicalVensimName, - cFunctionName, - decanonicalize, - isDelayFunction, - isSeparatedVar, - isSmoothFunction, - isTrendFunction, - isNpvFunction, - matchRegex, - newAuxVarName, - newLevelVarName, - newLookupVarName, - newFixedDelayVarName, - newDepreciationVarName, - cartesianProductOf -} from '../_shared/helpers.js' -import { - extractMarkedDims, - indexNamesForSubscript, - isDimension, - isIndex, - normalizeSubscripts, - separatedVariableIndex, - sub -} from '../_shared/subscript.js' -import ModelReader from '../parse/model-reader.js' -import { createParser } from '../parse/parser.js' - -import ExprReader from './expr-reader.js' -import Model from './model.js' -import VariableReader from './variable-reader.js' - -// Set this true to get a list of functions used in the model. This may include lookups. -const PRINT_FUNCTION_NAMES = false - -export default class EquationReader extends ModelReader { - constructor(variable) { - super() - // variable that will be read - this.var = variable - // reference id constructed in parts - this.refId = '' - // list of reference ids filled when a dimension reference is expanded; overrides this.refId - this.expandedRefIds = [] - // flag that indicates the RHS has something other than a constant - this.rhsNonConst = false - } - read() { - // Fill in more information about the variable by analyzing the equation parse tree. - // Variables that were added programmatically do not have a parse tree context. - if (this.var.eqnCtx) { - this.visitEquation(this.var.eqnCtx) - } - // Refine the var type based on the contents of the equation. - if (this.var.points.length > 0) { - this.var.varType = 'lookup' - } else if (this.var.isAux() && !this.rhsNonConst) { - this.var.varType = 'const' - } - } - // - // Helpers - // - addReferencesToList(list) { - // Add reference ids gathered while walking the RHS parse tree to the variable's reference list. - let add = refId => { - // In Vensim a variable can refer to its current value in the state. - // Do not add self-references to the lists of references. - // Do not duplicate references. - if (refId !== this.var.refId && !list.includes(refId)) { - list.push(refId) - } - } - // Add expanded reference ids if they exist, otherwise, add the single reference id. - if (R.isEmpty(this.expandedRefIds)) { - add(this.refId) - } else { - this.expandedRefIds.forEach(refId => add(refId)) - } - } - // - // Visitor callbacks - // - visitCall(ctx) { - // Mark the RHS as non-constant, since it has a function call. - this.rhsNonConst = true - // Convert the function name from Vensim to C format. - let fn = cFunctionName(ctx.Id().getText()) - this.callStack.push({ fn: fn }) - if (PRINT_FUNCTION_NAMES) { - console.error(fn) - } - if (fn === '_INTEG' || fn === '_DELAY_FIXED') { - this.var.varType = 'level' - this.var.hasInitValue = true - if (fn === '_DELAY_FIXED') { - this.var.varSubtype = 'fixedDelay' - this.var.fixedDelayVarName = canonicalName(newFixedDelayVarName()) - } - } else if (fn === '_INITIAL') { - this.var.varType = 'initial' - this.var.hasInitValue = true - } else if (fn === '_ACTIVE_INITIAL' || fn === '_SAMPLE_IF_TRUE') { - this.var.hasInitValue = true - } else if (fn === '_GET_DIRECT_DATA' || fn === '_GET_DIRECT_LOOKUPS') { - this.var.varType = 'data' - } else if (fn === '_GET_DIRECT_CONSTANTS') { - this.var.varType = 'const' - } else if (fn === '_DEPRECIATE_STRAIGHTLINE') { - this.var.hasInitValue = true - this.var.varSubtype = 'depreciation' - this.var.depreciationVarName = canonicalName(newDepreciationVarName()) - } - super.visitCall(ctx) - this.callStack.pop() - } - visitExprList(ctx) { - let fn = this.currentFunctionName() - if (isSmoothFunction(fn)) { - // Generate a level var to expand the SMOOTH* call. - // TODO consider allowing more than one SMOOTH* call substitution - // Get SMOOTH* arguments for the function expansion. - let args = R.map(expr => expr.getText(), ctx.expr()) - this.expandSmoothFunction(fn, args) - } else if (isTrendFunction(fn)) { - // Generate a level var to expand the TREND call. - // Get TREND arguments for the function expansion. - let args = R.map(expr => expr.getText(), ctx.expr()) - let input = args[0] - let avgTime = args[1] - let init = args[2] - let level = this.expandTrendFunction(fn, args) - let genSubs = this.genSubs(input, avgTime, init) - let aux = newAuxVarName() - this.addVariable( - `${aux}${genSubs} = ZIDZ(${input} - ${level}${genSubs}, ${avgTime} * ABS(${level}${genSubs})) ~~|` - ) - this.var.trendVarName = canonicalName(aux) - this.var.references.push(this.var.trendVarName) - } else if (isNpvFunction(fn)) { - // Generate level vars to expand the NPV call. - // Get NPV arguments for the function expansion. - let args = R.map(expr => expr.getText(), ctx.expr()) - let stream = args[0] - let discountRate = args[1] - let initVal = args[2] - let factor = args[3] - let level = this.generateNpvLevels(stream, discountRate, initVal, factor) - let genSubs = this.genSubs(stream, discountRate, initVal, factor) - let aux = newAuxVarName() - // npv = (ncum + stream * TIME STEP * df) * factor - this.addVariable(`${aux}${genSubs} = (${level.ncum} + ${stream} * TIME STEP * ${level.df}) * ${factor} ~~|`) - this.var.npvVarName = canonicalName(aux) - this.var.references.push(this.var.npvVarName) - } else if (isDelayFunction(fn)) { - // Generate a level var to expand the DELAY* call. - let args = R.map(expr => expr.getText(), ctx.expr()) - this.expandDelayFunction(fn, args) - } else if (fn === '_GET_DIRECT_DATA' || fn === '_GET_DIRECT_LOOKUPS') { - // Extract string constant arguments into an object used in code generation. - // For Excel files, the file argument names an indirect "?" file tag from the model settings. - // For CSV files, it gives a relative pathname in the model directory. - // For Excel files, the tab argument names an Excel worksheet. - // For CSV files, it gives the delimiter character. - let args = R.map( - arg => matchRegex(arg, /'(.*)'/), - R.map(expr => expr.getText(), ctx.expr()) - ) - this.var.directDataArgs = { - file: args[0], - tab: args[1], - timeRowOrCol: args[2], - startCell: args[3] - } - } else if (fn === '_GET_DIRECT_CONSTANTS') { - // Extract string constant arguments into an object used in code generation. - // The file argument gives a relative pathname in the model directory. - // The tab argument gives the delimiter character. - let args = R.map( - arg => matchRegex(arg, /'(.*)'/), - R.map(expr => expr.getText(), ctx.expr()) - ) - this.var.directConstArgs = { - file: args[0], - tab: args[1], - startCell: args[2] - } - } else if (fn === '_IF_THEN_ELSE') { - if (process.env.SDE_NONPUBLIC_REDUCE_VARIABLES !== '0') { - // Evaluate the condition expression of the `IF THEN ELSE`. If it resolves - // to a compile-time constant, we only need to visit one branch, which means - // that no references will be recorded for the other branch, therefore allowing - // it to be skipped in the unused reference elimination phase and during the - // final code generation phase. - const condText = ctx.expr(0).getText() - const exprReader = new ExprReader() - const condExpr = exprReader.read(condText) - if (condExpr.constantValue !== undefined) { - // Record the conditional expression and its constant value so that - // it can be accessed later by EquationGen. We need to record it - // this way because any variables referenced by the expression may - // be removed during the unused reference elimination phase. - Model.addConstantExpr(condText, condExpr.constantValue) - if (condExpr.constantValue !== 0) { - // Only visit the "if true" branch - this.setArgIndex(1) - ctx.expr(1).accept(this) - } else { - // Only visit the "if false" branch - this.setArgIndex(2) - ctx.expr(2).accept(this) - } - } else { - // Visit the condition and both branches - super.visitExprList(ctx) - } - } else { - // Optimization is disabled, visit the condition and both branches - super.visitExprList(ctx) - } - } else { - // Keep track of all function names referenced in this expression. Note that lookup - // variables are sometimes function-like, so they will be included here. This will be - // used later to decide whether a lookup variable needs to be included in generated code. - const canonicalFnName = canonicalName(fn) - if (this.var.referencedFunctionNames) { - if (!this.var.referencedFunctionNames.includes(canonicalFnName)) { - this.var.referencedFunctionNames.push(canonicalFnName) - } - } else { - this.var.referencedFunctionNames = [canonicalFnName] - } - super.visitExprList(ctx) - } - } - visitVar(ctx) { - // Mark the RHS as non-constant, since it has a variable reference. - this.rhsNonConst = true - // Get the var name of a variable in a call and save it as a reference. - let id = ctx.Id().getText() - let varName = canonicalName(id) - // Do not add a dimension or index name as a reference. - if (!isDimension(varName) && !isIndex(varName)) { - let fn = this.currentFunctionName() - this.refId = varName - this.expandedRefIds = [] - super.visitVar(ctx) - // Separate init references from eval references in level formulas. - if (isSmoothFunction(fn) || isTrendFunction(fn) || isNpvFunction(fn) || isDelayFunction(fn)) { - // Do not set references inside the call, since it will be replaced - // with the generated level var. - } else if (this.argIndexForFunctionName('_INTEG') === 1) { - this.addReferencesToList(this.var.initReferences) - } else if (this.argIndexForFunctionName('_DELAY_FIXED') === 1) { - this.addReferencesToList(this.var.initReferences) - } else if (this.argIndexForFunctionName('_DELAY_FIXED') === 2) { - this.addReferencesToList(this.var.initReferences) - } else if (this.argIndexForFunctionName('_DEPRECIATE_STRAIGHTLINE') === 1) { - this.addReferencesToList(this.var.initReferences) - } else if (this.argIndexForFunctionName('_DEPRECIATE_STRAIGHTLINE') === 2) { - this.addReferencesToList(this.var.initReferences) - } else if (this.argIndexForFunctionName('_ACTIVE_INITIAL') === 1) { - this.addReferencesToList(this.var.initReferences) - } else if (this.argIndexForFunctionName('_SAMPLE_IF_TRUE') === 2) { - this.addReferencesToList(this.var.initReferences) - } else if (this.argIndexForFunctionName('_ALLOCATE_AVAILABLE') === 1) { - // Reference the second and third elements of the priority profile argument instead of the first one - // that Vensim requires for ALLOCATE AVAILABLE. This is required to get correct dependencies. - let ptypeRefId = this.expandedRefIds[0] - let { subscripts } = Model.splitRefId(ptypeRefId) - let ptypeIndexName = subscripts[1] - let profileElementsDimName = sub(ptypeIndexName).family - let profileElementsDim = sub(profileElementsDimName) - let priorityRefId = ptypeRefId.replace(ptypeIndexName, profileElementsDim.value[1]) - let widthRefId = ptypeRefId.replace(ptypeIndexName, profileElementsDim.value[2]) - this.expandedRefIds = [priorityRefId, widthRefId] - this.addReferencesToList(this.var.references) - } else if (this.var.isInitial()) { - this.addReferencesToList(this.var.initReferences) - } else { - this.addReferencesToList(this.var.references) - } - } - } - visitLookupCall(ctx) { - // Mark the RHS as non-constant, since it has a lookup. - this.rhsNonConst = true - // Keep track of the lookup variable that is referenced on the RHS. - const id = ctx.Id().getText() - const lookupVarName = canonicalName(id) - if (this.var.referencedLookupVarNames) { - this.var.referencedLookupVarNames.push(lookupVarName) - } else { - this.var.referencedLookupVarNames = [lookupVarName] - } - // Complete the visit. - ctx.expr().accept(this) - super.visitLookupCall(ctx) - } - visitLookupArg(ctx) { - // When a call argument is a lookup, generate a new lookup variable and save the variable name to emit later. - // TODO consider expanding this to more than one lookup arg per equation - const lookupArgVarName = this.generateLookupArg(ctx) - this.var.lookupArgVarName = lookupArgVarName - // Keep track of all lookup variables that are referenced. This will be used later to decide - // whether a lookup variable needs to be included in generated code. - if (this.var.referencedLookupVarNames) { - this.var.referencedLookupVarNames.push(lookupArgVarName) - } else { - this.var.referencedLookupVarNames = [lookupArgVarName] - } - } - visitSubscriptList(ctx) { - // When an equation references a non-appy-to-all array, add its subscripts to the array var's refId. - if (ctx.parentCtx.ruleIndex === ModelParser.RULE_expr) { - // Get the referenced var's subscripts in canonical form. - let subscripts = R.map(id => canonicalName(id.getText()), ctx.Id()) - // Remove dimension subscripts marked with ! and save them for later. - let markedDims = extractMarkedDims(subscripts) - subscripts = normalizeSubscripts(subscripts) - // console.error(`${this.var.refId} → ${this.refId} [ ${subscripts} ]`); - if (subscripts.length > 0) { - // See if this variable is non-apply-to-all. At this point, the refId is just the var name. - // References to apply-to-all variables do not need subscripts since they refer to the whole array. - let expansionFlags = Model.expansionFlags(this.refId) - if (expansionFlags) { - // The reference is to a non-apply-to-all variable. - // Find the refIds of the vars that include the indices in the reference. - // Get the vars with the var name of the reference. We will choose from these vars. - let varsWithRefName = Model.varsWithName(this.refId) - // The refIds of actual vars containing the indices will accumulate with possible duplicates. - let expandedRefIds = [] - let iSub - // Accumulate an array of lists of the separated index names at each position. - let indexNames = [] - for (iSub = 0; iSub < expansionFlags.length; iSub++) { - if (expansionFlags[iSub]) { - // For each index name at the subscript position, find refIds for vars that include the index. - // This process ensures that we generate references to vars that are in the var table. - let indexNamesAtPos - // Use the single index name for a separated variable if it exists. - // But don't do this if the subscript is a marked dimension in a vector function. - let separatedIndexName = separatedVariableIndex(subscripts[iSub], this.var, subscripts) - if (!markedDims.includes(subscripts[iSub]) && separatedIndexName) { - indexNamesAtPos = [separatedIndexName] - } else { - // Generate references to all the indices for the subscript. - indexNamesAtPos = indexNamesForSubscript(subscripts[iSub]) - } - indexNames.push(indexNamesAtPos) - } - } - // Flatten the arrays of index names at each position into an array of index name combinations. - let separatedIndices = cartesianProductOf(indexNames) - // Find a separated variable for each combination of indices. - for (let separatedIndex of separatedIndices) { - // Consider each var with the same name as the reference in the equation. - for (let refVar of varsWithRefName) { - let iSeparatedIndex = 0 - for (iSub = 0; iSub < expansionFlags.length; iSub++) { - if (expansionFlags[iSub]) { - let refVarIndexNames = indexNamesForSubscript(refVar.subscripts[iSub]) - if (refVarIndexNames.length === 0) { - console.error( - `ERROR: no subscript at subscript position ${iSub} for var ${refVar.refId} with subscripts ${refVar.subscripts}` - ) - } - if (!refVarIndexNames.includes(separatedIndex[iSeparatedIndex++])) { - break - } - } - } - if (iSub >= expansionFlags.length) { - // All separated index names matched index names in the var, so add it as a reference. - expandedRefIds.push(refVar.refId) - break - } - } - } - // Sort the expandedRefIds and eliminate duplicates. - this.expandedRefIds = R.uniq(expandedRefIds.sort()) - } - } - } - super.visitSubscriptList(ctx) - } - visitLookupRange(ctx) { - this.var.range = R.map(p => this.getPoint(p), ctx.lookupPoint()) - super.visitLookupRange(ctx) - } - visitLookupPointList(ctx) { - this.var.points = R.map(p => this.getPoint(p), ctx.lookupPoint()) - super.visitLookupPointList(ctx) - } - getPoint(lookupPoint) { - let exprs = lookupPoint.expr() - if (exprs.length >= 2) { - return [parseFloat(exprs[0].getText()), parseFloat(exprs[1].getText())] - } - } - generateLookupArg(lookupArgCtx) { - // Generate a variable for a lookup argument found in the RHS. - let varName = newLookupVarName() - let eqn = `${varName}${lookupArgCtx.getText()} ~~|` - this.addVariable(eqn) - return canonicalName(varName) - } - expandSmoothFunction(fn, args) { - // Generate variables for a SMOOTH* call found in the RHS. - let input = args[0] - let delay = args[1] - let init = args[2] !== undefined ? args[2] : args[0] - if (fn === '_SMOOTH' || fn === '_SMOOTHI') { - this.generateSmoothLevel(input, delay, init, 1) - } else if (fn === '_SMOOTH3' || fn === '_SMOOTH3I') { - let delay3 = `(${delay} / 3)` - let level1 = this.generateSmoothLevel(input, delay3, init, 1) - let level2 = this.generateSmoothLevel(level1, delay3, init, 2) - this.generateSmoothLevel(level2, delay3, init, 3) - } - } - generateSmoothLevel(input, delay, init, levelNumber) { - // Generate a level equation to implement SMOOTH. - // The parameters are model names. Return the canonical name of the generated level var. - let genSubs = this.genSubs(input, delay, init) - // For SMOOTH3, the previous level is the input for level number 2 and 3. Add RHS subscripts. - if (levelNumber > 1 && genSubs) { - input += genSubs - } - let level, levelLHS, levelRefId - if (isSeparatedVar(this.var)) { - // Levels generated by separated vars are also separated. We have to compute the indices here instead - // of using the dimension on the LHS and letting addVariable do it, so that the whole array of - // separated variables are not added for each visit here by an already-separated index. - // Start by getting a level var based on the var name, so it is the same for all separated levels. - level = newLevelVarName(this.var.varName, levelNumber) - // Replace the dimension in the generated variable subscript with the separated index from the LHS. - // Find the index in the LHS that was expanded from the separation dimension. - let index - let sepDim - let r = genSubs.match(/\[(.*)\]/) - if (r) { - let rhsSubs = r[1].split(',').map(x => canonicalName(x)) - for (let rhsSub of rhsSubs) { - let separatedIndexName = separatedVariableIndex(rhsSub, this.var, rhsSubs) - if (separatedIndexName) { - index = decanonicalize(separatedIndexName) - sepDim = decanonicalize(rhsSub) - break - } - } - } - // Use the Vensim form of the index in the LHS and in all arguments. - if (index) { - let re = new RegExp(`\\[(.*?)${sepDim}(.*?)\\]`, 'gi') - let replacement = `[$1${index}$2]` - let newGenSubs = genSubs.replace(re, replacement) - levelLHS = `${level}${newGenSubs}` - levelRefId = canonicalVensimName(levelLHS) - input = input.replace(re, replacement) - delay = delay.replace(re, replacement) - init = init.replace(re, replacement) - } - } else { - // In the normal case, generate a unique variable name for the level var. - level = newLevelVarName() - levelLHS = level + genSubs - // If it has subscripts, the refId is still just the var name, because it is an apply-to-all array. - levelRefId = canonicalName(level) - } - let eqn = `${levelLHS} = INTEG((${input} - ${levelLHS}) / ${delay}, ${init}) ~~|` - if (isSeparatedVar(this.var)) { - Model.addNonAtoAVar(canonicalName(level), [true]) - } - this.addVariable(eqn) - // Add a reference to the new level var. - // For SMOOTH3, the smoothVarRefId is the final level refId. - this.var.smoothVarRefId = levelRefId - this.refId = levelRefId - this.expandedRefIds = [] - this.addReferencesToList(this.var.references) - return level - } - expandTrendFunction(fn, args) { - // Generate variables for a TREND call found in the RHS. - let input = args[0] - let avgTime = args[1] - let init = args[2] - let level = this.generateTrendLevel(input, avgTime, init) - return level - } - generateTrendLevel(input, avgTime, init) { - // Generate a level equation to implement TREND. - // The parameters are model names. Return the canonical name of the generated level var. - let genSubs = this.genSubs(input, avgTime, init) - let level = newLevelVarName() - let levelLHS = level + genSubs - let eqn = `${levelLHS} = INTEG((${input} - ${levelLHS}) / ${avgTime}, ${input} / (1 + ${init} * ${avgTime})) ~~|` - this.addVariable(eqn) - // Add a reference to the new level var. - // If it has subscripts, the refId is still just the var name, because it is an apply-to-all array. - this.refId = canonicalName(level) - this.expandedRefIds = [] - this.addReferencesToList(this.var.references) - return level - } - generateNpvLevels(stream, discountRate, initVal, factor) { - // Generate two level equations to implement NPV. - // Return the canonical names of the generated level vars as object properties. - let genSubs = this.genSubs(stream, discountRate, initVal, factor) - // df = INTEG((-df * discount rate) / (1 + discount rate * TIME STEP), 1) - let df = newLevelVarName() - let dfLHS = df + genSubs - let dfEqn = `${dfLHS} = INTEG((-${dfLHS} * ${discountRate}) / (1 + ${discountRate} * TIME STEP), 1) ~~|` - this.addVariable(dfEqn) - // ncum = INTEG(stream * df, init val) - let ncum = newLevelVarName() - let ncumLHS = ncum + genSubs - let ncumEqn = `${ncumLHS} = INTEG(${stream} * ${dfLHS}, ${initVal}) ~~|` - this.addVariable(ncumEqn) - // Add references to the new level vars. - // If they have subscripts, the refIds are still just the var name, because they are apply-to-all arrays. - this.refId = '' - this.expandedRefIds = [canonicalName(ncum), canonicalName(df)] - this.addReferencesToList(this.var.references) - return { ncum, df } - } - expandDelayFunction(fn, args) { - // Generate variables for a DELAY* call found in the RHS. - let input = args[0] - let delay = args[1] - let genSubs = this.genSubs(this.var.modelLHS) - - if (fn === '_DELAY1' || fn === '_DELAY1I') { - let level, levelLHS, levelRefId - let init = `${args[2] !== undefined ? args[2] : args[0]} * ${delay}` - let varLHS = this.var.modelLHS - if (isSeparatedVar(this.var)) { - level = newLevelVarName(this.var.varName, 1) - let index - let sepDim - let r = genSubs.match(/\[(.*)\]/) - if (r) { - let rhsSubs = r[1].split(',').map(x => canonicalName(x)) - for (let rhsSub of rhsSubs) { - let separatedIndexName = separatedVariableIndex(rhsSub, this.var, rhsSubs) - if (separatedIndexName) { - index = decanonicalize(separatedIndexName) - sepDim = decanonicalize(rhsSub) - break - } - } - } - if (index) { - let re = new RegExp(sepDim, 'gi') - genSubs = genSubs.replace(re, index) - levelLHS = `${level}${genSubs}` - levelRefId = canonicalVensimName(levelLHS) - input = input.replace(re, index) - varLHS = varLHS.replace(re, index) - delay = delay.replace(re, index) - init = init.replace(re, index) - } - Model.addNonAtoAVar(canonicalName(level), [true]) - } else { - level = newLevelVarName() - levelLHS = level + genSubs - levelRefId = canonicalName(level) - } - // Generate a level var that will replace the DELAY function call. - this.var.delayVarRefId = this.generateDelayLevel(levelLHS, levelRefId, input, varLHS, init) - // Generate an aux var to hold the delay time expression. - let delayTimeVarName = newAuxVarName() - this.var.delayTimeVarName = canonicalName(delayTimeVarName) - if (isSeparatedVar(this.var)) { - Model.addNonAtoAVar(this.var.delayTimeVarName, [true]) - } - let delayTimeEqn = `${delayTimeVarName}${genSubs} = ${delay} ~~|` - this.addVariable(delayTimeEqn) - // Add a reference to the var, since it won't show up until code gen time. - this.var.references.push(canonicalVensimName(`${delayTimeVarName}${genSubs}`)) - } else if (fn === '_DELAY3' || fn === '_DELAY3I') { - let level1, level1LHS, level1RefId - let level2, level2LHS, level2RefId - let level3, level3LHS, level3RefId - let delay3 = `((${delay}) / 3)` - let init = `${args[2] !== undefined ? args[2] : args[0]} * ${delay3}` - let aux1, aux1LHS - let aux2, aux2LHS - let aux3, aux3LHS - let aux4, aux4LHS - if (isSeparatedVar(this.var)) { - level1 = newLevelVarName(this.var.varName, 1) - level2 = newLevelVarName(this.var.varName, 2) - level3 = newLevelVarName(this.var.varName, 3) - aux1 = newAuxVarName(this.var.varName, 1) - aux2 = newAuxVarName(this.var.varName, 2) - aux3 = newAuxVarName(this.var.varName, 3) - aux4 = newAuxVarName(this.var.varName, 4) - let index - let sepDim - let r = genSubs.match(/\[(.*)\]/) - if (r) { - let rhsSubs = r[1].split(',').map(x => canonicalName(x)) - for (let rhsSub of rhsSubs) { - let separatedIndexName = separatedVariableIndex(rhsSub, this.var, rhsSubs) - if (separatedIndexName) { - index = decanonicalize(separatedIndexName) - sepDim = decanonicalize(rhsSub) - break - } - } - } - if (index) { - let re = new RegExp(sepDim, 'gi') - genSubs = genSubs.replace(re, index) - level1LHS = `${level1}${genSubs}` - level2LHS = `${level2}${genSubs}` - level3LHS = `${level3}${genSubs}` - aux1LHS = `${aux1}${genSubs}` - aux2LHS = `${aux2}${genSubs}` - aux3LHS = `${aux3}${genSubs}` - aux4LHS = `${aux4}${genSubs}` - level1RefId = canonicalVensimName(level1LHS) - level2RefId = canonicalVensimName(level2LHS) - level3RefId = canonicalVensimName(level3LHS) - input = input.replace(re, index) - delay3 = delay3.replace(re, index) - init = init.replace(re, index) - } - Model.addNonAtoAVar(canonicalName(level1), [true]) - Model.addNonAtoAVar(canonicalName(level2), [true]) - Model.addNonAtoAVar(canonicalName(level3), [true]) - } else { - level1 = newLevelVarName() - level2 = newLevelVarName() - level3 = newLevelVarName() - aux1 = newAuxVarName() - aux2 = newAuxVarName() - aux3 = newAuxVarName() - aux4 = newAuxVarName() - level1LHS = level1 + genSubs - level2LHS = level2 + genSubs - level3LHS = level3 + genSubs - aux1LHS = aux1 + genSubs - aux2LHS = aux2 + genSubs - aux3LHS = aux3 + genSubs - aux4LHS = aux4 + genSubs - level1RefId = canonicalName(level1) - level2RefId = canonicalName(level2) - level3RefId = canonicalName(level3) - } - // Generate a level var that will replace the DELAY function call. - this.var.delayVarRefId = this.generateDelayLevel(level3LHS, level3RefId, aux2LHS, aux3LHS, init) - this.generateDelayLevel(level2LHS, level2RefId, aux1LHS, aux2LHS, init) - this.generateDelayLevel(level1LHS, level1RefId, input, aux1LHS, init) - // Generate equations for the aux vars using the subs in the generated level var. - this.addVariable(`${aux1LHS} = ${level1LHS} / ${delay3} ~~|`) - this.addVariable(`${aux2LHS} = ${level2LHS} / ${delay3} ~~|`) - this.addVariable(`${aux3LHS} = ${level3LHS} / ${delay3} ~~|`) - // Generate an aux var to hold the delay time expression. - this.var.delayTimeVarName = canonicalName(aux4) - if (isSeparatedVar(this.var)) { - Model.addNonAtoAVar(this.var.delayTimeVarName, [true]) - } - this.addVariable(`${aux4LHS} = ${delay3} ~~|`) - // Add a reference to the var, since it won't show up until code gen time. - this.var.references.push(canonicalVensimName(`${aux4}${genSubs}`)) - } - } - generateDelayLevel(levelLHS, levelRefId, input, aux, init) { - // Generate a level equation to implement DELAY. - // The parameters are model names. Return the refId of the generated level var. - let eqn = `${levelLHS} = INTEG(${input} - ${aux}, ${init}) ~~|` - this.addVariable(eqn) - // Add a reference to the new level var. - this.refId = levelRefId - this.expandedRefIds = [] - this.addReferencesToList(this.var.references) - return levelRefId - } - addVariable(modelEquation) { - let parser = createParser(modelEquation) - let tree = parser.equation() - // Read the var and add it to the Model var table. - let variableReader = new VariableReader() - variableReader.visitEquation(tree) - // Fill in the rest of the var, which may been expanded on a separation dim. - let generatedVars = variableReader.expandedVars.length > 0 ? variableReader.expandedVars : [variableReader.var] - R.forEach(v => { - // Fill in the refId. - v.refId = Model.refIdForVar(v) - // Inhibit output for generated variables. - v.includeInOutput = false - // Finish the variable by parsing the RHS. - let equationReader = new EquationReader(v) - equationReader.read() - }, generatedVars) - } - genSubs(...varNames) { - // Get the subscripts from one or more varnames. Check if they agree. - // This is used to get the subscripts for generated variables. - let result = new Set() - const re = /\[[^\]]+\]/g - for (let varName of varNames) { - let subs = varName.match(re) - if (subs) { - for (let sub of subs) { - result.add(sub.trim()) - } - } - } - if (result.size > 1) { - console.error(`ERROR: genSubs subscripts do not agree: ${[...varNames]}`) - } - return [...result][0] || '' - } -} diff --git a/packages/compile/src/model/expr-reader.js b/packages/compile/src/model/expr-reader.js deleted file mode 100644 index ed08d47c..00000000 --- a/packages/compile/src/model/expr-reader.js +++ /dev/null @@ -1,202 +0,0 @@ -import { ModelLexer, ModelVisitor } from 'antlr4-vensim' - -import { canonicalName } from '../_shared/helpers.js' -import { createParser } from '../parse/parser.js' - -import Model from './model.js' - -/** - * Reads an expression and determines if it resolves to a constant numeric value. - * This depends on having access to the model variables, so this should be used - * only after the `readVariables` process has been completed and the spec file - * has been loaded. - */ -export default class ExprReader extends ModelVisitor { - constructor() { - super() - } - - read(exprText) { - let parser = createParser(exprText) - let expr = parser.expr() - expr.accept(this) - - return { - constantValue: this.constantValue - } - } - - // Constants - - visitConst(ctx) { - const constantValue = parseFloat(ctx.Const().getText()) - if (!Number.isNaN(constantValue)) { - this.constantValue = constantValue - } else { - this.constantValue = undefined - } - } - visitConstList() { - this.constantValue = undefined - } - - // Function calls and variables - - visitCall() { - // Treat function calls as non-constant (can't easily determine if they - // resolve to a constant) - this.constantValue = undefined - } - visitExprList() { - // Treat function calls as non-constant (can't easily determine if they - // resolve to a constant) - this.constantValue = undefined - } - visitVar(ctx) { - // Determine whether this variable has a constant value - const varName = ctx.Id().getText().trim() - const cName = canonicalName(varName) - const v = Model.varWithName(cName) - const modelFormula = v?.modelFormula?.trim() || '' - const isNumber = modelFormula.match(/^[+-]?\d+(\.\d+)?$/) - const canBeOverridden = Model.isInputVar(cName) - if (v && isNumber && !canBeOverridden) { - const numValue = parseFloat(modelFormula) - if (!Number.isNaN(numValue)) { - this.constantValue = numValue - } else { - this.constantValue = undefined - } - } else { - this.constantValue = undefined - } - } - - // Lookups - - visitLookup() { - this.constantValue = undefined - } - visitLookupCall() { - this.constantValue = undefined - } - - // Unary operators - - visitNegative(ctx) { - super.visitNegative(ctx) - if (this.constantValue !== undefined) { - this.constantValue = -this.constantValue - } - } - visitPositive(ctx) { - super.visitPositive(ctx) - if (this.constantValue !== undefined) { - this.constantValue = +this.constantValue - } - } - visitNot(ctx) { - super.visitNot(ctx) - if (this.constantValue !== undefined) { - this.constantValue = !this.constantValue - } - } - - // Binary operators - - visitBinaryArgs(ctx, combine) { - ctx.expr(0).accept(this) - const constantValue0 = this.constantValue - ctx.expr(1).accept(this) - const constantValue1 = this.constantValue - if (constantValue0 !== undefined && constantValue1 !== undefined) { - this.constantValue = combine(constantValue0, constantValue1) - } else { - this.constantValue = undefined - } - } - - visitPower(ctx) { - this.visitBinaryArgs(ctx, (a, b) => Math.pow(a, b)) - } - visitMulDiv(ctx) { - this.visitBinaryArgs(ctx, (a, b) => { - if (ctx.op.type === ModelLexer.Star) { - return a * b - } else { - return a / b - } - }) - } - visitAddSub(ctx) { - this.visitBinaryArgs(ctx, (a, b) => { - if (ctx.op.type === ModelLexer.Plus) { - return a + b - } else { - return a - b - } - }) - } - visitRelational(ctx) { - this.visitBinaryArgs(ctx, (a, b) => { - if (ctx.op.type === ModelLexer.Less) { - return a < b ? 1 : 0 - } else if (ctx.op.type === ModelLexer.Greater) { - return a > b ? 1 : 0 - } else if (ctx.op.type === ModelLexer.LessEqual) { - return a <= b ? 1 : 0 - } else { - return a >= b ? 1 : 0 - } - }) - } - visitEquality(ctx) { - this.visitBinaryArgs(ctx, (a, b) => { - if (ctx.op.type === ModelLexer.Equal) { - return a === b ? 1 : 0 - } else { - return a !== b ? 1 : 0 - } - }) - } - visitAnd(ctx) { - // For AND expressions, we don't need both sides to have a constant value; as - // long as one side is known to be false, then the expression resolves to false - ctx.expr(0).accept(this) - const constantValue0 = this.constantValue - ctx.expr(1).accept(this) - const constantValue1 = this.constantValue - if (constantValue0 !== undefined && constantValue1 !== undefined) { - this.constantValue = constantValue0 && constantValue1 - } else if (constantValue0 !== undefined && !constantValue0) { - this.constantValue = 0 - } else if (constantValue1 !== undefined && !constantValue1) { - this.constantValue = 0 - } else { - this.constantValue = undefined - } - } - visitOr(ctx) { - // For OR expressions, we don't need both sides to have a constant value; as - // long as one side is known to be true, then the expression resolves to true - ctx.expr(0).accept(this) - const constantValue0 = this.constantValue - ctx.expr(1).accept(this) - const constantValue1 = this.constantValue - if (constantValue0 !== undefined && constantValue1 !== undefined) { - this.constantValue = constantValue0 || constantValue1 - } else if (constantValue0 !== undefined && constantValue0) { - this.constantValue = 1 - } else if (constantValue1 !== undefined && constantValue1) { - this.constantValue = 1 - } else { - this.constantValue = undefined - } - } - - // Tokens - - visitParens(ctx) { - super.visitParens(ctx) - } -} diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index 19a9a907..e44927d9 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -14,9 +14,7 @@ import { sub, subscriptFamilies } from '../_shared/subscript.js' -import { createParser } from '../parse/parser.js' -import EquationReader from './equation-reader.js' import { readEquation } from './read-equations.js' import { readDimensionDefs } from './read-subscripts.js' import { readVariables as readVariables2 } from './read-variables.js' @@ -332,11 +330,7 @@ function analyze(parsedModelKind, inputVars, opts) { if (opts?.stopAfterReduceVariables === true) return // Read the RHS to list the refIds of vars that are referenced and set the var type. - if (parsedModelKind === 'vensim-legacy') { - readEquations() - } else { - variables.forEach(readEquation) - } + variables.forEach(readEquation) } function checkSpecVars(spec, extData) { @@ -567,27 +561,20 @@ function setRefIds() { v.refId = refIdForVar(v) }, variables) } -function readEquations() { - // Augment variables with information from their equations. - // This requires a refId for each var so that actual refIds can be resolved for the reference list. - R.forEach(v => { - let equationReader = new EquationReader(v) - equationReader.read() - }, variables) -} function addEquation(modelEquation) { - // Add an equation in Vensim model format. - let parser = createParser(modelEquation) - let tree = parser.equation() - // Read the var and add it to the Model var table. - let variableReader = new VariableReader() - variableReader.visitEquation(tree) - let v = variableReader.var - // Fill in the refId. - v.refId = refIdForVar(v) - // Finish the variable by parsing the RHS. - let equationReader = new EquationReader(v) - equationReader.read() + // TODO + // // Add an equation in Vensim model format. + // let parser = createParser(modelEquation) + // let tree = parser.equation() + // // Read the var and add it to the Model var table. + // let variableReader = new VariableReader() + // variableReader.visitEquation(tree) + // let v = variableReader.var + // // Fill in the refId. + // v.refId = refIdForVar(v) + // // Finish the variable by parsing the RHS. + // let equationReader = new EquationReader(v) + // equationReader.read() } // // Model API @@ -1252,7 +1239,6 @@ function jsonList() { export default { addConstantExpr, - addEquation, addNonAtoAVar, addVariable, allVars, From 20e90078ecd4eb3d015283ca35066fcfc40c29da Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 11:50:25 -0700 Subject: [PATCH 05/15] fix: delete variable-reader.js --- packages/compile/src/model/model.js | 51 +----- packages/compile/src/model/variable-reader.js | 172 ------------------ 2 files changed, 9 insertions(+), 214 deletions(-) delete mode 100644 packages/compile/src/model/variable-reader.js diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index e44927d9..4c8ff800 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -17,12 +17,11 @@ import { import { readEquation } from './read-equations.js' import { readDimensionDefs } from './read-subscripts.js' -import { readVariables as readVariables2 } from './read-variables.js' +import { readVariables } from './read-variables.js' import { reduceVariables } from './reduce-variables.js' import SubscriptRangeReader from './subscript-range-reader.js' import toposort from './toposort.js' import Variable from './variable.js' -import VariableReader from './variable-reader.js' let variables = [] let inputVars = [] @@ -80,22 +79,16 @@ function read(parsedModel, spec, extData, directData, modelDirname, opts) { if (opts?.stopAfterResolveSubscripts) return // Read variables from the model parse tree. - if (parsedModel.kind === 'vensim-legacy') { - // TODO: directData is actually unused in VariableReader - readVariables(parsedModel.parseTree, specialSeparationDims, directData) - } else { - // Read the variables - const vars = readVariables2(parsedModel, specialSeparationDims) + const vars = readVariables(parsedModel, specialSeparationDims) - // Include a placeholder variable for the exogenous `Time` variable - const timeVar = new Variable(null) - timeVar.modelLHS = 'Time' - timeVar.varName = '_time' - vars.push(timeVar) + // Include a placeholder variable for the exogenous `Time` variable + const timeVar = new Variable(null) + timeVar.modelLHS = 'Time' + timeVar.varName = '_time' + vars.push(timeVar) - // Add the variables to the `Model` - vars.forEach(addVariable) - } + // Add the variables to the `Model` + vars.forEach(addVariable) if (opts?.stopAfterReadVariables) return if (spec) { @@ -286,32 +279,6 @@ function resolveDimensions(dimensionFamilies) { } } -/** - * Read equations from the given model and generate `Variable` instances for all variables that - * are encountered while parsing. - * - * Note that this function currently does not return anything and instead stores the parsed - * variable definitions in the `model` module. - * - * @param {import('../parse/parser.js').VensimModelParseTree} tree The Vensim parse tree. - * @param {Object.} specialSeparationDims The variable names that need to be - * separated because of circular references. A mapping from "C" variable name to "C" dimension - * name to separate on. - * @param {Map} directData The mapping of dataset name used in a `GET DIRECT DATA` - * call (e.g., `?data`) to the tabular data contained in the loaded data file. - */ -function readVariables(tree, specialSeparationDims, directData) { - // Read all variables in the model parse tree. - // This populates the variables table with basic information for each variable - // such as the var name and subscripts. - let variableReader = new VariableReader(specialSeparationDims, directData) - variableReader.visitModel(tree) - // Add a placeholder variable for the exogenous variable Time. - let v = new Variable(null) - v.modelLHS = 'Time' - v.varName = '_time' - addVariable(v) -} function analyze(parsedModelKind, inputVars, opts) { // Analyze the RHS of each equation in stages after all the variables are read. // Find non-apply-to-all vars that are defined with more than one equation. diff --git a/packages/compile/src/model/variable-reader.js b/packages/compile/src/model/variable-reader.js deleted file mode 100644 index 0bd9d885..00000000 --- a/packages/compile/src/model/variable-reader.js +++ /dev/null @@ -1,172 +0,0 @@ -import { ModelParser } from 'antlr4-vensim' -import * as R from 'ramda' - -import { canonicalName, vlog, strlist, cartesianProductOf } from '../_shared/helpers.js' -import { - sub, - isDimension, - isIndex, - normalizeSubscripts, - subscriptsMatch, - isSubdimension -} from '../_shared/subscript.js' -import ModelReader from '../parse/model-reader.js' - -import Model from './model.js' -import Variable from './variable.js' - -// Set true to print extra debugging information to stderr. -const DEBUG_LOG = false -let debugLog = (title, value) => !DEBUG_LOG || vlog(title, value) - -export default class VariableReader extends ModelReader { - constructor(specialSeparationDims, directData) { - super() - // specialSeparationDims are var names that need to be separated because of - // circular references, mapped to the dimension subscript to separate on. - // '{c-variable-name}': '{c-dimension-name}' - this.specialSeparationDims = specialSeparationDims || {} - this.directData = directData || {} - } - visitModel(ctx) { - let equations = ctx.equation() - if (equations) { - for (let equation of equations) { - equation.accept(this) - } - } - } - visitEquation(ctx) { - // Start a new variable defined by this equation. - this.var = new Variable(ctx) - // Allow for an alternate array of variables that are expanded over subdimensions. - this.expandedVars = [] - // Fill in the variable by visiting the equation parse context. - super.visitEquation(ctx) - if (R.isEmpty(this.expandedVars)) { - // Add a single variable defined by the equation. - Model.addVariable(this.var) - } else { - // Add variables expanded over indices to the model. - R.forEach(v => Model.addVariable(v), this.expandedVars) - } - } - visitLhs(ctx) { - this.var.varName = canonicalName(ctx.Id().getText()) - super.visitLhs(ctx) - // Possibly expand the var on subdimensions. - if (!R.isEmpty(this.var.subscripts)) { - // Expand on LHS subscripts alone. - let expanding = this.subscriptPosToExpand() - this.expandVars(expanding) - } - } - subscriptPosToExpand() { - // Decide whether we need to expand each subscript on the LHS. - // Construct an array of booleans in each subscript position. - let expanding = [] - for (let iLhsSub = 0; iLhsSub < this.var.subscripts.length; iLhsSub++) { - let subscript = this.var.subscripts[iLhsSub] - let expand = false - // Expand a subdimension and special separation dims in the LHS. - if (isDimension(subscript)) { - expand = isSubdimension(subscript) - if (!expand) { - let specialSeparationDims = this.specialSeparationDims[this.var.varName] || [] - expand = specialSeparationDims.includes(subscript) - } - } - if (!expand) { - // Direct data vars with subscripts are separated because we generate a lookup for each index. - if ( - isDimension(subscript) && - (this.var.modelFormula.includes('GET DIRECT DATA') || this.var.modelFormula.includes('GET DIRECT LOOKUPS')) - ) { - expand = true - } - } - // Also expand on exception subscripts that are indices or subdimensions. - if (!expand) { - for (let exceptSubs of this.var.exceptSubscripts) { - expand = isIndex(exceptSubs[iLhsSub]) || isSubdimension(exceptSubs[iLhsSub]) - if (expand) { - break - } - } - } - expanding.push(expand) - } - return expanding - } - expandVars(expanding) { - // Expand the indicated subscripts into variable objects in the expandedVars list. - debugLog(`expanding ${this.var.varName}[${strlist(this.var.subscripts)}] subscripts`, strlist(this.var.subscripts)) - let expansion = [] - let separationDims = [] - // Construct an array with an array at each subscript position. If the subscript is expanded at that position, - // it will become an array of indices. Otherwise, it remains an index or dimension as a single-valued array. - for (let i = 0; i < this.var.subscripts.length; i++) { - let subscript = this.var.subscripts[i] - let value - if (expanding[i]) { - separationDims.push(subscript) - if (isDimension(subscript)) { - value = sub(subscript).value - } - } - expansion.push(value || [subscript]) - } - // Generate an array of fully expanded subscripts, which may be indices or dimensions. - let expandedSubs = cartesianProductOf(expansion) - let skipExpansion = subs => { - // Check the subscripts against each set of except subscripts. Skip expansion if one of them matches. - let subsRange = R.range(0, subs.length) - for (let exceptSubscripts of this.var.exceptSubscripts) { - if (subs.length === exceptSubscripts.length) { - if (R.all(i => subscriptsMatch(subs[i], exceptSubscripts[i]), subsRange)) { - return true - } - } else { - console.error(`WARNING: expandedSubs length ≠ exceptSubscripts length in ${this.var.varName}`) - } - } - return false - } - for (let subs of expandedSubs) { - // Skip expansions that match exception subscripts. - if (!skipExpansion(subs)) { - // Add a new variable to the expanded vars. - let v = new Variable(this.var.eqnCtx) - v.varName = this.var.varName - v.subscripts = subs - v.separationDims = separationDims - debugLog(` ${strlist(v.subscripts)}`) - this.expandedVars.push(v) - } - } - } - visitSubscriptList(ctx) { - if (ctx.parentCtx.ruleIndex === ModelParser.RULE_lhs) { - let subscripts = normalizeSubscripts(R.map(id => canonicalName(id.getText()), ctx.Id())) - // Save subscripts in the Variable instance. Subscripts after the first one are exception subscripts. - if (R.isEmpty(this.var.subscripts)) { - this.var.subscripts = subscripts - } else { - this.var.exceptSubscripts.push(subscripts) - } - } - super.visitSubscriptList(ctx) - } - visitConstList(ctx) { - // Expand a subscripted equation with a constant list. - let exprs = ctx.expr() - if (exprs.length > 1) { - let expanding = R.map(subscript => isDimension(subscript), this.var.subscripts) - // If the var was already expanded, do it over to make sure we expand on all subscripts. - if (!R.isEmpty(this.expandedVars)) { - this.expandedVars = [] - } - this.expandVars(expanding) - } - } -} From 5072ff7da56d033d4080cedd9e932cbe21c73534 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 11:51:45 -0700 Subject: [PATCH 06/15] fix: delete subscript-range-reader.js --- packages/compile/src/model/model.js | 23 +-- .../src/model/subscript-range-reader.js | 143 ------------------ 2 files changed, 1 insertion(+), 165 deletions(-) delete mode 100644 packages/compile/src/model/subscript-range-reader.js diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index 4c8ff800..ca41b17e 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -19,7 +19,6 @@ import { readEquation } from './read-equations.js' import { readDimensionDefs } from './read-subscripts.js' import { readVariables } from './read-variables.js' import { reduceVariables } from './reduce-variables.js' -import SubscriptRangeReader from './subscript-range-reader.js' import toposort from './toposort.js' import Variable from './variable.js' @@ -69,11 +68,7 @@ function read(parsedModel, spec, extData, directData, modelDirname, opts) { let specialSeparationDims = spec.specialSeparationDims // Dimensions must be defined before reading variables that use them. - if (parsedModel.kind === 'vensim-legacy') { - readSubscriptRanges(parsedModel.parseTree, modelDirname) - } else { - readDimensionDefs(parsedModel, modelDirname) - } + readDimensionDefs(parsedModel, modelDirname) if (opts?.stopAfterReadSubscripts) return resolveDimensions(spec.dimensionFamilies) if (opts?.stopAfterResolveSubscripts) return @@ -120,22 +115,6 @@ function read(parsedModel, spec, extData, directData, modelDirname, opts) { resolveDuplicateDeclarations() } -/** - * Read subscript ranges from the given model. - * - * Note that this function currently does not return anything and instead stores the parsed subscript - * range definitions in the `subscript` module. - * - * @param {import('../parse/parser.js').VensimModelParseTree} parseTree The Vensim parse tree. - * @param {string} modelDirname The path to the directory containing the model (used for resolving data - * files for `GET DIRECT SUBSCRIPT`). - */ -function readSubscriptRanges(parseTree, modelDirname) { - // Read subscript ranges from the model. - let subscriptRangeReader = new SubscriptRangeReader(modelDirname) - subscriptRangeReader.visitModel(parseTree) -} - /** * Process the previously read subscript/dimension definitions (stored in the `subscript` module) to * resolve aliases, families, and indices. diff --git a/packages/compile/src/model/subscript-range-reader.js b/packages/compile/src/model/subscript-range-reader.js deleted file mode 100644 index 3cd23ff0..00000000 --- a/packages/compile/src/model/subscript-range-reader.js +++ /dev/null @@ -1,143 +0,0 @@ -import path from 'path' -import { ModelParser } from 'antlr4-vensim' -import * as R from 'ramda' -import XLSX from 'xlsx' - -import { cFunctionName, matchRegex, readCsv } from '../_shared/helpers.js' -import { Subscript } from '../_shared/subscript.js' -import ModelReader from '../parse/model-reader.js' - -export default class SubscriptRangeReader extends ModelReader { - constructor(modelDirname) { - super() - // The model directory is required when reading data files for GET DIRECT SUBSCRIPT. - this.modelDirname = modelDirname - // Index names from a subscript list or GET DIRECT SUBSCRIPT - this.indNames = [] - // Dimension mappings with model names - this.modelMappings = [] - } - visitModel(ctx) { - let subscriptRanges = ctx.subscriptRange() - if (subscriptRanges) { - for (let subscriptRange of subscriptRanges) { - subscriptRange.accept(this) - } - } - } - visitSubscriptRange(ctx) { - // When entering a new subscript range definition, reset the properties that will be filled in. - this.indNames = [] - this.modelMappings = [] - // A subscript alias has two Ids, while a regular subscript range definition has just one. - if (ctx.Id().length === 1) { - // Subscript range definitions have a dimension name. - let modelName = ctx.Id()[0].getText() - // Visit children to fill in the subscript range definition. - super.visitSubscriptRange(ctx) - // Create a new subscript range definition from Vensim-format names. - // The family is provisionally set to the dimension name. - // It will be updated to the maximal dimension if this is a subdimension. - // The mapping value contains dimensions and indices in the toDim. - // It will be expanded and inverted to fromDim indices later. - Subscript(modelName, this.indNames, modelName, this.modelMappings) - } else { - let modelName = ctx.Id()[0].getText() - let modelFamily = ctx.Id()[1].getText() - Subscript(modelName, '', modelFamily, []) - } - } - visitSubscriptList(ctx) { - // Get the subscripts from each subscript index in the list. - for (let child of ctx.children) { - if (child.symbol?.type === ModelParser.Id) { - this.addSubscriptIndex(ctx, child) - } - } - } - visitSubscriptDefList(ctx) { - // Subscript range definitions can have indices and numeric subscript sequences. - for (let child of ctx.children) { - if (child.symbol?.type === ModelParser.Id) { - this.addSubscriptIndex(ctx, child) - } else if (child.ruleIndex === ModelParser.RULE_subscriptSequence) { - this.visitSubscriptSequence(child) - } - } - } - addSubscriptIndex(ctx, child) { - let subscript = child.getText() - if (ctx.parentCtx.ruleIndex === ModelParser.RULE_subscriptRange) { - this.indNames.push(subscript) - } else if (ctx.parentCtx.ruleIndex === ModelParser.RULE_subscriptMapping) { - this.mappingValue.push(subscript) - } - } - visitSubscriptMapping(ctx) { - let toDim = ctx.Id().getText() - // If a subscript list is part of the mapping, mappingValue will be set by visitSubscriptList. - this.mappingValue = [] - super.visitSubscriptMapping(ctx) - this.modelMappings.push({ toDim, value: this.mappingValue }) - } - visitSubscriptSequence(ctx) { - // Construct index names from the sequence start and end indices. - // This assumes the indices begin with the same string and end with numbers. - let r = /^(.*?)(\d+)$/ - let ids = R.map(id => id.getText(), ctx.Id()) - let matches = R.map(id => r.exec(id), ids) - if (matches[0][1] === matches[1][1]) { - let prefix = matches[0][1] - let start = parseInt(matches[0][2]) - let end = parseInt(matches[1][2]) - // TODO get this to work with subscript mappings too - for (let i = start; i <= end; i++) { - this.indNames.push(prefix + i) - } - } - } - visitCall(ctx) { - // A subscript range can have a GET DIRECT SUBSCRIPT call on the RHS. - let fn = cFunctionName(ctx.Id().getText()) - if (fn === '_GET_DIRECT_SUBSCRIPT') { - super.visitCall(ctx) - } - } - visitExprList(ctx) { - // We assume the only call that ends up here is GET DIRECT SUBSCRIPT. - let args = R.map( - arg => matchRegex(arg, /'(.*)'/), - R.map(expr => expr.getText(), ctx.expr()) - ) - let pathname = args[0] - let delimiter = args[1] - let firstCell = args[2] - let lastCell = args[3] - // let prefix = args[4] - // If lastCell is a column letter, scan the column, else scan the row. - let dataAddress = XLSX.utils.decode_cell(firstCell.toUpperCase()) - let col = dataAddress.c - let row = dataAddress.r - if (col < 0 || row < 0) { - throw new Error(`Failed to parse 'firstcell' argument for GET DIRECT SUBSCRIPT call: ${firstCell}`) - } - let nextCell - if (isNaN(parseInt(lastCell))) { - nextCell = () => row++ - } else { - nextCell = () => col++ - } - // Read subscript names from the CSV file at the given position. - let csvPathname = path.resolve(this.modelDirname, pathname) - let data = readCsv(csvPathname, delimiter) - if (data) { - let indexName = data[row][col] - while (indexName != null) { - this.indNames.push(indexName) - nextCell() - indexName = data[row] != null ? data[row][col] : null - } - } - super.visitExprList(ctx) - } -} From 038249a4fb24e5352d64675c750bd4c732827825 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 12:16:45 -0700 Subject: [PATCH 07/15] fix: delete model-reader.js and parser.js --- packages/compile/src/_tests/test-support.ts | 8 +- packages/compile/src/index.js | 3 +- .../compile/src/model/read-equations.spec.ts | 6 +- .../compile/src/model/read-subscripts.spec.ts | 4 +- .../src/model/reduce-variables.spec.ts | 6 +- packages/compile/src/parse-and-generate.js | 9 -- packages/compile/src/parse/model-reader.js | 141 ------------------ packages/compile/src/parse/parser.js | 36 ----- 8 files changed, 11 insertions(+), 202 deletions(-) delete mode 100644 packages/compile/src/parse/model-reader.js delete mode 100644 packages/compile/src/parse/parser.js diff --git a/packages/compile/src/_tests/test-support.ts b/packages/compile/src/_tests/test-support.ts index 03fc8e82..4a47f998 100644 --- a/packages/compile/src/_tests/test-support.ts +++ b/packages/compile/src/_tests/test-support.ts @@ -4,7 +4,6 @@ import { fileURLToPath } from 'url' import type { Model } from '@sdeverywhere/parse' -import type { VensimModelParseTree } from '../parse/parser' import { preprocessModel } from '../preprocess/preprocessor' import { canonicalName } from '../_shared/helpers' import { parseModel } from '../parse-and-generate' @@ -14,12 +13,7 @@ export interface ParsedVensimModel { root: Model } -export interface LegacyParsedVensimModel { - kind: 'vensim-legacy' - parseTree: VensimModelParseTree -} - -export type ParsedModel = ParsedVensimModel | LegacyParsedVensimModel +export type ParsedModel = ParsedVensimModel export type DimModelName = string export type DimCName = string diff --git a/packages/compile/src/index.js b/packages/compile/src/index.js index 14afd496..5ea7bdf4 100644 --- a/packages/compile/src/index.js +++ b/packages/compile/src/index.js @@ -3,6 +3,7 @@ export { canonicalName } from './_shared/helpers.js' export { readDat } from './_shared/read-dat.js' export { preprocessModel } from './preprocess/preprocessor.js' -export { parseModel } from './parse/parser.js' +// TODO +// export { parseModel } from './parse/parser.js' export { generateCode } from './generate/code-gen.js' export { parseAndGenerate, printNames } from './parse-and-generate.js' diff --git a/packages/compile/src/model/read-equations.spec.ts b/packages/compile/src/model/read-equations.spec.ts index 6052a110..615b48a0 100644 --- a/packages/compile/src/model/read-equations.spec.ts +++ b/packages/compile/src/model/read-equations.spec.ts @@ -2,12 +2,12 @@ import { describe, expect, it } from 'vitest' import { canonicalName, resetHelperState } from '../_shared/helpers' import { resetSubscriptsAndDimensions } from '../_shared/subscript' -import type { VensimModelParseTree } from '../parse/parser' import Model from './model' import { default as VariableImpl } from './variable' -import { parseInlineVensimModel, parseVensimModel, sampleModelDir, type Variable } from '../_tests/test-support' +import type { ParsedModel, Variable } from '../_tests/test-support' +import { parseInlineVensimModel, parseVensimModel, sampleModelDir } from '../_tests/test-support' /** * This is a shorthand for the following steps to read equations: @@ -32,7 +32,7 @@ function readSubscriptsAndEquationsFromSource( resetSubscriptsAndDimensions() Model.resetModelState() - let parsedModel: VensimModelParseTree + let parsedModel: ParsedModel if (source.modelText) { parsedModel = parseInlineVensimModel(source.modelText) } else { diff --git a/packages/compile/src/model/read-subscripts.spec.ts b/packages/compile/src/model/read-subscripts.spec.ts index 7a0c4a89..85e02623 100644 --- a/packages/compile/src/model/read-subscripts.spec.ts +++ b/packages/compile/src/model/read-subscripts.spec.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from 'vitest' import { allSubscripts, resetSubscriptsAndDimensions } from '../_shared/subscript' -import type { VensimModelParseTree } from '../parse/parser' import Model from './model' +import type { ParsedModel } from '../_tests/test-support' import { dim, dimMapping, parseInlineVensimModel, parseVensimModel, sampleModelDir, sub } from '../_tests/test-support' /** @@ -27,7 +27,7 @@ function readSubscriptsFromSource( // XXX: This is needed due to subs/dims being in module-level storage resetSubscriptsAndDimensions() - let parsedModel: VensimModelParseTree + let parsedModel: ParsedModel if (source.modelText) { parsedModel = parseInlineVensimModel(source.modelText) } else { diff --git a/packages/compile/src/model/reduce-variables.spec.ts b/packages/compile/src/model/reduce-variables.spec.ts index 3affe81d..12341714 100644 --- a/packages/compile/src/model/reduce-variables.spec.ts +++ b/packages/compile/src/model/reduce-variables.spec.ts @@ -2,12 +2,12 @@ import { describe, expect, it } from 'vitest' import { canonicalName, resetHelperState } from '../_shared/helpers' import { resetSubscriptsAndDimensions } from '../_shared/subscript' -import type { VensimModelParseTree } from '../parse/parser' import Model from './model' import { default as VariableImpl } from './variable' -import { parseInlineVensimModel, parseVensimModel, sampleModelDir, type Variable } from '../_tests/test-support' +import type { ParsedModel, Variable } from '../_tests/test-support' +import { parseInlineVensimModel, parseVensimModel, sampleModelDir } from '../_tests/test-support' /** * This is a shorthand for the following steps to read equations: @@ -31,7 +31,7 @@ function readSubscriptsAndEquationsFromSource( resetSubscriptsAndDimensions() Model.resetModelState() - let parsedModel: VensimModelParseTree + let parsedModel: ParsedModel if (source.modelText) { parsedModel = parseInlineVensimModel(source.modelText) } else { diff --git a/packages/compile/src/parse-and-generate.js b/packages/compile/src/parse-and-generate.js index 677ea746..355fed3e 100644 --- a/packages/compile/src/parse-and-generate.js +++ b/packages/compile/src/parse-and-generate.js @@ -8,7 +8,6 @@ import { parseVensimModel } from '@sdeverywhere/parse' import { readXlsx } from './_shared/helpers.js' import { readDat } from './_shared/read-dat.js' import { printSubscripts, yamlSubsList } from './_shared/subscript.js' -import { parseModel as legacyParseVensimModel } from './parse/parser.js' import Model from './model/model.js' import { getDirectSubscripts } from './model/read-subscripts.js' import { generateCode } from './generate/code-gen.js' @@ -131,14 +130,6 @@ export function printNames(namesPathname, operation) { * @return {*} A parsed tree representation of the model. */ export function parseModel(input, modelDir, sort = false) { - if (process.env.SDE_NONPUBLIC_USE_NEW_PARSE === '0') { - // Use the legacy parser - return { - kind: 'vensim-legacy', - parseTree: legacyParseVensimModel(input) - } - } - // Prepare the parse context that provides access to external data files let parseContext /*: VensimParseContext*/ if (modelDir) { diff --git a/packages/compile/src/parse/model-reader.js b/packages/compile/src/parse/model-reader.js deleted file mode 100644 index 7b8970e6..00000000 --- a/packages/compile/src/parse/model-reader.js +++ /dev/null @@ -1,141 +0,0 @@ -import { ModelVisitor } from 'antlr4-vensim' - -export default class ModelReader extends ModelVisitor { - constructor() { - super() - // stack of function names and argument indices encountered on the RHS - this.callStack = [] - } - currentFunctionName() { - // Return the name of the current function on top of the call stack. - let n = this.callStack.length - return n > 0 ? this.callStack[n - 1].fn : '' - } - setArgIndex(argIndex) { - // Set the argument index in the current function call on top of the call stack. - // This may be set in the exprList visitor and picked up in the var visitor to facilitate special argument handling. - let n = this.callStack.length - if (n > 0) { - this.callStack[n - 1].argIndex = argIndex - } - } - argIndexForFunctionName(name) { - // Search the call stack for the function name. Return the current argument index or undefined if not found. - let argIndex - for (let i = this.callStack.length - 1; i >= 0; i--) { - if (this.callStack[i].fn === name) { - argIndex = this.callStack[i].argIndex - break - } - } - return argIndex - } - visitEquation(ctx) { - ctx.lhs().accept(this) - if (ctx.expr()) { - ctx.expr().accept(this) - } else if (ctx.constList()) { - ctx.constList().accept(this) - } else if (ctx.lookup()) { - ctx.lookup().accept(this) - } else { - this.var.varType = 'data' - } - } - visitLhs(ctx) { - // An LHS may have a subscript list after the var name. - // If it has an EXCEPT clause, it will have one or more other subscript lists there too. - let subscriptLists = ctx.subscriptList() - if (subscriptLists.length > 0) { - for (let i = 0; i < subscriptLists.length; i++) { - subscriptLists[i].accept(this) - } - } - } - - // Function calls and variables - - visitCall(ctx) { - if (ctx.exprList()) { - ctx.exprList().accept(this) - } - } - visitExprList(ctx) { - let exprs = ctx.expr() - // Set the argument index in an instance property so derived classes can determine argument position. - for (let i = 0; i < exprs.length; i++) { - this.setArgIndex(i) - exprs[i].accept(this) - } - } - visitVar(ctx) { - if (ctx.subscriptList()) { - ctx.subscriptList().accept(this) - } - } - - // Lookups - - visitLookup(ctx) { - if (ctx.lookupRange()) { - ctx.lookupRange().accept(this) - } - if (ctx.lookupPointList()) { - ctx.lookupPointList().accept(this) - } - } - visitLookupCall(ctx) { - if (ctx.subscriptList()) { - ctx.subscriptList().accept(this) - } - } - - // Unary operators - - visitNegative(ctx) { - ctx.expr().accept(this) - } - visitPositive(ctx) { - ctx.expr().accept(this) - } - visitNot(ctx) { - ctx.expr().accept(this) - } - - // Binary operators - - visitPower(ctx) { - ctx.expr(0).accept(this) - ctx.expr(1).accept(this) - } - visitMulDiv(ctx) { - ctx.expr(0).accept(this) - ctx.expr(1).accept(this) - } - visitAddSub(ctx) { - ctx.expr(0).accept(this) - ctx.expr(1).accept(this) - } - visitRelational(ctx) { - ctx.expr(0).accept(this) - ctx.expr(1).accept(this) - } - visitEquality(ctx) { - ctx.expr(0).accept(this) - ctx.expr(1).accept(this) - } - visitAnd(ctx) { - ctx.expr(0).accept(this) - ctx.expr(1).accept(this) - } - visitOr(ctx) { - ctx.expr(0).accept(this) - ctx.expr(1).accept(this) - } - - // Tokens - - visitParens(ctx) { - ctx.expr().accept(this) - } -} diff --git a/packages/compile/src/parse/parser.js b/packages/compile/src/parse/parser.js deleted file mode 100644 index d2823971..00000000 --- a/packages/compile/src/parse/parser.js +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2022 Climate Interactive / New Venture Fund - -import antlr4 from 'antlr4' -import { ModelLexer, ModelParser } from 'antlr4-vensim' - -/** - * @typedef {object} VensimModelParseTree - */ - -/** - * Create a `ModelParser` for the given model text, which can be the - * contents of an entire `mdl` file, or a portion of one (e.g., an - * expression or definition). - * - * @param {string} input The string containing the model text. - * @return {ModelParser} A `ModelParser` from which a parse tree can be obtained. - */ -export function createParser(input) { - let chars = new antlr4.InputStream(input) - let lexer = new ModelLexer(chars) - let tokens = new antlr4.CommonTokenStream(lexer) - let parser = new ModelParser(tokens) - parser.buildParseTrees = true - return parser -} - -/** - * Read the given model text and return a parse tree. - * - * @param {string} input The string containing the model text. - * @return {VensimModelParseTree} A parse tree representation of the model. - */ -export function parseModel(input) { - let parser = createParser(input) - return parser.model() -} From d9c807e09487365ddeb5a83f26abe0ca8f7dcde4 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 12:25:50 -0700 Subject: [PATCH 08/15] fix: remove Variable.eqnCtx --- .../src/generate/expand-var-names.spec.ts | 18 ++++---------- packages/compile/src/model/model.js | 8 +++---- .../compile/src/model/read-equations.spec.ts | 6 +---- packages/compile/src/model/read-variables.js | 4 ++-- .../compile/src/model/read-variables.spec.ts | 6 +---- .../src/model/reduce-variables.spec.ts | 6 +---- packages/compile/src/model/variable.js | 24 +++---------------- 7 files changed, 16 insertions(+), 56 deletions(-) diff --git a/packages/compile/src/generate/expand-var-names.spec.ts b/packages/compile/src/generate/expand-var-names.spec.ts index 84e7c8e0..6c73620f 100644 --- a/packages/compile/src/generate/expand-var-names.spec.ts +++ b/packages/compile/src/generate/expand-var-names.spec.ts @@ -41,23 +41,15 @@ describe('expandVarNames', () => { expect(expandVarNames(true)).toEqual(['_time', '_x[0]', '_x[1]', '_x[2]']) }) - // TODO: Note that the EXCEPT clause refers to a subscript that is not in the subdimension. - // Due to the way that the legacy ModelLHSReader works (it looks at all subscripts, even those - // in the EXCEPT clause), it will pick up the `A1` and include it in the set of expanded names - // even though this doesn't seem appropriate. We will treat this as a quirk of the legacy - // reader and therefore skip this test when the legacy code is in use. - it.skipIf(process.env.SDE_NONPUBLIC_USE_NEW_PARSE === '0')( - 'should return names for a subscripted 1D variable that uses a disjoint EXCEPT clause', - () => { - readInlineModel(` + it('should return names for a subscripted 1D variable that uses a disjoint EXCEPT clause', () => { + readInlineModel(` DimA: A1, A2, A3 ~~| SubA: A2, A3 ~~| X[SubA] :EXCEPT: [A1] = 1 ~~| `) - expect(expandVarNames(false)).toEqual(['Time', 'X[A2]', 'X[A3]']) - expect(expandVarNames(true)).toEqual(['_time', '_x[1]', '_x[2]']) - } - ) + expect(expandVarNames(false)).toEqual(['Time', 'X[A2]', 'X[A3]']) + expect(expandVarNames(true)).toEqual(['_time', '_x[1]', '_x[2]']) + }) it('should return names for a subscripted 1D variable that refers to subdimensions', () => { readInlineModel(` diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index ca41b17e..c405b9f8 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -267,11 +267,9 @@ function analyze(parsedModelKind, inputVars, opts) { setRefIds() // If enabled, reduce expressions used in variable definitions. - if (parsedModelKind !== 'vensim-legacy') { - if (opts?.reduceVariables !== false && process.env.SDE_NONPUBLIC_REDUCE_VARIABLES !== '0') { - let reduceMode = opts?.reduceVariables || process.env.SDE_NONPUBLIC_REDUCE_VARIABLES || 'default' - reduceVariables(variables, inputVars || [], reduceMode) - } + if (opts?.reduceVariables !== false && process.env.SDE_NONPUBLIC_REDUCE_VARIABLES !== '0') { + let reduceMode = opts?.reduceVariables || process.env.SDE_NONPUBLIC_REDUCE_VARIABLES || 'default' + reduceVariables(variables, inputVars || [], reduceMode) } if (opts?.stopAfterReduceVariables === true) return diff --git a/packages/compile/src/model/read-equations.spec.ts b/packages/compile/src/model/read-equations.spec.ts index 615b48a0..b1991c4f 100644 --- a/packages/compile/src/model/read-equations.spec.ts +++ b/packages/compile/src/model/read-equations.spec.ts @@ -56,8 +56,6 @@ function readSubscriptsAndEquationsFromSource( }) return Model.variables.map(v => { - // XXX: Strip out the legacy ANTLR eqnCtx to avoid vitest hang when comparing - delete v.eqnCtx // XXX: Strip out the new parsedEqn field, since we don't need it for comparing delete v.parsedEqn return v @@ -82,9 +80,7 @@ function readSubscriptsAndEquations(modelName: string): Variable[] { } function v(lhs: string, formula: string, overrides?: Partial): Variable { - const variable = new VariableImpl(undefined) - // XXX: Strip out the ANTLR eqnCtx to avoid vitest hang when comparing - delete variable.eqnCtx + const variable = new VariableImpl() variable.modelLHS = lhs variable.modelFormula = formula variable.varName = canonicalName(lhs.split('[')[0]) diff --git a/packages/compile/src/model/read-variables.js b/packages/compile/src/model/read-variables.js index 4de975f7..512b9ee2 100644 --- a/packages/compile/src/model/read-variables.js +++ b/packages/compile/src/model/read-variables.js @@ -49,7 +49,7 @@ export function readVariables(parsedModel, specialSeparationDims) { */ function variablesForEquation(eqn, specialSeparationDims) { // Start a new variable defined by this equation - const variable = new Variable(null) + const variable = new Variable() // Fill in the LHS details const lhs = eqn.lhs.varDef @@ -137,7 +137,7 @@ function variablesForEquation(eqn, specialSeparationDims) { // Generate variables expanded over subscripts to the model const variables = [] for (const expansion of expansions) { - const v = new Variable(null) + const v = new Variable() v.varName = baseVarId v.modelLHS = lhsText v.modelFormula = rhsText diff --git a/packages/compile/src/model/read-variables.spec.ts b/packages/compile/src/model/read-variables.spec.ts index 21483e6c..bab406b1 100644 --- a/packages/compile/src/model/read-variables.spec.ts +++ b/packages/compile/src/model/read-variables.spec.ts @@ -28,8 +28,6 @@ function readSubscriptsAndVariables(modelName: string): Variable[] { }) return Model.variables.map(v => { - // XXX: Strip out the ANTLR eqnCtx to avoid vitest hang when comparing - delete v.eqnCtx // XXX: Strip out the new parsedEqn field, since we don't need it for comparing delete v.parsedEqn return v @@ -37,9 +35,7 @@ function readSubscriptsAndVariables(modelName: string): Variable[] { } function v(lhs: string, formula: string, overrides?: Partial): Variable { - const variable = new VariableImpl(undefined) - // XXX: Strip out the ANTLR eqnCtx to avoid vitest hang when comparing - delete variable.eqnCtx + const variable = new VariableImpl() variable.modelLHS = lhs variable.modelFormula = formula variable.varName = canonicalName(lhs.split('[')[0]) diff --git a/packages/compile/src/model/reduce-variables.spec.ts b/packages/compile/src/model/reduce-variables.spec.ts index 12341714..d5a570c4 100644 --- a/packages/compile/src/model/reduce-variables.spec.ts +++ b/packages/compile/src/model/reduce-variables.spec.ts @@ -52,8 +52,6 @@ function readSubscriptsAndEquationsFromSource( }) return Model.variables.map(v => { - // XXX: Strip out the legacy ANTLR eqnCtx to avoid vitest hang when comparing - delete v.eqnCtx // XXX: Strip out the new `parsedEqn` field, since we don't need it for comparing delete v.parsedEqn // XXX: Strip out the `origModelFormula` field, since we don't need it for comparing @@ -74,9 +72,7 @@ function readInlineModel(reduceVariables: 'default' | 'aggressive', modelText: s // } function v(lhs: string, formula: string, overrides?: Partial): Variable { - const variable = new VariableImpl(undefined) - // XXX: Strip out the ANTLR eqnCtx to avoid vitest hang when comparing - delete variable.eqnCtx + const variable = new VariableImpl() variable.modelLHS = lhs variable.modelFormula = formula variable.varName = canonicalName(lhs.split('[')[0]) diff --git a/packages/compile/src/model/variable.js b/packages/compile/src/model/variable.js index 0fb10aab..11085e0a 100644 --- a/packages/compile/src/model/variable.js +++ b/packages/compile/src/model/variable.js @@ -1,15 +1,8 @@ export default class Variable { - constructor(eqnCtx) { - // The equation rule context allows us to generate code by visiting the parse tree. - this.eqnCtx = eqnCtx + constructor() { // Save both sides of the equation text in the model for documentation purposes. - if (eqnCtx) { - this.modelLHS = eqnCtx.lhs().getText() - this.modelFormula = this.formula(eqnCtx) - } else { - this.modelLHS = '' - this.modelFormula = '' - } + this.modelLHS = '' + this.modelFormula = '' // An equation defines a variable with a var name, saved in canonical form here. this.varName = '' // Subscripts are canonical dimension or index names on the LHS in normal order. @@ -57,7 +50,6 @@ export default class Variable { } copy() { let c = new Variable() - c.eqnCtx = this.eqnCtx c.modelLHS = this.modelLHS c.modelFormula = this.modelFormula c.varName = this.varName @@ -78,16 +70,6 @@ export default class Variable { c.includeInOutput = this.includeInOutput return c } - formula(eqnCtx) { - if (eqnCtx) { - if (eqnCtx.expr()) { - return eqnCtx.expr().getText() - } else if (eqnCtx.constList()) { - return eqnCtx.constList().getText() - } - } - return '' - } hasSubscripts() { return this.subscripts.length > 0 } From 193790023313a13390e74e9b1434d390de425cb7 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 12:33:39 -0700 Subject: [PATCH 09/15] test: update tests to remove logic related to old code path --- models/prune/prune_check.sh | 36 ++-- packages/compile/src/_tests/test-support.ts | 18 +- .../compile/src/model/read-equations.spec.ts | 8 +- .../src/model/reduce-variables.spec.ts | 174 +++++++++--------- tests/modeltests | 8 +- 5 files changed, 112 insertions(+), 132 deletions(-) diff --git a/models/prune/prune_check.sh b/models/prune/prune_check.sh index d61af51f..cc6359ae 100755 --- a/models/prune/prune_check.sh +++ b/models/prune/prune_check.sh @@ -44,31 +44,17 @@ expect_present "_constant_partial_2" expect_present "_initial_partial" expect_present "_partial" expect_present "_test_1_result = _IF_THEN_ELSE(_input_1 == 10.0, _test_1_t, _test_1_f);" -if [[ $SDE_NONPUBLIC_USE_NEW_PARSE == "0" ]]; then - expect_present "_test_2_result = (_test_2_f);" - expect_present "_test_3_result = (_test_3_t);" - expect_present "_test_4_result = (_test_4_f);" - expect_present "_test_5_result = (_test_5_t);" - expect_present "_test_6_result = (_test_6_f);" - expect_present "_test_7_result = (_test_7_t);" - expect_present "_test_8_result = (_test_8_f);" - expect_present "_test_9_result = (_test_9_t);" - expect_present "_test_10_result = _IF_THEN_ELSE(_ABS(_test_10_cond), _test_10_t, _test_10_f);" - expect_present "_test_11_result = (_test_11_f);" - expect_present "_test_12_result = (_test_12_t);" -else - expect_present "_test_2_result = _test_2_f;" - expect_present "_test_3_result = _test_3_t;" - expect_present "_test_4_result = _test_4_f;" - expect_present "_test_5_result = _test_5_t;" - expect_present "_test_6_result = _test_6_f;" - expect_present "_test_7_result = _test_7_t;" - expect_present "_test_8_result = _test_8_f;" - expect_present "_test_9_result = _test_9_t;" - expect_present "_test_10_result = _test_10_t;" - expect_present "_test_11_result = _test_11_f;" - expect_present "_test_12_result = _test_12_t;" -fi +expect_present "_test_2_result = _test_2_f;" +expect_present "_test_3_result = _test_3_t;" +expect_present "_test_4_result = _test_4_f;" +expect_present "_test_5_result = _test_5_t;" +expect_present "_test_6_result = _test_6_f;" +expect_present "_test_7_result = _test_7_t;" +expect_present "_test_8_result = _test_8_f;" +expect_present "_test_9_result = _test_9_t;" +expect_present "_test_10_result = _test_10_t;" +expect_present "_test_11_result = _test_11_f;" +expect_present "_test_12_result = _test_12_t;" expect_present "_test_13_result = (_test_13_t1 + _test_13_t2) \* 10.0;" # Verify that unreferenced variables do not appear in the generated C file diff --git a/packages/compile/src/_tests/test-support.ts b/packages/compile/src/_tests/test-support.ts index 4a47f998..580cbeec 100644 --- a/packages/compile/src/_tests/test-support.ts +++ b/packages/compile/src/_tests/test-support.ts @@ -4,7 +4,6 @@ import { fileURLToPath } from 'url' import type { Model } from '@sdeverywhere/parse' -import { preprocessModel } from '../preprocess/preprocessor' import { canonicalName } from '../_shared/helpers' import { parseModel } from '../parse-and-generate' @@ -153,16 +152,13 @@ export function sampleModelDir(modelName: string): string { export function parseVensimModel(modelName: string): ParsedModel { const modelDir = sampleModelDir(modelName) const modelFile = resolve(modelDir, `${modelName}.mdl`) - let mdlContent: string - if (process.env.SDE_NONPUBLIC_USE_NEW_PARSE !== '0') { - // Note that the new parser implicitly runs the preprocessor on the input model text, - // so we don't need to do that here. (We should make it configurable so that we can - // skip the preprocess step in `parse-and-generate.js` when the input model text has - // already been run through a preprocessor.) - mdlContent = readFileSync(modelFile, 'utf8') - } else { - mdlContent = preprocessModel(modelFile, undefined, 'genc', false) - } + + // Note that the new parser implicitly runs the preprocessor on the input model text, + // so we don't need to do that here. (We should make it configurable so that we can + // skip the preprocess step in `parse-and-generate.js` when the input model text has + // already been run through a preprocessor.) + const mdlContent = readFileSync(modelFile, 'utf8') + // We currently sort the preprocessed definitions alphabetically for // compatibility with the legacy preprocessor. Once we drop the legacy code // we could remove this step and update the tests to use the original order. diff --git a/packages/compile/src/model/read-equations.spec.ts b/packages/compile/src/model/read-equations.spec.ts index b1991c4f..b6c97450 100644 --- a/packages/compile/src/model/read-equations.spec.ts +++ b/packages/compile/src/model/read-equations.spec.ts @@ -3148,7 +3148,7 @@ describe('readEquations', () => { // where the new reader differs from the old (in `IF THEN ELSE(integer supply, ...)` // where the condition resolves to a constant). We should add an option to disable // the pruning code so that we can test this more deterministically. - it.skipIf(process.env.SDE_NONPUBLIC_USE_NEW_PARSE !== '0')('should work for Vensim "allocate" model', () => { + it.skip('should work for Vensim "allocate" model', () => { const vars = readSubscriptsAndEquations('allocate') expect(vars).toEqual([ v('demand[region]', '3,2,4', { @@ -6527,8 +6527,8 @@ describe('readEquations', () => { // }) // TODO: This test depends on the dependency trimming code that isn't yet implemented - // in the new reader, so skip it when `NEW_PARSE` flag is enabled - it.skipIf(process.env.SDE_NONPUBLIC_USE_NEW_PARSE !== '0')('should work for Vensim "prune" model', () => { + // in the new reader, so skip it for now + it.skip('should work for Vensim "prune" model', () => { const vars = readSubscriptsAndEquations('prune') expect(vars).toEqual([ v('A Totals', 'SUM(A Values[DimA!])', { @@ -7100,7 +7100,7 @@ describe('readEquations', () => { // where the new reader differs from the old (in `IF THEN ELSE(switch=1,1,0)` // where the condition resolves to a constant). We should add an option to disable // the pruning code so that we can test this more deterministically. - it.skipIf(process.env.SDE_NONPUBLIC_USE_NEW_PARSE !== '0')('should work for Vensim "sample" model', () => { + it.skip('should work for Vensim "sample" model', () => { const vars = readSubscriptsAndEquations('sample') expect(vars).toEqual([ v('a', 'SAMPLE IF TRUE(MODULO(Time,5)=0,Time,0)', { diff --git a/packages/compile/src/model/reduce-variables.spec.ts b/packages/compile/src/model/reduce-variables.spec.ts index d5a570c4..0a603e24 100644 --- a/packages/compile/src/model/reduce-variables.spec.ts +++ b/packages/compile/src/model/reduce-variables.spec.ts @@ -89,110 +89,104 @@ function v(lhs: string, formula: string, overrides?: Partial): Variabl return variable as Variable } -describe.skipIf(process.env.SDE_NONPUBLIC_USE_NEW_PARSE === '0')( - 'reduceVariables (default mode: reduce conditionals only)', - () => { - it('should reduce a simple equation when the condition resolves to a constant', () => { - const vars = readInlineModel( - 'default', - ` +describe('reduceVariables (default mode: reduce conditionals only)', () => { + it('should reduce a simple equation when the condition resolves to a constant', () => { + const vars = readInlineModel( + 'default', + ` x = 1 ~~| y = IF THEN ELSE(x, (x + 2) * 3, 5) ~~| ` - ) - expect(vars).toEqual([ - v('x', '1', { - refId: '_x' - }), - v('y', '((x+2)*3)', { - refId: '_y' - }) - ]) - }) - - it('should not reduce an equation that does not involve a conditional', () => { - const vars = readInlineModel( - 'default', - ` + ) + expect(vars).toEqual([ + v('x', '1', { + refId: '_x' + }), + v('y', '((x+2)*3)', { + refId: '_y' + }) + ]) + }) + + it('should not reduce an equation that does not involve a conditional', () => { + const vars = readInlineModel( + 'default', + ` x = 1 ~~| y = (x + 2) * 3 ~~| ` - ) - expect(vars).toEqual([ - v('x', '1', { - refId: '_x' - }), - v('y', '(x+2)*3', { - refId: '_y' - }) - ]) - }) - - it('should not reduce an equation when the condition cannot be reduced', () => { - const vars = readInlineModel( - 'default', - ` + ) + expect(vars).toEqual([ + v('x', '1', { + refId: '_x' + }), + v('y', '(x+2)*3', { + refId: '_y' + }) + ]) + }) + + it('should not reduce an equation when the condition cannot be reduced', () => { + const vars = readInlineModel( + 'default', + ` x = Time ~~| y = Time + 2 ~~| z = (x + y) * 3 ~~| ` - ) - expect(vars).toEqual([ - v('x', 'Time', { - refId: '_x' - }), - v('y', 'Time+2', { - refId: '_y' - }), - v('z', '(x+y)*3', { - refId: '_z' - }) - ]) - }) - } -) - -describe.skipIf(process.env.SDE_NONPUBLIC_USE_NEW_PARSE === '0')( - 'reduceVariables (aggressive mode: reduce everything)', - () => { - it('should reduce a simple equation to a constant', () => { - const vars = readInlineModel( - 'aggressive', - ` + ) + expect(vars).toEqual([ + v('x', 'Time', { + refId: '_x' + }), + v('y', 'Time+2', { + refId: '_y' + }), + v('z', '(x+y)*3', { + refId: '_z' + }) + ]) + }) +}) + +describe('reduceVariables (aggressive mode: reduce everything)', () => { + it('should reduce a simple equation to a constant', () => { + const vars = readInlineModel( + 'aggressive', + ` x = 1 ~~| y = (x + 2) * 3 ~~| ` - ) - expect(vars).toEqual([ - v('x', '1', { - refId: '_x' - }), - v('y', '9', { - refId: '_y' - }) - ]) - }) - - it('should not reduce an equation when variables cannot be reduced', () => { - const vars = readInlineModel( - 'aggressive', - ` + ) + expect(vars).toEqual([ + v('x', '1', { + refId: '_x' + }), + v('y', '9', { + refId: '_y' + }) + ]) + }) + + it('should not reduce an equation when variables cannot be reduced', () => { + const vars = readInlineModel( + 'aggressive', + ` x = Time ~~| y = Time + 2 ~~| z = (x + y) * 3 ~~| ` - ) - expect(vars).toEqual([ - v('x', 'Time', { - refId: '_x' - }), - v('y', 'Time+2', { - refId: '_y' - }), - v('z', '(x+y)*3', { - refId: '_z' - }) - ]) - }) - } -) + ) + expect(vars).toEqual([ + v('x', 'Time', { + refId: '_x' + }), + v('y', 'Time+2', { + refId: '_y' + }), + v('z', '(x+y)*3', { + refId: '_z' + }) + ]) + }) +}) diff --git a/tests/modeltests b/tests/modeltests index e5e31473..540741a2 100755 --- a/tests/modeltests +++ b/tests/modeltests @@ -68,11 +68,15 @@ function compare { MODEL_NEW_C=$COMPARE_DIR/${MODEL}_new.c mkdir -p $COMPARE_DIR - export SDE_NONPUBLIC_USE_NEW_PARSE=0 + # TODO: We no longer have an environment variable to switch between the old + # and new parser implementations, so this code is not useful in its current + # form but is being left here as a guide for the future in case we want to + # be able to compare compiler output for two different code paths + #export SDE_NONPUBLIC_USE_NEW_PARSE=0 test $1 "with legacy parser" cp $MODEL_DIR/build/$MODEL.c $MODEL_OLD_C - export SDE_NONPUBLIC_USE_NEW_PARSE=1 + #export SDE_NONPUBLIC_USE_NEW_PARSE=1 test $1 "with new parser" cp $MODEL_DIR/build/$MODEL.c $MODEL_NEW_C From 4a2fe1e8e5d5d4062792d005d3592b7dc2dfe94b Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 13:03:53 -0700 Subject: [PATCH 10/15] fix: update `sde causes` command to work with new `parseModel` function --- packages/cli/src/sde-causes.js | 8 ++++++-- packages/compile/src/index.js | 4 +--- packages/compile/src/model/model.js | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/sde-causes.js b/packages/cli/src/sde-causes.js index 58962145..e87b3fb0 100644 --- a/packages/cli/src/sde-causes.js +++ b/packages/cli/src/sde-causes.js @@ -21,11 +21,15 @@ let causes = (model, varname, opts) => { let directData = new Map() let spec = parseSpec(opts.spec) // Preprocess model text into parser input. + // TODO: The legacy `parseModel` function previously required the `preprocessModel` + // step to be performed first, but the new `parseModel` runs the preprocessor + // implicitly, so we can remove this step (and can simplify this code to use + // `parseAndGenerate` instead) let input = preprocessModel(modelPathname, spec) // Parse the model to get variable and subscript information. - let parseTree = parseModel(input) + let parsedModel = parseModel(input, modelDirname) let operations = ['printRefGraph'] - generateCode(parseTree, { spec, operations, extData, directData, modelDirname, varname }) + generateCode(parsedModel, { spec, operations, extData, directData, modelDirname, varname }) } export default { command, diff --git a/packages/compile/src/index.js b/packages/compile/src/index.js index 5ea7bdf4..49c89ef5 100644 --- a/packages/compile/src/index.js +++ b/packages/compile/src/index.js @@ -3,7 +3,5 @@ export { canonicalName } from './_shared/helpers.js' export { readDat } from './_shared/read-dat.js' export { preprocessModel } from './preprocess/preprocessor.js' -// TODO -// export { parseModel } from './parse/parser.js' export { generateCode } from './generate/code-gen.js' -export { parseAndGenerate, printNames } from './parse-and-generate.js' +export { parseAndGenerate, parseModel, printNames } from './parse-and-generate.js' diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index c405b9f8..8e8615f4 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -77,7 +77,7 @@ function read(parsedModel, spec, extData, directData, modelDirname, opts) { const vars = readVariables(parsedModel, specialSeparationDims) // Include a placeholder variable for the exogenous `Time` variable - const timeVar = new Variable(null) + const timeVar = new Variable() timeVar.modelLHS = 'Time' timeVar.varName = '_time' vars.push(timeVar) From 13da3b43e8695931b463469414b32eb4ce36f2f3 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 14:18:15 -0700 Subject: [PATCH 11/15] fix: temporarily disable code in checkSpecVars --- packages/compile/src/model/model.js | 49 +++++++++++++++-------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index 8e8615f4..c0c8dc5c 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -290,15 +290,16 @@ function checkSpecVars(spec, extData) { if (!varWithRefId(varName)) { // Look for a variable in external data. if (extData?.has(varName)) { - // console.error(`found ${specType} ${varName} in extData`) - // Copy data from an external file to an equation that does a lookup. - let lookup = R.reduce( - (a, p) => listConcat(a, `(${p[0]}, ${p[1]})`, true), - '', - Array.from(extData.get(varName)) - ) - let modelEquation = `${decanonicalize(varName)} = WITH LOOKUP(Time, (${lookup}))` - addEquation(modelEquation) + throw new Error('Not implemented') + // // console.error(`found ${specType} ${varName} in extData`) + // // Copy data from an external file to an equation that does a lookup. + // let lookup = R.reduce( + // (a, p) => listConcat(a, `(${p[0]}, ${p[1]})`, true), + // '', + // Array.from(extData.get(varName)) + // ) + // let modelEquation = `${decanonicalize(varName)} = WITH LOOKUP(Time, (${lookup}))` + // addEquation(modelEquation) } else { throw new Error( `The ${specType} variable ${varName} was declared in spec.json, but no matching variable was found in the model or external data sources` @@ -505,21 +506,21 @@ function setRefIds() { v.refId = refIdForVar(v) }, variables) } -function addEquation(modelEquation) { - // TODO - // // Add an equation in Vensim model format. - // let parser = createParser(modelEquation) - // let tree = parser.equation() - // // Read the var and add it to the Model var table. - // let variableReader = new VariableReader() - // variableReader.visitEquation(tree) - // let v = variableReader.var - // // Fill in the refId. - // v.refId = refIdForVar(v) - // // Finish the variable by parsing the RHS. - // let equationReader = new EquationReader(v) - // equationReader.read() -} +// TODO +// function addEquation(modelEquation) { +// // Add an equation in Vensim model format. +// let parser = createParser(modelEquation) +// let tree = parser.equation() +// // Read the var and add it to the Model var table. +// let variableReader = new VariableReader() +// variableReader.visitEquation(tree) +// let v = variableReader.var +// // Fill in the refId. +// v.refId = refIdForVar(v) +// // Finish the variable by parsing the RHS. +// let equationReader = new EquationReader(v) +// equationReader.read() +// } // // Model API // From 8fdb31c43ee626f0cf355d10bd85f9bfa8f080ca Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 22 Apr 2024 14:18:40 -0700 Subject: [PATCH 12/15] fix: comment out import --- packages/compile/src/model/model.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index c0c8dc5c..13c853ed 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -2,7 +2,7 @@ import B from 'bufx' import yaml from 'js-yaml' import * as R from 'ramda' -import { canonicalName, decanonicalize, isIterable, listConcat, strlist, vlog, vsort } from '../_shared/helpers.js' +import { canonicalName, decanonicalize, isIterable, /*listConcat,*/ strlist, vlog, vsort } from '../_shared/helpers.js' import { addIndex, allAliases, From a13eb2498f285fd8b0ec7225aaf47a0ae138397a Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Thu, 25 Apr 2024 11:33:33 -0700 Subject: [PATCH 13/15] build: update examples and templates to use more recent versions of SDE dependencies --- examples/hello-world/package.json | 10 +++++----- examples/template-default/package.json | 12 ++++++------ examples/template-minimal/package.json | 10 +++++----- pnpm-lock.yaml | 10 +++++----- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/examples/hello-world/package.json b/examples/hello-world/package.json index c5cfd08c..084522d8 100644 --- a/examples/hello-world/package.json +++ b/examples/hello-world/package.json @@ -12,11 +12,11 @@ "save-baseline": "sde-check baseline --save" }, "dependencies": { - "@sdeverywhere/build": "^0.3.0", - "@sdeverywhere/cli": "^0.7.6", - "@sdeverywhere/plugin-check": "^0.3.0", - "@sdeverywhere/plugin-wasm": "^0.2.0", - "@sdeverywhere/plugin-worker": "^0.2.0", + "@sdeverywhere/build": "^0.3.4", + "@sdeverywhere/cli": "^0.7.23", + "@sdeverywhere/plugin-check": "^0.3.5", + "@sdeverywhere/plugin-wasm": "^0.2.3", + "@sdeverywhere/plugin-worker": "^0.2.3", "vite": "^4.4.9" } } diff --git a/examples/template-default/package.json b/examples/template-default/package.json index ee5bddee..b87e16ac 100644 --- a/examples/template-default/package.json +++ b/examples/template-default/package.json @@ -13,13 +13,13 @@ "packages/app" ], "dependencies": { - "@sdeverywhere/build": "^0.3.2", - "@sdeverywhere/check-core": "^0.1.1", - "@sdeverywhere/cli": "^0.7.12", - "@sdeverywhere/plugin-check": "^0.3.4", - "@sdeverywhere/plugin-config": "^0.2.3", + "@sdeverywhere/build": "^0.3.4", + "@sdeverywhere/check-core": "^0.1.2", + "@sdeverywhere/cli": "^0.7.23", + "@sdeverywhere/plugin-check": "^0.3.5", + "@sdeverywhere/plugin-config": "^0.2.4", "@sdeverywhere/plugin-vite": "^0.1.8", - "@sdeverywhere/plugin-wasm": "^0.2.1", + "@sdeverywhere/plugin-wasm": "^0.2.3", "@sdeverywhere/plugin-worker": "^0.2.3" } } diff --git a/examples/template-minimal/package.json b/examples/template-minimal/package.json index 3ec9740f..e9178382 100644 --- a/examples/template-minimal/package.json +++ b/examples/template-minimal/package.json @@ -9,11 +9,11 @@ "save-baseline": "sde-check baseline --save" }, "dependencies": { - "@sdeverywhere/build": "^0.3.2", - "@sdeverywhere/check-core": "^0.1.1", - "@sdeverywhere/cli": "^0.7.12", - "@sdeverywhere/plugin-check": "^0.3.4", - "@sdeverywhere/plugin-wasm": "^0.2.1", + "@sdeverywhere/build": "^0.3.4", + "@sdeverywhere/check-core": "^0.1.2", + "@sdeverywhere/cli": "^0.7.23", + "@sdeverywhere/plugin-check": "^0.3.5", + "@sdeverywhere/plugin-wasm": "^0.2.3", "@sdeverywhere/plugin-worker": "^0.2.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 57c0984e..6be68386 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,19 +54,19 @@ importers: examples/hello-world: dependencies: '@sdeverywhere/build': - specifier: ^0.3.0 + specifier: ^0.3.4 version: link:../../packages/build '@sdeverywhere/cli': - specifier: ^0.7.6 + specifier: ^0.7.23 version: link:../../packages/cli '@sdeverywhere/plugin-check': - specifier: ^0.3.0 + specifier: ^0.3.5 version: link:../../packages/plugin-check '@sdeverywhere/plugin-wasm': - specifier: ^0.2.0 + specifier: ^0.2.3 version: link:../../packages/plugin-wasm '@sdeverywhere/plugin-worker': - specifier: ^0.2.0 + specifier: ^0.2.3 version: link:../../packages/plugin-worker vite: specifier: ^4.4.9 From 794afdfd468dd06b008638fdcd84b1c3eee03433 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Thu, 25 Apr 2024 11:35:05 -0700 Subject: [PATCH 14/15] fix: remove direct antlr4 dependencies from compile package (these are dependencies for the parse package only) --- packages/compile/package.json | 2 -- pnpm-lock.yaml | 6 ------ 2 files changed, 8 deletions(-) diff --git a/packages/compile/package.json b/packages/compile/package.json index 9848b46a..47b03fdd 100644 --- a/packages/compile/package.json +++ b/packages/compile/package.json @@ -17,8 +17,6 @@ }, "dependencies": { "@sdeverywhere/parse": "^0.1.0", - "antlr4": "4.12.0", - "antlr4-vensim": "0.6.2", "bufx": "^1.0.5", "byline": "^5.0.0", "csv-parse": "^5.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6be68386..1e34fc82 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -271,12 +271,6 @@ importers: '@sdeverywhere/parse': specifier: ^0.1.0 version: link:../parse - antlr4: - specifier: 4.12.0 - version: 4.12.0 - antlr4-vensim: - specifier: 0.6.2 - version: 0.6.2 bufx: specifier: ^1.0.5 version: 1.0.5 From d21d062c0f8084dadaa21fcbe82793409aa922df Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Thu, 2 May 2024 13:05:18 -0700 Subject: [PATCH 15/15] fix: remove unreachable code in checkSpecVars This code was used long ago for synthesizing lookups for external data that is not referenced in the model, but Todd and I decided that the code is no longer relevant and should be removed. --- packages/compile/src/model/model.js | 39 ++++------------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/packages/compile/src/model/model.js b/packages/compile/src/model/model.js index 13c853ed..e13dc99d 100644 --- a/packages/compile/src/model/model.js +++ b/packages/compile/src/model/model.js @@ -106,7 +106,7 @@ function read(parsedModel, spec, extData, directData, modelDirname, opts) { if (opts?.stopAfterAnalyze) return // Check that all input and output vars in the spec actually exist in the model. - checkSpecVars(spec, extData) + checkSpecVars(spec) // Remove variables that are not referenced by an input or output variable. removeUnusedVariables(spec) @@ -277,7 +277,7 @@ function analyze(parsedModelKind, inputVars, opts) { variables.forEach(readEquation) } -function checkSpecVars(spec, extData) { +function checkSpecVars(spec) { // Look up each var in the spec and issue and throw error if it does not exist. function check(varNames, specType) { @@ -288,23 +288,9 @@ function checkSpecVars(spec, extData) { // out of the valid range) if (!R.contains('[', varName)) { if (!varWithRefId(varName)) { - // Look for a variable in external data. - if (extData?.has(varName)) { - throw new Error('Not implemented') - // // console.error(`found ${specType} ${varName} in extData`) - // // Copy data from an external file to an equation that does a lookup. - // let lookup = R.reduce( - // (a, p) => listConcat(a, `(${p[0]}, ${p[1]})`, true), - // '', - // Array.from(extData.get(varName)) - // ) - // let modelEquation = `${decanonicalize(varName)} = WITH LOOKUP(Time, (${lookup}))` - // addEquation(modelEquation) - } else { - throw new Error( - `The ${specType} variable ${varName} was declared in spec.json, but no matching variable was found in the model or external data sources` - ) - } + throw new Error( + `The ${specType} variable ${varName} was declared in spec.json, but no matching variable was found in the model or external data sources` + ) } } } @@ -506,21 +492,6 @@ function setRefIds() { v.refId = refIdForVar(v) }, variables) } -// TODO -// function addEquation(modelEquation) { -// // Add an equation in Vensim model format. -// let parser = createParser(modelEquation) -// let tree = parser.equation() -// // Read the var and add it to the Model var table. -// let variableReader = new VariableReader() -// variableReader.visitEquation(tree) -// let v = variableReader.var -// // Fill in the refId. -// v.refId = refIdForVar(v) -// // Finish the variable by parsing the RHS. -// let equationReader = new EquationReader(v) -// equationReader.read() -// } // // Model API //