diff --git a/models/directdata/directdata.dat b/models/directdata/directdata.dat index 119edbed..41d78ef5 100644 --- a/models/directdata/directdata.dat +++ b/models/directdata/directdata.dat @@ -184,8 +184,194 @@ d 2048 422 2049 446 2050 470 +f[A1] +1990 6100 +1991 6093.33 +1992 6086.67 +1993 6080 +1994 6073.33 +1995 6066.67 +1996 6060 +1997 6053.33 +1998 6046.67 +1999 6040 +2000 6033.33 +2001 6026.67 +2002 6020 +2003 6013.33 +2004 6006.67 +2005 6000 +2006 5990 +2007 5980 +2008 5970 +2009 5960 +2010 5950 +2011 5940 +2012 5930 +2013 5920 +2014 5910 +2015 5900 +2016 5902 +2017 5904 +2018 5906 +2019 5908 +2020 5910 +2021 5912 +2022 5914 +2023 5916 +2024 5918 +2025 5920 +2026 5922 +2027 5924 +2028 5926 +2029 5928 +2030 5930 +2031 5925 +2032 5920 +2033 5915 +2034 5910 +2035 5905 +2036 5900 +2037 5895 +2038 5890 +2039 5885 +2040 5880 +2041 5875 +2042 5870 +2043 5865 +2044 5860 +2045 5855 +2046 5850 +2047 5845 +2048 5840 +2049 5835 +2050 5830 +f[A2] +1990 2100 +1991 2093.33 +1992 2086.67 +1993 2080 +1994 2073.33 +1995 2066.67 +1996 2060 +1997 2053.33 +1998 2046.67 +1999 2040 +2000 2033.33 +2001 2026.67 +2002 2020 +2003 2013.33 +2004 2006.67 +2005 2000 +2006 1990 +2007 1980 +2008 1970 +2009 1960 +2010 1950 +2011 1940 +2012 1930 +2013 1920 +2014 1910 +2015 1900 +2016 1896.67 +2017 1893.33 +2018 1890 +2019 1886.67 +2020 1883.33 +2021 1880 +2022 1876.67 +2023 1873.33 +2024 1870 +2025 1866.67 +2026 1863.33 +2027 1860 +2028 1856.67 +2029 1853.33 +2030 1850 +2031 1847.5 +2032 1845 +2033 1842.5 +2034 1840 +2035 1837.5 +2036 1835 +2037 1832.5 +2038 1830 +2039 1827.5 +2040 1825 +2041 1822.5 +2042 1820 +2043 1817.5 +2044 1815 +2045 1812.5 +2046 1810 +2047 1807.5 +2048 1805 +2049 1802.5 +2050 1800 FINAL TIME 1990 2050 +h +1990 970 +1991 806 +1992 642 +1993 478 +1994 314 +1995 150 +1996 198 +1997 246 +1998 294 +1999 342 +2000 390 +2001 450 +2002 510 +2003 570 +2004 630 +2005 690 +2006 712 +2007 734 +2008 756 +2009 778 +2010 800 +2011 722 +2012 644 +2013 566 +2014 488 +2015 410 +2016 512 +2017 614 +2018 716 +2019 818 +2020 920 +2021 836 +2022 752 +2023 668 +2024 584 +2025 500 +2026 518 +2027 536 +2028 554 +2029 572 +2030 590 +2031 580 +2032 570 +2033 560 +2034 550 +2035 540 +2036 466 +2037 392 +2038 318 +2039 244 +2040 170 +2041 206 +2042 242 +2043 278 +2044 314 +2045 350 +2046 374 +2047 398 +2048 422 +2049 446 +2050 470 INITIAL TIME 1990 1990 SAVEPER @@ -278,3 +464,29 @@ c 2040 17 2045 35 2050 47 +e[A1] +1990 610 +2005 600 +2015 590 +2030 593 +2050 583 +e[A2] +1990 210 +2005 200 +2015 190 +2030 185 +2050 180 +g +1990 97 +1995 15 +2000 39 +2005 69 +2010 80 +2015 41 +2020 92 +2025 50 +2030 59 +2035 54 +2040 17 +2045 35 +2050 47 diff --git a/models/directdata/directdata.mdl b/models/directdata/directdata.mdl index ba822284..10afb071 100644 --- a/models/directdata/directdata.mdl +++ b/models/directdata/directdata.mdl @@ -1,9 +1,17 @@ {UTF-8} -a[DimA] := GET DIRECT DATA('?data', 'A Data', 'A', 'B2') ~~| DimA: A1, A2 ~~| -b[DimA] = a[DimA] * 10 ~~| + +a[DimA] := GET DIRECT DATA('?data', 'A Data', 'A', 'B2') ~~| +b[DimA] = a[DimA] * 10 ~~~:SUPPLEMENTARY| + c:= GET DIRECT DATA('?data', 'C Data', 'A', 'B2') ~~| -d = c * 10 ~~| +d = c * 10 ~~~:SUPPLEMENTARY| + +e[DimA] := GET DIRECT DATA('e_data.csv', ',', 'A', 'B2') ~~| +f[DimA] = e[DimA] * 10 ~~~:SUPPLEMENTARY| + +g:= GET DIRECT DATA('g_data.csv', ',', 'A', 'B2') ~~| +h = g * 10 ~~~:SUPPLEMENTARY| ******************************************************** .Control diff --git a/models/directdata/e_data.csv b/models/directdata/e_data.csv new file mode 100644 index 00000000..429ec617 --- /dev/null +++ b/models/directdata/e_data.csv @@ -0,0 +1,6 @@ +Year,A1,A2 +1990,610,210 +2005,600,200 +2015,590,190 +2030,593,185 +2050,583,180 \ No newline at end of file diff --git a/models/directdata/g_data.csv b/models/directdata/g_data.csv new file mode 100644 index 00000000..b8072144 --- /dev/null +++ b/models/directdata/g_data.csv @@ -0,0 +1,14 @@ +,g +1990,97 +1995,15 +2000,39 +2005,69 +2010,80 +2015,41 +2020,92 +2025,50 +2030,59 +2035,54 +2040,17 +2045,35 +2050,47 \ No newline at end of file diff --git a/src/CodeGen.js b/src/CodeGen.js index 786b1fc4..a73ae0b5 100644 --- a/src/CodeGen.js +++ b/src/CodeGen.js @@ -19,7 +19,7 @@ export let codeGenerator = (parseTree, opts) => { outputAllVars = true } // Function to generate a section of the code - let generateSection = R.map(v => new EquationGen(v, extData, directData, mode).generate()) + let generateSection = R.map(v => new EquationGen(v, extData, directData, mode, modelDirname).generate()) let section = R.pipe(generateSection, R.flatten, lines) function generate() { // Read variables and subscript ranges from the model parse tree. diff --git a/src/EquationGen.js b/src/EquationGen.js index 83474b18..28683839 100644 --- a/src/EquationGen.js +++ b/src/EquationGen.js @@ -1,3 +1,4 @@ +import path from 'path' import R from 'ramda' import XLSX from 'xlsx' import { ModelLexer, ModelParser } from 'antlr4-vensim' @@ -28,12 +29,13 @@ import { listConcat, newTmpVarName, permutationsOf, + readCsv, strToConst, vlog } from './Helpers.js' export default class EquationGen extends ModelReader { - constructor(variable, extData, directData, mode) { + constructor(variable, extData, directData, mode, modelDirname) { super() // the variable we are generating code for this.var = variable @@ -43,6 +45,8 @@ export default class EquationGen extends ModelReader { this.directData = directData // set to 'decl', 'init-lookups', 'eval', etc depending on the section being generated this.mode = mode + // The model directory is required when reading data files for GET DIRECT DATA. + this.modelDirname = modelDirname // Maps of LHS subscript families to loop index vars for lookup on the RHS this.loopIndexVars = new LoopIndexVars(['i', 'j', 'k']) this.arrayIndexVars = new LoopIndexVars(['v', 'w']) @@ -330,24 +334,42 @@ export default class EquationGen extends ModelReader { // If direct data exists for this variable, copy it from the workbook into one or more lookups. let result = [] if (this.mode === 'init-lookups') { - let { tag, sheetName, timeRowOrCol, startCell } = this.var.directDataArgs - let workbook = this.directData.get(tag) - if (workbook) { - let sheet = workbook.Sheets[sheetName] - if (sheet) { - let indexNum = 0 - if (!R.isEmpty(this.var.subscripts)) { - // Generate a lookup for a separated index in the variable's dimension. - // TODO allow the index to be in either position of a 2D subscript - let ind = sub(this.var.subscripts[0]) - indexNum = ind.value + let getCellValue + let { file, tab, timeRowOrCol, startCell } = this.var.directDataArgs + if (file.startsWith('?')) { + // The file is a tag for an Excel file with data in the directData map. + let workbook = this.directData.get(file) + if (workbook) { + let sheet = workbook.Sheets[tab] + if (sheet) { + getCellValue = (c, r) => { + let cell = sheet[XLSX.utils.encode_cell({ c, r })] + return cell != null ? cdbl(cell.v) : null + } + } else { + throw new Error(`ERROR: Direct data worksheet ${tab} tagged ${file} not found`) } - result.push(this.generateDirectDataLookup(sheet, timeRowOrCol, startCell, indexNum)) } else { - throw new Error(`ERROR: Direct data worksheet ${sheetName} tagged ${tag} not found`) + throw new Error(`ERROR: Direct data workbook tagged ${file} not found`) } } else { - throw new Error(`ERROR: Direct data workbook tagged ${tag} not found`) + // The file is a CSV pathname. Read it now. + let csvPathname = path.resolve(this.modelDirname, file) + let data = readCsv(csvPathname, tab) + if (data) { + getCellValue = (c, r) => (data[r] != null ? cdbl(data[r][c]) : null) + } + } + // If the data was found, convert it to a lookup. + if (getCellValue) { + let indexNum = 0 + if (!R.isEmpty(this.var.subscripts)) { + // Generate a lookup for a separated index in the variable's dimension. + // TODO allow the index to be in either position of a 2D subscript + let ind = sub(this.var.subscripts[0]) + indexNum = ind.value + } + result.push(this.generateDirectDataLookup(getCellValue, timeRowOrCol, startCell, indexNum)) } } return result @@ -368,7 +390,11 @@ export default class EquationGen extends ModelReader { if (mode === 'decl') { // In decl mode, declare a static data array that will be used to create the associated `Lookup` // at init time. See `generateLookup` for more details. - const points = R.reduce((a, p) => listConcat(a, `${cdbl(p[0])}, ${cdbl(p[1])}`, true), '', Array.from(data.entries())) + const points = R.reduce( + (a, p) => listConcat(a, `${cdbl(p[0])}, ${cdbl(p[1])}`, true), + '', + Array.from(data.entries()) + ) return `double ${dataName}[${data.size * 2}] = { ${points} };` } else if (mode === 'init-lookups') { // In init mode, create the `Lookup`, passing in a pointer to the static data array declared in decl mode. @@ -444,13 +470,12 @@ export default class EquationGen extends ModelReader { } return result } - - generateDirectDataLookup(sheet, timeRowOrCol, startCell, indexNum) { + generateDirectDataLookup(getCellValue, timeRowOrCol, startCell, indexNum) { // Read a row or column of data as (time, value) pairs from the worksheet. - let dataCol, dataRow, dataCell, timeCol, timeRow, timeCell, nextCell + // The cell(c,r) function wraps data access by column and row. + let dataCol, dataRow, dataValue, timeCol, timeRow, timeValue, nextCell let lookupData = '' let lookupSize = 0 - let cell = (c, r) => sheet[XLSX.utils.encode_cell({ c, r })] let dataAddress = XLSX.utils.decode_cell(startCell) dataCol = dataAddress.c dataRow = dataAddress.r @@ -473,14 +498,14 @@ export default class EquationGen extends ModelReader { timeCol++ } } - timeCell = cell(timeCol, timeRow) - dataCell = cell(dataCol, dataRow) - while (timeCell && dataCell) { - lookupData = listConcat(lookupData, `${cdbl(timeCell.v)}, ${cdbl(dataCell.v)}`, true) + timeValue = getCellValue(timeCol, timeRow) + dataValue = getCellValue(dataCol, dataRow) + while (timeValue != null && dataValue != null) { + lookupData = listConcat(lookupData, `${timeValue}, ${dataValue}`, true) lookupSize++ nextCell() - dataCell = cell(dataCol, dataRow) - timeCell = cell(timeCol, timeRow) + dataValue = getCellValue(dataCol, dataRow) + timeValue = getCellValue(timeCol, timeRow) } return [` ${this.lhs} = __new_lookup(${lookupSize}, /*copy=*/true, (double[]){ ${lookupData} });`] } @@ -725,7 +750,7 @@ export default class EquationGen extends ModelReader { // (since Vensim indices are one-based). let s = this.rhsSubscriptGen([varName]) // Remove the brackets around the C subscript expression. - s = s.slice(1, s.length-1) + s = s.slice(1, s.length - 1) this.emit(`(${s} + 1)`) } } else { diff --git a/src/EquationReader.js b/src/EquationReader.js index 29dfc582..581234b5 100644 --- a/src/EquationReader.js +++ b/src/EquationReader.js @@ -127,13 +127,17 @@ export default class EquationReader extends ModelReader { this.expandDelayFunction(fn, args) } else if (fn === '_GET_DIRECT_DATA') { // Extract string constant arguments into an object used in code generation. + // For Excel files, the file argument names an indirect "?" file tag from the model settings. + // For CSV files, it gives a relative pathname in the model directory. + // For Excel files, the tab argument names an Excel worksheet. + // For CSV files, it gives the delimiter character. let args = R.map( arg => matchRegex(arg, /'(.*)'/), R.map(expr => expr.getText(), ctx.expr()) ) this.var.directDataArgs = { - tag: args[0], - sheetName: args[1], + file: args[0], + tab: args[1], timeRowOrCol: args[2], startCell: args[3] } diff --git a/src/Helpers.js b/src/Helpers.js index ca229fdd..af45b717 100644 --- a/src/Helpers.js +++ b/src/Helpers.js @@ -325,13 +325,8 @@ export let readCsv = (pathname, delimiter = ',') => { skip_empty_lines: true, skip_lines_with_empty_values: true } - try { - let data = B.read(pathname) - result = parseCsv(data, CSV_PARSE_OPTS) - } catch (error) { - console.error(`ERROR: CSV file ${pathname} not found`) - } - return result + let data = B.read(pathname) + return parseCsv(data, CSV_PARSE_OPTS) } // Convert the var name and subscript names to canonical form separately. export let canonicalVensimName = vname => { diff --git a/src/SubscriptRangeReader.js b/src/SubscriptRangeReader.js index dae67113..2d3f030f 100644 --- a/src/SubscriptRangeReader.js +++ b/src/SubscriptRangeReader.js @@ -109,11 +109,13 @@ export default class SubscriptRangeReader extends ModelReader { // Read subscript names from the CSV file at the given position. let csvPathname = path.resolve(this.modelDirname, pathname) let data = readCsv(csvPathname, delimiter) - let indexName = data[row][col] - while (indexName != null) { - this.indNames.push(indexName) - nextCell() - indexName = data[row] != null ? data[row][col] : null + if (data) { + let indexName = data[row][col] + while (indexName != null) { + this.indNames.push(indexName) + nextCell() + indexName = data[row] != null ? data[row][col] : null + } } super.visitExprList(ctx) } diff --git a/src/sde-generate.js b/src/sde-generate.js index 78fa4406..229ea966 100644 --- a/src/sde-generate.js +++ b/src/sde-generate.js @@ -107,12 +107,12 @@ export let generate = async (model, opts) => { extData = new Map([...extData, ...data]) } } - // Attach Excel workbook data to directData entries by tag name. + // Attach Excel workbook data to directData entries by file name. let directData = new Map() if (spec.directData) { - for (let [tag, xlsxFilename] of Object.entries(spec.directData)) { + for (let [file, xlsxFilename] of Object.entries(spec.directData)) { let pathname = path.join(modelDirname, xlsxFilename) - directData.set(tag, readXlsx(pathname)) + directData.set(file, readXlsx(pathname)) } } // Produce a runnable model with the "genc" and "preprocess" options.