From 7fa249ec14af298926b865bb437034b7f1bc6b24 Mon Sep 17 00:00:00 2001 From: Todd Fincannon <66087780+ToddFincannonEI@users.noreply.github.com> Date: Thu, 8 Dec 2022 15:47:47 -0800 Subject: [PATCH] fix: check for valid numbers when reading a CSV file (#288) Fixes #284 Co-authored-by: Chris Campbell --- models/directconst/data/h.csv | 4 +++ models/directconst/directconst.dat | 12 +++++++ models/directconst/directconst.mdl | 13 +++++++ packages/compile/package.json | 2 +- packages/compile/src/_shared/helpers.js | 34 ++++++++++++++++--- packages/compile/src/generate/equation-gen.js | 20 +++++++++-- pnpm-lock.yaml | 8 +++-- 7 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 models/directconst/data/h.csv diff --git a/models/directconst/data/h.csv b/models/directconst/data/h.csv new file mode 100644 index 00000000..00b94159 --- /dev/null +++ b/models/directconst/data/h.csv @@ -0,0 +1,4 @@ +c,C1,C2 +B1,1,2 +B2,3,ZERO +B3,5,6 \ No newline at end of file diff --git a/models/directconst/directconst.dat b/models/directconst/directconst.dat index dcda9002..cc10d161 100644 --- a/models/directconst/directconst.dat +++ b/models/directconst/directconst.dat @@ -64,6 +64,18 @@ g[C2,C1] 0 21 g[C2,C2] 0 22 +h[B1,C1] +0 1 +h[B1,C2] +0 2 +h[B2,C1] +0 3 +h[B2,C2] +0 0 +h[B3,C1] +0 5 +h[B3,C2] +0 6 INITIAL TIME 0 0 SAVEPER diff --git a/models/directconst/directconst.mdl b/models/directconst/directconst.mdl index 1cc7d4d2..639336ea 100644 --- a/models/directconst/directconst.mdl +++ b/models/directconst/directconst.mdl @@ -59,6 +59,19 @@ g[From DimC, To DimC] = 'B2' ) ~~~:SUPPLEMENTARY| +h[DimB, DimC] = + GET DIRECT CONSTANTS( + 'data/h.csv', + ',', + 'B2' + ) + ~ + ~ This csv file has a cell that does not contain a number so that we can check that + SDE does not throw an error in this case. Note that Vensim will raise an error if + a cell in the csv file does not contain a number, so to run this test in Vensim, + temporarily change cell C3 to 0 in `h.csv`. + ~:SUPPLEMENTARY| + ******************************************************** .Control ********************************************************~ diff --git a/packages/compile/package.json b/packages/compile/package.json index cf649981..7c9da16d 100644 --- a/packages/compile/package.json +++ b/packages/compile/package.json @@ -20,7 +20,7 @@ "antlr4-vensim": "0.6.0", "bufx": "^1.0.5", "byline": "^5.0.0", - "csv-parse": "^4.15.4", + "csv-parse": "^5.3.3", "js-yaml": "^3.13.1", "ramda": "^0.27.0", "split-string": "^6.0.0", diff --git a/packages/compile/src/_shared/helpers.js b/packages/compile/src/_shared/helpers.js index ffcc7e49..3b3233dd 100644 --- a/packages/compile/src/_shared/helpers.js +++ b/packages/compile/src/_shared/helpers.js @@ -1,6 +1,6 @@ import util from 'util' import B from 'bufx' -import parseCsv from 'csv-parse/lib/sync.js' +import { parse as parseCsv } from 'csv-parse/sync' import R from 'ramda' import split from 'split-string' import XLSX from 'xlsx' @@ -124,9 +124,29 @@ export let listConcat = (a, x, addSpaces = false) => { return a + (R.isEmpty(a) ? '' : `,${s}`) + x } } +// Convert a number or string into a C double constant string. +// A blank string is converted to zero, following Excel. +// A string that cannot be converted throws an exception. export let cdbl = x => { - // Convert a number into a C double constant. - let s = x.toString() + function throwError() { + throw new Error(`ERROR: cannot convert "${x}" to a number`) + } + let s = '0.0' + if (typeof x === 'number') { + s = x.toString() + } else if (typeof x === 'string') { + if (x.trim() !== '') { + let f = parseFloat(x) + if (!Number.isNaN(f)) { + s = f.toString() + } else { + throwError() + } + } + } else { + throwError() + } + // Format as a C double literal with a decimal point. if (!s.includes('.') && !s.toLowerCase().includes('e')) { s += '.0' } @@ -207,8 +227,12 @@ export let readCsv = (pathname, delimiter = ',') => { skip_empty_lines: true, skip_lines_with_empty_values: true } - let data = B.read(pathname) - csv = parseCsv(data, CSV_PARSE_OPTS) + try { + let data = B.read(pathname) + csv = parseCsv(data, CSV_PARSE_OPTS) + } catch (err) { + console.error(`ERROR: readCsv ${pathname} ${err.message}`) + } csvData.set(pathname, csv) } return csv diff --git a/packages/compile/src/generate/equation-gen.js b/packages/compile/src/generate/equation-gen.js index f0091c85..0369e8fc 100644 --- a/packages/compile/src/generate/equation-gen.js +++ b/packages/compile/src/generate/equation-gen.js @@ -383,7 +383,15 @@ export default class EquationGen extends ModelReader { let csvPathname = path.resolve(this.modelDirname, file) let data = readCsv(csvPathname, tab) if (data) { - getCellValue = (c, r) => (data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null) + getCellValue = (c, r) => { + let value = '0.0' + try { + value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null + } catch (error) { + console.error(`${error.message} in ${csvPathname}`) + } + return value + } } } // If the data was found, convert it to a lookup. @@ -463,7 +471,15 @@ export default class EquationGen extends ModelReader { let csvPathname = path.resolve(this.modelDirname, file) let data = readCsv(csvPathname, tab) if (data) { - let getCellValue = (c, r) => (data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null) + let getCellValue = (c, r) => { + let value = '0.0' + try { + value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null + } catch (error) { + console.error(`${error.message} in ${csvPathname}`) + } + return value + } // Get C subscripts in text form for the LHS in normal order. let modelLHSReader = new ModelLHSReader() modelLHSReader.read(this.var.modelLHS) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5d7ca1df..bad50949 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -201,7 +201,7 @@ importers: antlr4-vensim: 0.6.0 bufx: ^1.0.5 byline: ^5.0.0 - csv-parse: ^4.15.4 + csv-parse: ^5.3.3 js-yaml: ^3.13.1 ramda: ^0.27.0 split-string: ^6.0.0 @@ -211,7 +211,7 @@ importers: antlr4-vensim: 0.6.0 bufx: 1.0.5 byline: 5.0.0 - csv-parse: 4.16.3 + csv-parse: 5.3.3 js-yaml: 3.14.1 ramda: 0.27.2 split-string: 6.1.0 @@ -1250,6 +1250,10 @@ packages: resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==} dev: false + /csv-parse/5.3.3: + resolution: {integrity: sha512-kEWkAPleNEdhFNkHQpFHu9RYPogsFj3dx6bCxL847fsiLgidzWg0z/O0B1kVWMJUc5ky64zGp18LX2T3DQrOfw==} + dev: false + /debug/4.3.4: resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} engines: {node: '>=6.0'}