fix: check for valid numbers when reading a CSV file (#288)

Fixes #284 

Co-authored-by: Chris Campbell <[email protected]>

models/directconst/data/h.csv

@@ -0,0 +1,4 @@
+c,C1,C2
+B1,1,2
+B2,3,ZERO
+B3,5,6

models/directconst/directconst.dat

@@ -64,6 +64,18 @@ g[C2,C1]
 0	21
 g[C2,C2]
 0	22
+h[B1,C1]
+0	1
+h[B1,C2]
+0	2
+h[B2,C1]
+0	3
+h[B2,C2]
+0	0
+h[B3,C1]
+0	5
+h[B3,C2]
+0	6
 INITIAL TIME
 0	0
 SAVEPER

models/directconst/directconst.mdl

@@ -59,6 +59,19 @@ g[From DimC, To DimC] =
     'B2'
   ) ~~~:SUPPLEMENTARY|
 
+h[DimB, DimC] =
+  GET DIRECT CONSTANTS(
+    'data/h.csv',
+    ',',
+    'B2'
+  )
+  ~
+  ~ This csv file has a cell that does not contain a number so that we can check that
+    SDE does not throw an error in this case.  Note that Vensim will raise an error if
+    a cell in the csv file does not contain a number, so to run this test in Vensim,
+    temporarily change cell C3 to 0 in `h.csv`.
+  ~:SUPPLEMENTARY|
+
 ********************************************************
 	.Control
 ********************************************************~

packages/compile/package.json

@@ -20,7 +20,7 @@
     "antlr4-vensim": "0.6.0",
     "bufx": "^1.0.5",
     "byline": "^5.0.0",
-    "csv-parse": "^4.15.4",
+    "csv-parse": "^5.3.3",
     "js-yaml": "^3.13.1",
     "ramda": "^0.27.0",
     "split-string": "^6.0.0",

packages/compile/src/_shared/helpers.js

@@ -1,6 +1,6 @@
 import util from 'util'
 import B from 'bufx'
-import parseCsv from 'csv-parse/lib/sync.js'
+import { parse as parseCsv } from 'csv-parse/sync'
 import R from 'ramda'
 import split from 'split-string'
 import XLSX from 'xlsx'
@@ -124,9 +124,29 @@ export let listConcat = (a, x, addSpaces = false) => {
     return a + (R.isEmpty(a) ? '' : `,${s}`) + x
   }
 }
+// Convert a number or string into a C double constant string.
+// A blank string is converted to zero, following Excel.
+// A string that cannot be converted throws an exception.
 export let cdbl = x => {
-  // Convert a number into a C double constant.
-  let s = x.toString()
+  function throwError() {
+    throw new Error(`ERROR: cannot convert "${x}" to a number`)
+  }
+  let s = '0.0'
+  if (typeof x === 'number') {
+    s = x.toString()
+  } else if (typeof x === 'string') {
+    if (x.trim() !== '') {
+      let f = parseFloat(x)
+      if (!Number.isNaN(f)) {
+        s = f.toString()
+      } else {
+        throwError()
+      }
+    }
+  } else {
+    throwError()
+  }
+  // Format as a C double literal with a decimal point.
   if (!s.includes('.') && !s.toLowerCase().includes('e')) {
     s += '.0'
   }
@@ -207,8 +227,12 @@ export let readCsv = (pathname, delimiter = ',') => {
       skip_empty_lines: true,
       skip_lines_with_empty_values: true
     }
-    let data = B.read(pathname)
-    csv = parseCsv(data, CSV_PARSE_OPTS)
+    try {
+      let data = B.read(pathname)
+      csv = parseCsv(data, CSV_PARSE_OPTS)
+    } catch (err) {
+      console.error(`ERROR: readCsv ${pathname} ${err.message}`)
+    }
     csvData.set(pathname, csv)
   }
   return csv

packages/compile/src/generate/equation-gen.js

@@ -383,7 +383,15 @@ export default class EquationGen extends ModelReader {
         let csvPathname = path.resolve(this.modelDirname, file)
         let data = readCsv(csvPathname, tab)
         if (data) {
-          getCellValue = (c, r) => (data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null)
+          getCellValue = (c, r) => {
+            let value = '0.0'
+            try {
+              value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null
+            } catch (error) {
+              console.error(`${error.message} in ${csvPathname}`)
+            }
+            return value
+          }
         }
       }
       // If the data was found, convert it to a lookup.
@@ -463,7 +471,15 @@ export default class EquationGen extends ModelReader {
     let csvPathname = path.resolve(this.modelDirname, file)
     let data = readCsv(csvPathname, tab)
     if (data) {
-      let getCellValue = (c, r) => (data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null)
+      let getCellValue = (c, r) => {
+        let value = '0.0'
+        try {
+          value = data[r] != null && data[r][c] != null ? cdbl(data[r][c]) : null
+        } catch (error) {
+          console.error(`${error.message} in ${csvPathname}`)
+        }
+        return value
+      }
       // Get C subscripts in text form for the LHS in normal order.
       let modelLHSReader = new ModelLHSReader()
       modelLHSReader.read(this.var.modelLHS)