fix: read blank lines and blank cells in CSV files (#309)

Fixes #308

Co-authored-by: Chris Campbell <[email protected]>

models/directconst/data/h.csv

@@ -1,4 +1,6 @@
-c,C1,C2
-B1,1,2
-B2,3,ZERO
-B3,5,6
+h,
+A,1
+B,2
+,
+C,3
+D,4

models/directconst/directconst.dat

@@ -64,18 +64,8 @@ g[C2,C1]
 0	21
 g[C2,C2]
 0	22
-h[B1,C1]
-0	1
-h[B1,C2]
-0	2
-h[B2,C1]
+h
 0	3
-h[B2,C2]
-0	0
-h[B3,C1]
-0	5
-h[B3,C2]
-0	6
 INITIAL TIME
 0	0
 SAVEPER

models/directconst/directconst.mdl

@@ -59,18 +59,17 @@ g[From DimC, To DimC] =
     'B2'
   ) ~~~:SUPPLEMENTARY|
 
-h[DimB, DimC] =
+h =
   GET DIRECT CONSTANTS(
     'data/h.csv',
     ',',
-    'B2'
+    'B5'
   )
-  ~
-  ~ This csv file has a cell that does not contain a number so that we can check that
-    SDE does not throw an error in this case.  Note that Vensim will raise an error if
-    a cell in the csv file does not contain a number, so to run this test in Vensim,
-    temporarily change cell C3 to 0 in `h.csv`.
-  ~:SUPPLEMENTARY|
+	~	
+	~	This csv file has a blank line. The formula will have the correct value
+		only if SDEverywhere retains blank lines.
+	~	:SUPPLEMENTARY 
+	|
 
 ********************************************************
 	.Control

models/directconst_invalidcell/data/c.csv

@@ -0,0 +1,4 @@
+c,C1,C2
+B1,1,2
+B2,3,ZERO
+B3,5,6

models/directconst_invalidcell/directconst_invalidcell.dat

@@ -0,0 +1,21 @@
+c[B1,C1]
+0	1
+c[B1,C2]
+0	2
+c[B2,C1]
+0	3
+c[B2,C2]
+0	0
+c[B3,C1]
+0	5
+c[B3,C2]
+0	6
+FINAL TIME
+0	1
+INITIAL TIME
+0	0
+SAVEPER
+0	1
+1	1
+TIME STEP
+0	1

models/directconst_invalidcell/directconst_invalidcell.mdl

@@ -0,0 +1,51 @@
+{UTF-8}
+DimB: B1, B2, B3 ~~|
+DimC: C1, C2 ~~|
+
+c[DimB, DimC] =
+  GET DIRECT CONSTANTS(
+    'data/c.csv',
+    ',',
+    'B2'
+  )
+  ~
+  ~ This csv file has a cell that does not contain a number so that we can check that
+    SDE does not throw an error in this case.  Note that Vensim will raise an error if
+    a cell in the csv file does not contain a number, so to run this test in Vensim,
+    temporarily change cell C3 to 0 in `c.csv`.
+  ~:SUPPLEMENTARY|
+
+********************************************************
+	.Control
+********************************************************~
+		Simulation Control Parameters
+	|
+
+INITIAL TIME = 0 ~~|
+FINAL TIME = 1 ~~|
+TIME STEP = 1 ~~|
+SAVEPER = TIME STEP ~~|
+
+\\\---/// Sketch information - do not modify anything except names
+V300  Do not put anything below this section - it will be ignored
+*View 1
+$0-0-0,0,|0||0-0-0|0-0-0|0-0-0|0-0-0|0-0-0|0,0,100,0
+///---\\\
+:L<%^E!@
+9:directconst
+15:0,0,0,0,0,0
+19:100,0
+27:2,
+34:0,
+5:FINAL TIME
+35:Date
+36:YYYY-MM-DD
+37:2000
+38:1
+39:1
+40:2
+41:0
+42:1
+24:0
+25:0
+26:0

packages/cli/src/sde-compare.js

@@ -79,6 +79,10 @@ export let compare = async (vensimfile, sdefile, opts) => {
   }
   if (noDATDifference) {
     pr(`Data were the same for ${vensimfile} and ${sdefile}`)
+    return true
+  } else {
+    pr(`Data differences detected for ${vensimfile} and ${sdefile}`)
+    return false
   }
 }
 let isZero = value => {

packages/cli/src/sde-test.js

@@ -53,7 +53,12 @@ export let test = async (model, opts) => {
     let vensimPathname = path.join(modelDirname, `${modelName}.dat`)
     let p = path.parse(logPathname)
     let sdePathname = path.format({ dir: p.dir, name: p.name, ext: '.dat' })
-    await compare(vensimPathname, sdePathname, opts)
+    let noDiffs = await compare(vensimPathname, sdePathname, opts)
+    if (!noDiffs) {
+      // Exit with a non-zero error code if differences were detected
+      console.error()
+      process.exit(1)
+    }
   } catch (e) {
     // Exit with a non-zero error code if any step failed
     console.error(`ERROR: ${e.message}\n`)

packages/compile/src/_shared/helpers.js

@@ -224,8 +224,8 @@ export let readCsv = (pathname, delimiter = ',') => {
       delimiter,
       columns: false,
       trim: true,
-      skip_empty_lines: true,
-      skip_records_with_empty_values: true
+      skip_empty_lines: false,
+      skip_records_with_empty_values: false
     }
     try {
       let data = B.read(pathname)