fix: make sde log wait for DAT file to be fully written and improve…

… error handling (#123) Fixes #122
climateinteractive · Sep 28, 2021 · 34f25f8 · 34f25f8
1 parent d92343e
commit 34f25f8
Show file tree

Hide file tree

Showing 7 changed files with 88 additions and 69 deletions.
diff --git a/src/Helpers.js b/src/Helpers.js
@@ -212,11 +212,7 @@ export let ensureDir = (dir, defaultDir, modelDirname) => {
   return dirName
 }
 export let fileExists = pathname => {
-  let exists = fs.existsSync(pathname)
-  if (!exists) {
-    console.error(`${pathname} not found`)
-  }
-  return exists
+  return fs.existsSync(pathname)
 }
 export let linkCSourceFiles = (modelDirname, buildDirname) => {
   let cDirname = path.join(new URL('.', import.meta.url).pathname, 'c')
@@ -255,9 +251,9 @@ export let execCmd = cmd => {
   // Run a command line silently in the "sh" shell. Print error output on error.
   let exitCode = 0
   let result = sh.exec(cmd, { silent: true })
-  if (sh.error()) {
-    console.log(result.stderr)
-    exitCode = 1
+  if (result.code !== 0) {
+    console.error(result.stderr)
+    exitCode = result.code
   }
   return exitCode
 }

diff --git a/src/sde-build.js b/src/sde-build.js
@@ -19,9 +19,15 @@ export let handler = argv => {
   build(argv.model, argv)
 }
 export let build = async (model, opts) => {
-  opts.genc = true
-  await generate(model, opts)
-  compile(model, opts)
+  try {
+    opts.genc = true
+    await generate(model, opts)
+    compile(model, opts)
+  } catch (e) {
+    // Exit with a non-zero error code if any step failed
+    console.error(`ERROR: ${e.message}\n`)
+    process.exit(1)
+  }
 }
 
 export default {

diff --git a/src/sde-compare.js b/src/sde-compare.js
@@ -28,50 +28,55 @@ export let handler = argv => {
   compare(argv.vensimlog, argv.sdelog, argv)
 }
 export let compare = async (vensimfile, sdefile, opts) => {
+  if (!fileExists(vensimfile)) {
+    throw new Error(`Vensim DAT file not found: ${vensimfile}`)
+  }
+  if (!fileExists(sdefile)) {
+    throw new Error(`SDEverywhere DAT file not found: ${sdefile}`)
+  }
+
+  let vensimLog = await readDat(vensimfile)
+  let sdeLog = await readDat(sdefile)
+
+  if (vensimLog.size === 0) {
+    throw new Error(`Vensim DAT file did not contain data: ${vensimfile}`)
+  }
+  if (sdeLog.size === 0) {
+    throw new Error(`SDEverywhere DAT file did not contain data: ${sdefile}`)
+  }
+
   if (opts.precision) {
     ε = opts.precision
   }
-  if (fileExists(vensimfile) && fileExists(sdefile)) {
-    let vensimLog = await readDat(vensimfile)
-    let sdeLog = await readDat(sdefile)
-    if (vensimLog.size > 0 && sdeLog.size > 0) {
-      let noDATDifference = true
-      for (let varName of vensimLog.keys()) {
-        let sdeValues = sdeLog.get(varName)
-        // Ignore variables that are not found in the SDE log file.
-        if (sdeValues && (!opts.name || varName === opts.name)) {
-          let vensimValue = undefined
-          let vensimValues = vensimLog.get(varName)
-          // Filter on time t, the key in the values list.
-          for (let t of vensimValues.keys()) {
-            if (!opts.times || R.find(time => isEqual(time, t), opts.times)) {
-              // In Vensim log files, const vars only have one value at the initial time.
-              if (vensimValues.size > 1 || !vensimValue) {
-                vensimValue = vensimValues.get(t)
-              }
-              let sdeValue = sdeValues.get(t)
-              let diff = difference(sdeValue, vensimValue)
-              if (diff > ε) {
-                let diffPct = (diff * 100).toFixed(6)
-                pr(`${varName} time=${t.toFixed(2)} vensim=${vensimValue} sde=${sdeValue} diff=${diffPct}%`)
-                noDATDifference = false
-              }
-            }
+
+  let noDATDifference = true
+  for (let varName of vensimLog.keys()) {
+    let sdeValues = sdeLog.get(varName)
+    // Ignore variables that are not found in the SDE log file.
+    if (sdeValues && (!opts.name || varName === opts.name)) {
+      let vensimValue = undefined
+      let vensimValues = vensimLog.get(varName)
+      // Filter on time t, the key in the values list.
+      for (let t of vensimValues.keys()) {
+        if (!opts.times || R.find(time => isEqual(time, t), opts.times)) {
+          // In Vensim log files, const vars only have one value at the initial time.
+          if (vensimValues.size > 1 || !vensimValue) {
+            vensimValue = vensimValues.get(t)
+          }
+          let sdeValue = sdeValues.get(t)
+          let diff = difference(sdeValue, vensimValue)
+          if (diff > ε) {
+            let diffPct = (diff * 100).toFixed(6)
+            pr(`${varName} time=${t.toFixed(2)} vensim=${vensimValue} sde=${sdeValue} diff=${diffPct}%`)
+            noDATDifference = false
           }
         }
       }
-      if (noDATDifference) {
-        pr(`Data were the same for ${vensimfile} and ${sdefile}`)
-      }
-    } else {
-      if (vensimLog.size === 0) {
-        console.error(`${vensimfile} did not contain Vensim data`)
-      }
-      if (sdeLog.size === 0) {
-        console.error(`${sdefile} did not contain Vensim data`)
-      }
     }
   }
+  if (noDATDifference) {
+    pr(`Data were the same for ${vensimfile} and ${sdefile}`)
+  }
 }
 let isZero = value => {
   return Math.abs(value) < ε

diff --git a/src/sde-compile.js b/src/sde-compile.js
@@ -14,7 +14,7 @@ export let handler = argv => {
   compile(argv.model, argv)
 }
 export let compile = (model, opts) => {
-  let { modelDirname, modelName, modelPathname } = modelPathProps(model)
+  let { modelDirname, modelName } = modelPathProps(model)
   // Ensure the build directory exists.
   let buildDirname = buildDir(opts.builddir, modelDirname)
   // Link SDEverywhere C source files into the build directory.

diff --git a/src/sde-exec.js b/src/sde-exec.js
@@ -20,7 +20,7 @@ export let handler = argv => {
   exec(argv.model, argv)
 }
 export let exec = (model, opts) => {
-  let { modelDirname, modelName, modelPathname } = modelPathProps(model)
+  let { modelDirname, modelName } = modelPathProps(model)
   // Ensure the build and output directories exist.
   let buildDirname = buildDir(opts.builddir, modelDirname)
   let outputDirname = outputDir(opts.outfile, modelDirname)

diff --git a/src/sde-log.js b/src/sde-log.js
@@ -57,6 +57,11 @@ let exportDat = async (logPathname, datPathname) => {
     let stream = fs.createWriteStream(datPathname, 'utf8')
     let iVarKey = 0
     let finished = () => {}
+    stream.on('finish', () => {
+      // Resolve the promise after `stream.end()` has been called and all data has been flushed
+      // to the output file
+      finished()
+    })
     let writeContinuation = () => {
       while (iVarKey < varKeys.length) {
         let varKey = varKeys[iVarKey++]
@@ -74,8 +79,9 @@ let exportDat = async (logPathname, datPathname) => {
           }
         }
       }
+      // End the stream after the last line is emitted; this will trigger a `finish` event after
+      // all data has been flushed to the output file
       stream.end()
-      finished()
     }
     return new Promise(resolve => {
       // Start the write on the next event loop tick so we can return immediately.

diff --git a/src/sde-test.js b/src/sde-test.js
@@ -32,26 +32,32 @@ export let handler = argv => {
   test(argv.model, argv)
 }
 export let test = async (model, opts) => {
-  // Run the model and save output to an SDE log file.
-  let { modelDirname, modelName } = modelPathProps(model)
-  let logPathname
-  if (opts.outfile) {
-    logPathname = opts.outfile
-  } else {
-    let outputDirname = outputDir(opts.outfile, modelDirname)
-    logPathname = path.join(outputDirname, `${modelName}.txt`)
-    opts.outfile = logPathname
+  try {
+    // Run the model and save output to an SDE log file.
+    let { modelDirname, modelName } = modelPathProps(model)
+    let logPathname
+    if (opts.outfile) {
+      logPathname = opts.outfile
+    } else {
+      let outputDirname = outputDir(opts.outfile, modelDirname)
+      logPathname = path.join(outputDirname, `${modelName}.txt`)
+      opts.outfile = logPathname
+    }
+    await run(model, opts)
+    // Convert the TSV log file to a DAT file in the same directory.
+    opts.dat = true
+    await log(logPathname, opts)
+    // Assume there is a Vensim-created DAT file named {modelName}.dat in the model directory.
+    // Compare it to the SDE DAT file.
+    let vensimPathname = path.join(modelDirname, `${modelName}.dat`)
+    let p = path.parse(logPathname)
+    let sdePathname = path.format({ dir: p.dir, name: p.name, ext: '.dat' })
+    await compare(vensimPathname, sdePathname, opts)
+  } catch (e) {
+    // Exit with a non-zero error code if any step failed
+    console.error(`ERROR: ${e.message}\n`)
+    process.exit(1)
   }
-  await run(model, opts)
-  // Convert the TSV log file to a DAT file in the same directory.
-  opts.dat = true
-  await log(logPathname, opts)
-  // Assume there is a Vensim-created DAT file named {modelName}.dat in the model directory.
-  // Compare it to the SDE DAT file.
-  let vensimPathname = path.join(modelDirname, `${modelName}.dat`)
-  let p = path.parse(logPathname)
-  let sdePathname = path.format({ dir: p.dir, name: p.name, ext: '.dat' })
-  await compare(vensimPathname, sdePathname, opts)
 }
 
 export default {