fix: use case-insensitive sort and remove trailing whitespace in preprocessor

models/preprocess/expected.mdl

@@ -0,0 +1,65 @@
+{UTF-8}
+
+Apple 1  = 1
+	~~|
+
+Apple 2  = 1
+	~~|
+
+apple 3  = 1
+	~~|
+
+Banana 1  = 1
+	~~|
+
+Banana 2  = 1
+	~~|
+
+banana 3  =
+  Banana 1
+    * Banana 2
+	~~|
+
+carrot 1  = 1
+	~~|
+
+carrot 2  = 1
+	~~|
+
+carrot 3 Data 1
+	~~|
+
+Carrot 3 Data 2
+	~~|
+
+DimA: A1, A2, A3
+	~~|
+
+dimB: B1, B2, B3
+	~~|
+
+FINAL TIME  = 10
+	~~|
+
+INITIAL TIME  = 0
+	~~|
+
+Look1((0,0),(1,1),(2,2))
+	~~|
+
+Look2((0,0),(1,1),(2,2))
+	~~|
+
+"Quotes 1"  = 1
+	~~|
+
+" quotes 2 with extra whitespace "  = 1
+	~~|
+
+SAVEPER  =
+        TIME STEP
+	~~|
+
+TIME STEP  = 1
+	~~|
+

models/preprocess/input.mdl

@@ -0,0 +1,104 @@
+{UTF-8}
+
+" quotes 2 with extra whitespace "  = 1
+	~ dmnl
+	~ Comment
+	|
+
+"Quotes 1"  = 1
+	~ dmnl
+	~ Comment
+	|
+
+apple 3  = 1
+	~ dmnl
+	~ Comment
+	|
+
+Carrot 3 Data 2 :RAW:
+	~	Tons
+	~		|
+
+banana 3  =   	
+  Banana 1
+    * Banana 2
+	~ dmnl
+	~ Comment
+	|
+
+carrot 3 Data 1 :RAW:
+	~	Tons
+	~		|
+
+carrot 2  = 1  
+	~ dmnl  
+	~ Comment
+	|
+
+Look2((0,0),(1,1),(2,2))
+	~~|
+
+Look1((0,0),(1,1),(2,2))
+	~~|
+
+Apple 2  = 1   
+	~ dmnl
+	~ Comment
+	|
+
+carrot 1  = 1
+	~ dmnl
+	~ Comment
+	|
+
+Banana 2  = 1
+	~ dmnl
+	~ Comment
+	|
+
+Banana 1  = 1
+	~ dmnl
+	~ Comment
+	|
+
+Apple 1  = 1
+	~ dmnl
+	~ Comment
+	|
+
+dimB: B1, B2, B3
+  ~~|
+
+DimA: A1, A2, A3
+  ~~|
+
+********************************************************
+	.Control
+********************************************************~
+		Simulation Control Parameters
+	|
+
+FINAL TIME  = 10
+	~	Month
+	~	The final time for the simulation.
+	|
+
+INITIAL TIME  = 0
+	~	Month
+	~	The initial time for the simulation.
+	|
+
+SAVEPER  =
+        TIME STEP
+	~	Month [0,?]
+	~	The frequency with which output is stored.
+	|
+
+TIME STEP  = 1
+	~	Month [0,?]
+	~	The time step for the simulation.
+	|
+
+\\\---/// Sketch information - do not modify anything except names
+V300  Do not put anything below this section - it will be ignored
+*View 1

models/preprocess/preprocess_check.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+MODEL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SDE='node ../../src/sde.js'
+
+cd $MODEL_DIR
+$SDE generate --preprocess input.mdl
+
+diff expected.mdl build/input.mdl > build/diff.txt 2>&1
+if [ $? != 0 ]; then
+  echo
+  echo "ERROR: 'sde generate --preprocess' produced unexpected results:"
+  echo
+  cat build/diff.txt
+  echo
+  exit 1
+fi
+
+echo "All validation checks passed!"

src/Preprocessor.js

@@ -126,59 +126,76 @@ let preprocessModel = (mdlFilename, spec, profile = 'genc', writeFiles = false)
   // Split into separate equations
   eqns = splitEquations(mdl)
 
-  // Extract the LHS variable name (minus any double quotes) for each equation,
-  // which we will use to sort the equations alphabetically
-  const unsortedEqns = []
+  // Extract the LHS variable name for each equation, which we will use to sort
+  // the equations alphabetically
+  const unsorted = []
   for (let eqn of eqns) {
     // Ignore the encoding
     eqn = eqn.replace('{UTF-8}', '')
-    // Remove ":RAW:" flag
-    eqn = eqn.replace(/:RAW:/, '')
+    // Remove ":RAW:" flag; it is not needed by SDE and causes problems if left in
+    eqn = eqn.replace(/:RAW:/g, '')
     // Remove whitespace
     eqn = eqn.trim()
     if (eqn.length > 0) {
-      // Split on newlines so that we look only at the first line of each equation
-      let key = eqn.split(/\n/)[0]
+      // Split on newlines so that we look only at the first line of each declaration
+      let line = eqn.split(/\n/)[0].trim()
+      // If the line contains an '=', treat this as an equation, otherwise it is a
+      // basic declaration
+      let kind
+      let key = line
+      if (key.includes('=')) {
+        kind = 'eqn'
+        key = key.split('=')[0].trim()
+      } else {
+        kind = 'decl'
+      }
       // Ignore double quotes
-      key = key.replace(/\"/, '')
+      key = key.replace(/\"/g, '')
       // Ignore any whitespace that remains
       key = key.trim()
       // Ignore case
-      key.toLowerCase()
-      unsortedEqns.push({
+      key = key.toLowerCase()
+      unsorted.push({
         key,
+        kind,
         eqn
       })
     }
   }
 
   // Sort the equations alphabetically by LHS variable name
-  const sortedEqns = unsortedEqns.sort((a, b) => {
+  const sorted = unsorted.sort((a, b) => {
     return (a.key < b.key) ? -1 : (a.key > b.key) ? 1 : 0;
   })
 
   // Emit formula lines without comment contents.
-  for (const elem of sortedEqns) {
+  for (const elem of sorted) {
     const eqn = elem.eqn
     let iComment = eqn.indexOf('~')
     if (iComment >= 0) {
       let formula = B.lines(eqn.substr(0, iComment))
       for (let i = 0; i < formula.length; i++) {
+        let line = formula[i]
+        // Remove trailing whitespace
+        line = line.replace(/\s+$/, '')
         if (i === 0) {
-          if (formula[i] !== ENCODING) {
-            emitPP(formula[i])
+          if (line !== ENCODING) {
+            emitPP(line)
           }
         } else {
           if (opts.joinFormulaLines) {
-            emitPP(formula[i].replace(/^\t+/, ''))
+            // Remove any leading tabs
+            emitPP(line.replace(/^\t+/, ''))
           } else {
-            B.emitLine('', 'pp')
-            emitPP(formula[i])
+            // Only emit the line if it has non-whitespace characters
+            if (line.length > 0) {
+              emitPP(`\n${line}`)
+            }
           }
         }
       }
       if (opts.emitCommentMarkers) {
-        B.emitLine('~~|\n', 'pp')
+        B.emitLine('\n\t~~|\n', 'pp')
       } else {
         B.emitLine('', 'pp')
       }

src/tests/modeltests

@@ -19,14 +19,16 @@ function test {
   # Clean up before
   $SDE clean --modeldir $MODEL_DIR
 
-  # Test
-  SPEC_FILE=$MODEL_DIR/${MODEL}_spec.json
-  if [[ -f $SPEC_FILE ]]; then
-    TEST_ARGS="--spec $SPEC_FILE"
-  else
-    TEST_ARGS=
+  # Test (only if there is a dat file to compare against)
+  if [[ -f $MODEL_DIR/${MODEL}.dat ]]; then
+    SPEC_FILE=$MODEL_DIR/${MODEL}_spec.json
+    if [[ -f $SPEC_FILE ]]; then
+      TEST_ARGS="--spec $SPEC_FILE"
+    else
+      TEST_ARGS=
+    fi
+    $SDE test $TEST_ARGS -p 1e-4 $MODEL_DIR/$MODEL
   fi
-  $SDE test $TEST_ARGS -p 1e-4 $MODEL_DIR/$MODEL
 
   # Run additional script to validate output
   VALIDATE_SCRIPT=$MODEL_DIR/${MODEL}_check.sh