fix: remove legacy preprocessor implementation and use newer one from parse package

README.md

@@ -159,7 +159,7 @@ Here are some prominent current limitations:
 - You must rewrite tabbed arrays as separate, non-apply-to-all variables.
 - You must rewrite equations that use macros or code them in C.
 
-Tabbed arrays and macros are removed from the model during preprocessing and written to the `removals.txt` file for your reference.
+Note that tabbed arrays and macros are automatically removed from the model by the transpiler prior to parsing.
 
 ## SDEverywhere in Production
 

models/comments/expected.mdl

@@ -1,36 +1,21 @@
 {UTF-8}
 
-a[DimA]=
-  0,
-  1,
-  2
-	~~|
+DimA: A1, A2, A3 ~~|
 
-b = 3
-	~~|
+a[DimA]= 0, 1, 2 ~~|
 
-c = 4
-	~~|
+b = 3 ~~|
 
-d=
-  8760
-	~~|
+c = 4 ~~|
 
-DimA: A1, A2, A3
-	~~|
+d= 8760 ~~|
 
-e = 41  + 1
-	~~|
+e = 41 + 1 ~~|
 
-FINAL TIME  = 1
-	~~|
+INITIAL TIME = 0 ~~|
 
-INITIAL TIME  = 0
-	~~|
+FINAL TIME = 1 ~~|
 
-SAVEPER  = 1
-	~~|
-
-TIME STEP  = 1
-	~~|
+SAVEPER = 1 ~~|
 
+TIME STEP = 1 ~~|

models/flatten/expected.mdl

@@ -1,84 +1,51 @@
 {UTF-8}
 
-Apple 1  = 1
-	~~|
+Apple 1 = 1 ~~|
 
-Apple 2  = 1
-	~~|
+Apple 2 = 1 ~~|
 
-apple 3  = 1
-	~~|
+apple 3 = 1 ~~|
 
-Banana 1  = 1
-	~~|
+Banana 1 = 1 ~~|
 
-Banana 2  = 1
-	~~|
+Banana 2 = 1 ~~|
 
-banana 3  =
-  Banana 1
-    * Banana 2
-	~~|
+banana 3 = Banana 1 * Banana 2 ~~|
 
-carrot 1  = 1
-	~~|
+carrot 1 = 1 ~~|
 
-carrot 2  = 1
-	~~|
+carrot 2 = 1 ~~|
 
-DimA: A1, A2, A3
-	~~|
+DimA: A1, A2, A3 ~~|
 
-DimAExceptA1: A2, A3
-	~~|
+DimAExceptA1: A2, A3 ~~|
 
-dimB: B1, B2, B3
-	~~|
+dimB: B1, B2, B3 ~~|
 
-Equation with backslashes[A1] = 2
-	~~|
+Equation with backslashes[A1] = 2 ~~|
 
-Equation with backslashes[DimAExceptA1] = 1
-	~~|
+Equation with backslashes[DimAExceptA1] = 1 ~~|
 
-Equation without backslashes[A1
-  ] =
-  2
-	~~|
+Equation without backslashes[A1 ] = 2 ~~|
 
-Equation without backslashes[DimAExceptA1
-  ] =
-  1
-	~~|
+Equation without backslashes[DimAExceptA1 ] = 1 ~~|
 
-FINAL TIME  = 10
-	~~|
+FINAL TIME = 10 ~~|
 
-INITIAL TIME  = 0
-	~~|
+INITIAL TIME = 0 ~~|
 
-Look1((0,0),(1,1),(2,2))
-	~~|
+Look1((0,0),(1,1),(2,2)) ~~|
 
-Look2((0,0),(1,1),(2,2))
-	~~|
+Look2((0,0),(1,1),(2,2)) ~~|
 
-"Quotes 1"  = 1
-	~~|
+"Quotes 1" = 1 ~~|
 
-" quotes 2 with extra whitespace "  = 1
-	~~|
+" quotes 2 with extra whitespace " = 1 ~~|
 
-SAVEPER  =
-        TIME STEP
-	~~|
+SAVEPER = TIME STEP ~~|
 
-TIME STEP  = 1
-	~~|
+TIME STEP = 1 ~~|
 
-Using equation with backslashes[DimA] = Equation with backslashes[DimA] * 2
-	~~|
-
-Using equation without backslashes[DimA] = Equation without backslashes[DimA] * 2
-	~~|
+Using equation with backslashes[DimA] = Equation with backslashes[DimA] * 2 ~~|
 
+Using equation without backslashes[DimA] = Equation without backslashes[DimA] * 2 ~~|

models/preprocess/expected.mdl

@@ -1,65 +1,43 @@
 {UTF-8}
 
-Apple 1  = 1
-	~~|
+" quotes 2 with extra whitespace " = 1 ~~|
 
-Apple 2  = 1
-	~~|
+"Quotes 1" = 1 ~~|
 
-apple 3  = 1
-	~~|
+"quotes 3 with | pipe character " = 2 ~~|
 
-Banana 1  = 1
-	~~|
+apple 3 = 1 ~~|
 
-Banana 2  = 1
-	~~|
+Carrot 3 Data 2 ~~|
 
-banana 3  =
-  Banana 1
-    * Banana 2
-	~~|
+banana 3 = Banana 1 * Banana 2 ~~|
 
-carrot 1  = 1
-	~~|
+carrot 3 Data 1 ~~|
 
-carrot 2  = 1
-	~~|
+carrot 2 = 1 ~~|
 
-carrot 3 Data 1
-	~~|
+Look2((0,0),(1,1),(2,2)) ~~|
 
-Carrot 3 Data 2
-	~~|
+Look1((0,0),(1,1),(2,2)) ~~|
 
-DimA: A1, A2, A3
-	~~|
+Apple 2 = 1 ~~|
 
-dimB: B1, B2, B3
-	~~|
+carrot 1 = 1 ~~|
 
-FINAL TIME  = 10
-	~~|
+Banana 2 = 1 ~~|
 
-INITIAL TIME  = 0
-	~~|
+Banana 1 = 1 ~~|
 
-Look1((0,0),(1,1),(2,2))
-	~~|
+Apple 1 = 1 ~~|
 
-Look2((0,0),(1,1),(2,2))
-	~~|
+dimB: B1, B2 ~~|
 
-"Quotes 1"  = 1
-	~~|
+DimA: A1, A2 ~~|
 
-" quotes 2 with extra whitespace "  = 1
-	~~|
+FINAL TIME = 10 ~~|
 
-SAVEPER  =
-        TIME STEP
-	~~|
+INITIAL TIME = 0 ~~|
 
-TIME STEP  = 1
-	~~|
+SAVEPER = TIME STEP ~~|
 
+TIME STEP = 1 ~~|

models/preprocess/input.mdl

@@ -10,6 +10,11 @@
 	~ Comment
 	|
 
+"quotes 3 with | pipe character "  = 2
+	~ dmnl
+	~ Comment
+	|
+
 apple 3  = 1
 	~ dmnl
 	~ Comment
@@ -66,12 +71,25 @@ Apple 1  = 1
 	~ Comment
 	|
 
-dimB: B1, B2, B3
+:MACRO: VSMOOTH(input,SMOOTH TIME)
+
+Vsmooth = INTEG((input - Vsmooth)/SMOOTH TIME, input)
+  ~ input
+  ~ The first order smoothed value of a variable.
+  |
+
+:END OF MACRO:
+
+dimB: B1, B2
   ~~|
 
-DimA: A1, A2, A3
+DimA: A1, A2
   ~~|
 
+Population[DimA,dimB] = TABBED ARRAY(
+\t1\t2
+\t5\t6) ~Person~|
+
 ********************************************************
 	.Control
 ********************************************************~

packages/cli/README.md

@@ -97,7 +97,10 @@ sde bundle [--config sde.config.js] [--verbose]
 sde generate --list {model}
 ```
 
-#### Preprocess a model to remove macros and tabbed arrays to removals.txt
+#### Preprocess a model to remove macros and tabbed arrays
+
+_NOTE:_ For most users it is not necessary to run the preprocessor manually
+because the transpiler runs the preprocessor automatically prior to parsing.
 
 ```
 sde generate --preprocess {model}
@@ -204,21 +207,6 @@ There is an example in the `directdata` sample model.
 }
 ```
 
-#### Specify equations to remove from the model
-
-When SDEverywhere cannot handle a certain Vensim construct yet, you will need to remove equations that use the construct from the model, convert it by hand into something that SDEverywhere can handle, and then insert it back into the model.
-To have the preprocessor remove equations from the model into a `removals.txt` file, specify substrings to match in the equation in the `removalKeys` section.
-Macros and `TABBED ARRAY` equations are already automatically removed by the preprocessor.
-
-For instance, you could key on the variable name in the equation definition.
-
-```json
-"removalKeys": [
-  "varname1 =",
-  "varname2 ="
-]
-```
-
 #### Generating, compiling, running, and testing the C code
 
 To generate C code using the `--spec` argument, enter the following command:
@@ -260,15 +248,6 @@ That is, the values are separated by spaces, and each pair has an index number,
 The zero-based index maps into a static array of input variable pointers held in the function.
 These are used to set the value directly into the static `double` variable in the generated code.
 
-#### Inserting a file into the model
-
-Some constructs like macros are not supported by SDEverywhere.
-They are removed from the model by the preprocessor into the `removals.txt` file.
-You can edit these constructs into a form that SDEverywhere supports and insert them back into the model.
-Create a file called `mdl-edits.txt` in the model directory with the constructs to insert.
-For instance, manually expand macros and place them into the `mdl-edits.txt` file.
-The preprocessor will read this file and insert its contents unchanged into the beginning of the model.
-
 ## License
 
 SDEverywhere is distributed under the MIT license. See `LICENSE` for more details.

packages/cli/src/sde-causes.js

@@ -1,4 +1,6 @@
-import { generateCode, parseModel, preprocessModel } from '@sdeverywhere/compile'
+import { readFileSync } from 'fs'
+
+import { parseAndGenerate } from '@sdeverywhere/compile'
 
 import { modelPathProps, parseSpec } from './utils.js'
 
@@ -14,22 +16,13 @@ let builder = {
 let handler = argv => {
   causes(argv.model, argv.c_varname, argv)
 }
-let causes = (model, varname, opts) => {
+let causes = async (model, varname, opts) => {
   // Get the model name and directory from the model argument.
-  let { modelDirname, modelPathname } = modelPathProps(model)
-  let extData = new Map()
-  let directData = new Map()
+  let { modelDirname, modelPathname, modelName } = modelPathProps(model)
   let spec = parseSpec(opts.spec)
-  // Preprocess model text into parser input.
-  // TODO: The legacy `parseModel` function previously required the `preprocessModel`
-  // step to be performed first, but the new `parseModel` runs the preprocessor
-  // implicitly, so we can remove this step (and can simplify this code to use
-  // `parseAndGenerate` instead)
-  let input = preprocessModel(modelPathname, spec)
   // Parse the model to get variable and subscript information.
-  let parsedModel = parseModel(input, modelDirname)
-  let operations = ['printRefGraph']
-  generateCode(parsedModel, { spec, operations, extData, directData, modelDirname, varname })
+  let input = readFileSync(modelPathname, 'utf8')
+  await parseAndGenerate(input, spec, ['printRefGraph'], modelDirname, modelName, '', varname)
 }
 export default {
   command,