fix: update default plugin-check configuration to include comparison …

…yaml files (#574)

Fixes #571

examples/hello-world/README.md

@@ -24,9 +24,11 @@ cd ./hello-world
 npm install
 
 # Enter development mode for the sample model.  This will start a live
-# development environment that will build a WebAssembly version of the
-# sample model and run checks on it any time you make changes to the
-# Vensim model file (sample.mdl) or the checks file (sample.check.yaml).
+# development environment that will build a JavaScript version of the
+# sample model and run checks on it any time you make changes to:
+#   - the Vensim model file (sample.mdl)
+#   - the model check definitions (model/checks/*.yaml)
+#   - the model comparison definitions (model/comparisons/*.yaml)
 npm run dev
 ```
 

examples/hello-world/model/checks/checks.yaml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=../../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/check.schema.json
+
+#
+# This is a simple example of "check" tests that exercise the model under different
+# input scenarios.  For more guidance, consult the following wiki page:
+#   https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model
+#
+
+- describe: Total inventory
+  tests:
+    - it: should be constant for years <= 2020 for all input scenarios
+      scenarios:
+        - preset: matrix
+      datasets:
+        - name: Total inventory
+      predicates:
+        - eq: 1000
+          time:
+            before_incl: 2020
+    - it: should be in the range [1000,1300] for all input scenarios
+      scenarios:
+        - preset: matrix
+      datasets:
+        - name: Total inventory
+      predicates:
+        - gte: 1000
+          lte: 1300

examples/hello-world/model/comparisons/comparisons.yaml

@@ -0,0 +1,41 @@
+# yaml-language-server: $schema=../../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/comparison.schema.json
+
+#
+# This is a simple example of defining custom comparison scenarios, which allow you to see
+# how the behavior of the model compares to that of previous versions.  For more guidance,
+# consult the following wiki page:
+#   https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model
+#
+
+- scenario:
+    title: Custom production scenario
+    subtitle: early/gradual ramp-up
+    with:
+      - input: Production start year
+        at: 2020
+      - input: Production years
+        at: 25
+      - input: Production slope
+        at: 2
+
+- scenario:
+    title: Custom production scenario
+    subtitle: delayed/faster ramp-up
+    with:
+      - input: Production start year
+        at: 2040
+      - input: Production years
+        at: 30
+      - input: Production slope
+        at: 5
+
+- scenario:
+    title: Custom production scenario
+    subtitle: late/fast ramp-up
+    with:
+      - input: Production start year
+        at: 2070
+      - input: Production years
+        at: 10
+      - input: Production slope
+        at: 10

examples/hello-world/model/sample.mdl

@@ -1,13 +1,24 @@
 {UTF-8}
 
-X = TIME ~~|
+Production slope = 1
+  ~ [0,10,1]
+  ~
+  |
 
-Y = 0
-  ~ [-10,10,0.1]
+Production start year = 2020
+  ~ [2020,2070,1]
   ~
   |
 
-Z = X + Y
+Production years = 30
+  ~ [0,30,1]
+  ~
+  |
+
+Initial inventory = 1000
+  ~~|
+
+Total inventory = Initial inventory + RAMP(Production slope, Production start year, Production start year + Production years)
   ~~|
 
 INITIAL TIME = 2000 ~~|

examples/hello-world/sde.config.js

@@ -7,8 +7,12 @@ export async function config() {
 
     modelSpec: async () => {
       return {
-        inputs: [{ varName: 'Y', defaultValue: 0, minValue: -10, maxValue: 10 }],
-        outputs: [{ varName: 'Z' }]
+        inputs: [
+          { varName: 'Production slope', defaultValue: 1, minValue: 1, maxValue: 10 },
+          { varName: 'Production start year', defaultValue: 2020, minValue: 2020, maxValue: 2070 },
+          { varName: 'Production years', defaultValue: 10, minValue: 0, maxValue: 30 }
+        ],
+        outputs: [{ varName: 'Total inventory' }]
       }
     },
 

examples/sir/README.md

@@ -29,11 +29,12 @@ cd ./sir
 npm create @sdeverywhere@latest
 
 # Enter development mode for the sample model.  This will start a live
-# development environment that will build a WebAssembly version of the
+# development environment that will build a JavaScript version of the
 # sample model and run checks on it any time you make changes to:
 #   - the config files
-#   - the Vensim model file (sir.mdl)
-#   - the checks file (sir.check.yaml)
+#   - the Vensim model file (model/sir.mdl)
+#   - the model check definitions (model/checks/*.yaml)
+#   - the model comparison definitions (model/comparisons/*.yaml)
 npm run dev
 ```
 

examples/sir/model/checks/checks.yaml

@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=../../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/check.schema.json
+
+#
+# This is a simple example of "check" tests that exercise the model under different
+# input scenarios.  For more guidance, consult the following wiki page:
+#   https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model
+#
+
+- describe: Population Variables
+  tests:
+    - it: should be between 0 and 10000 for all input scenarios
+      scenarios:
+        - preset: matrix
+      datasets:
+        - name: Infectious Population I
+        - name: Recovered Population R
+        - name: Susceptible Population S
+      predicates:
+        - gte: 0
+          lte: 10000

examples/sir/model/comparisons/comparisons.yaml

@@ -0,0 +1,26 @@
+# yaml-language-server: $schema=../../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/comparison.schema.json
+
+#
+# This is a simple example of defining custom comparison scenarios, which allow you to see
+# how the behavior of the model compares to that of previous versions.  For more guidance,
+# consult the following wiki page:
+#   https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model
+#
+
+- scenario:
+    title: Custom scenario
+    subtitle: with avg duration=4 and contact rate=2
+    with:
+      - input: Average Duration of Illness d
+        at: 4
+      - input: Initial contact rate
+        at: 2
+
+- scenario:
+    title: Custom scenario
+    subtitle: with avg duration=4 and contact rate=4
+    with:
+      - input: Average Duration of Illness d
+        at: 4
+      - input: Initial contact rate
+        at: 4

examples/template-default/README.md

@@ -26,11 +26,12 @@ The project includes:
 npm create @sdeverywhere@latest
 
 # Enter development mode for your model.  This will start a live
-# development environment that will build a WebAssembly version of the
+# development environment that will build a JavaScript version of the
 # model and run checks on it any time you make changes to:
 #   - the config files
 #   - the Vensim model file (<name>.mdl)
-#   - the checks file (<name>.check.yaml)
+#   - the model check definitions (model/checks/*.yaml)
+#   - the model comparison definitions (model/comparisons/*.yaml)
 npm run dev
 ```
 

examples/template-minimal/README.md

@@ -22,10 +22,11 @@ to add the `@sdeverywhere/plugin-config` package to your project.)
 npm create @sdeverywhere@latest
 
 # Enter development mode for your model.  This will start a live
-# development environment that will build a WebAssembly version of the
+# development environment that will build a JavaScript version of the
 # model and run checks on it any time you make changes to:
 #   - the Vensim model file (<name>.mdl)
-#   - the checks file (<name>.check.yaml)
+#   - the model check definitions (model/checks/*.yaml)
+#   - the model comparison definitions (model/comparisons/*.yaml)
 npm run dev
 ```
 

packages/create/src/index.ts

@@ -10,7 +10,7 @@ import detectPackageManager from 'which-pm-runs'
 import yargs from 'yargs-parser'
 
 import { chooseCodeFormat } from './step-code-format'
-import { chooseGenConfig, generateCheckYaml, updateSdeConfig } from './step-config'
+import { chooseGenConfig, generateSampleYamlFiles, updateSdeConfig } from './step-config'
 import { chooseInstallDeps } from './step-deps'
 import { chooseProjectDir } from './step-directory'
 import { chooseInstallEmsdk } from './step-emsdk'
@@ -54,10 +54,10 @@ export async function main(): Promise<void> {
   const genFormat = await chooseCodeFormat()
 
   // Update the `sde.config.js` file to use the chosen mdl file and
-  // generate a sample `.check.yaml` file
+  // generate sample `checks.yaml` and `comparisons.yaml` files
   if (!args.dryRun) {
     await updateSdeConfig(projDir, mdlPath, genFormat)
-    await generateCheckYaml(projDir, mdlPath)
+    await generateSampleYamlFiles(projDir)
   }
   console.log()
 

packages/create/src/step-config.ts

@@ -30,12 +30,17 @@ interface MdlLevelVariable {
 
 type MdlVariable = MdlConstVariable | MdlAuxVariable | MdlLevelVariable
 
-const sampleCheckContent = `\
+const sampleChecksContent = `\
 # yaml-language-server: $schema=SCHEMA_PATH
 
-# NOTE: This is just a simple check to get you started.  Replace "Some output" with
-# the name of some variable you'd like to test.  Additional tests can be developed
-# in the "playground" (beta) inside the model-check report.
+#
+# This file contains "check" tests that exercise your model under different input
+# scenarios.  For more guidance, consult this wiki page:
+#   https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model
+#
+
+# NOTE: The following is an example of a simple check just to get you started.
+# Replace "Some output" with the name of some variable you'd like to test.
 - describe: Some output
   tests:
     - it: should be > 0 for all input scenarios
@@ -47,6 +52,29 @@ const sampleCheckContent = `\
         - gt: 0
 `
 
+const sampleComparisonsContent = `\
+# yaml-language-server: $schema=SCHEMA_PATH
+
+#
+# This file contains definitions of custom comparison scenarios, which allow you to see
+# how the behavior of the model compares to that of previous versions.  For more guidance,
+# consult the following wiki page:
+#   https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model
+#
+
+# NOTE: The following is an example of a custom scenario just to get you started.
+# Replace "Some input" and "Another input" with the names of some variables you'd
+# like to test.
+- scenario:
+    title: Custom scenario
+    subtitle: gradual ramp-up
+    with:
+      - input: Some input
+        at: 10
+      - input: Another input
+        at: 20
+`
+
 export async function updateSdeConfig(projDir: string, mdlPath: string, genFormat: string): Promise<void> {
   // Read the `sde.config.js` file from the template
   const configPath = joinPath(projDir, 'sde.config.js')
@@ -62,18 +90,19 @@ export async function updateSdeConfig(projDir: string, mdlPath: string, genForma
   await writeFile(configPath, configText)
 }
 
-export async function generateCheckYaml(projDir: string, mdlPath: string): Promise<void> {
-  // Generate a sample `{mdl}.check.yaml` file if one doesn't already exist
-  // TODO: Make this optional (ask user first)?
-  const checkYamlFile = mdlPath.replace('.mdl', '.check.yaml')
-  const checkYamlPath = joinPath(projDir, checkYamlFile)
-  if (!existsSync(checkYamlPath)) {
+async function generateYaml(projDir: string, kind: 'checks' | 'comparisons', template: string): Promise<void> {
+  const yamlDir = joinPath(projDir, 'model', kind)
+  const yamlPath = joinPath(yamlDir, `${kind}.yaml`)
+  if (!existsSync(yamlPath)) {
     // Get relative path from yaml file parent dir to project dir
-    let relProjPath = relative(dirname(checkYamlPath), projDir)
+    let relProjPath = relative(dirname(yamlPath), projDir)
     if (relProjPath.length === 0) {
       relProjPath = './'
     }
 
+    // Create the directory for the yaml file, if needed
+    await mkdir(yamlDir, { recursive: true })
+
     // TODO: This path is normally different depending on whether using npm/yarn or
     // pnpm. For npm/yarn, `check-core` is hoisted under top-level `node_modules`,
     // but for pnpm, it is nested under `node_modules/.pnpm`.  As an ugly workaround
@@ -82,13 +111,22 @@ export async function generateCheckYaml(projDir: string, mdlPath: string): Promi
     // suffice).  This allows us to use the same path here that works for all
     // three package managers.
     const nodeModulesPart = joinPath(relProjPath, 'node_modules')
-    const checkCorePart = '@sdeverywhere/check-core/schema/check.schema.json'
+    const schemaName = kind === 'checks' ? 'check' : 'comparison'
+    const checkCorePart = `@sdeverywhere/check-core/schema/${schemaName}.schema.json`
     const schemaPath = `${nodeModulesPart}/${checkCorePart}`
-    const checkContent = sampleCheckContent.replace('SCHEMA_PATH', schemaPath)
-    await writeFile(checkYamlPath, checkContent)
+    const yamlContent = template.replace('SCHEMA_PATH', schemaPath)
+    await writeFile(yamlPath, yamlContent)
   }
 }
 
+export async function generateSampleYamlFiles(projDir: string): Promise<void> {
+  // Generate a sample `checks.yaml` file if one doesn't already exist
+  await generateYaml(projDir, 'checks', sampleChecksContent)
+
+  // Generate a sample `comparisons.yaml` file if one doesn't already exist
+  await generateYaml(projDir, 'comparisons', sampleComparisonsContent)
+}
+
 export async function chooseGenConfig(projDir: string, mdlPath: string): Promise<void> {
   // TODO: For now we eagerly read the mdl file; maybe change this to only load it if
   // the user chooses to generate graph and/or slider config

packages/plugin-check/src/options.ts

@@ -15,14 +15,6 @@ export interface CheckPluginOptions {
   /** The current bundle, i.e., the bundle that is being developed and checked. */
   current?: CheckBundle
 
-  // TODO
-  // /**
-  //  * The glob-style path to the check yaml files to be included.  If undefined,
-  //  * a default pattern will be used that finds all `*.check.yaml` files under
-  //  * the configured `rootDir`.
-  //  */
-  // yamlPath?: string
-
   /**
    * The absolute path to the JS file containing the test configuration.  If undefined,
    * a default test configuration will be used.