From 734708e1a7791dc17a1382d661967b014e5c4ce3 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 2 Dec 2024 13:15:32 -0800 Subject: [PATCH 1/8] fix: update plugin-check to search for check and comparison yaml files --- .../plugin-check/src/vite-config-for-tests.ts | 27 ++++++++----- .../plugin-check/template-tests/src/index.ts | 40 +++++++++++++------ 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/packages/plugin-check/src/vite-config-for-tests.ts b/packages/plugin-check/src/vite-config-for-tests.ts index 9d802b64..76af8660 100644 --- a/packages/plugin-check/src/vite-config-for-tests.ts +++ b/packages/plugin-check/src/vite-config-for-tests.ts @@ -13,20 +13,25 @@ export function createViteConfigForTests(projDir: string, prepDir: string, mode: // Use `template-tests` as the root directory for the tests project const root = resolvePath(__dirname, '..', 'template-tests') - // Include `*.check.yaml` files under the configured project root directory. This - // glob path apparently must be a relative path (relative to the `template-tests/src` - // directory where the glob is used). + // Get the base glob path; apparently this must be a relative path (relative to + // the `template-tests/src` directory where the glob is used) const templateSrcDir = resolvePath(root, 'src') const relProjDir = relative(templateSrcDir, projDir) // XXX: The glob pattern must use forward slashes only, so on Windows we need to // convert backslashes to slashes const relProjDirPath = relProjDir.replaceAll('\\', '/') - // TODO: Use yamlPath from options - const yamlPath = `${relProjDirPath}/**/*.check.yaml` - // // Read the `package.json` for the template project - // const pkgPath = resolvePath(root, 'package.json') - // const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) + // Include check test definitions in files matching `checks/*.yaml` under + // the configured project root directory. We also include `*.check.yaml`, + // which was the naming used in earlier versions of the create package and + // related examples. + // TODO: Use yaml path/pattern from options + const yamlCheckGlobPatterns = `['${relProjDirPath}/**/checks/*.yaml', '${relProjDirPath}/**/*.check.yaml']` + + // Include comparison test definitions in files matching `comparisons/*.yaml` + // under the configured project root directory + // TODO: Use yaml path/pattern from options + const yamlComparisonGlobPatterns = `['${relProjDirPath}/**/comparisons/*.yaml']` // Calculate output directory relative to the template root // TODO: For now we write it to `prepDir`; make this configurable? @@ -54,8 +59,10 @@ export function createViteConfigForTests(projDir: string, prepDir: string, mode: preventAssignment: true, delimiters: ['', ''], values: { - // Inject the glob pattern for matching check yaml files - './__YAML_PATH__': yamlPath + // Inject the glob patterns for matching model check yaml files + '"./__YAML_CHECK_GLOB_PATTERNS__"': yamlCheckGlobPatterns, + // Inject the glob patterns for matching model comparison yaml files + '"./__YAML_COMPARISON_GLOB_PATTERNS__"': yamlComparisonGlobPatterns } }) as unknown as PluginOption ], diff --git a/packages/plugin-check/template-tests/src/index.ts b/packages/plugin-check/template-tests/src/index.ts index 14b88c24..69cc0bbe 100644 --- a/packages/plugin-check/template-tests/src/index.ts +++ b/packages/plugin-check/template-tests/src/index.ts @@ -5,6 +5,7 @@ import type { ComparisonOptions, ComparisonScenarioSpec, ComparisonSpecs, + ComparisonSpecsSource, ConfigInitOptions, ConfigOptions, DatasetKey, @@ -12,21 +13,36 @@ import type { InputVar } from '@sdeverywhere/check-core' -// Load the yaml test files. The `./__YAML_PATH__` part will be replaced by Vite -// (see `vite-config-for-tests.ts`). Note that we provide a placeholder here that -// looks like a valid glob pattern, since Vite's dependency resolver will report -// errors if it is invalid (not a literal). -const yamlGlob = import.meta.glob('./__YAML_PATH__', { +// Load the yaml files containing model check test definitions. The +// `./__YAML_*_GLOB_PATTERNS__` part will be replaced by Vite (see +// `vite-config-for-tests.ts`). Note that we provide a placeholder here +// that looks like a valid glob pattern, since Vite's dependency resolver +// will report errors if it is invalid (not a literal). +const yamlCheckSpecFilesGlob = import.meta.glob('./__YAML_CHECK_GLOB_PATTERNS__', { eager: true, query: '?raw', import: 'default' }) -const tests: string[] = [] -for (const yamlKey of Object.keys(yamlGlob)) { - const yaml = yamlGlob[yamlKey] - tests.push(yaml as unknown as string) +const yamlCheckSpecFiles: string[] = [] +for (const yamlKey of Object.keys(yamlCheckSpecFilesGlob)) { + const yaml = yamlCheckSpecFilesGlob[yamlKey] + yamlCheckSpecFiles.push(yaml as unknown as string) } +// Load the yaml files containing model comparison test definitions +const yamlComparisonFilesGlob = import.meta.glob('./__YAML_COMPARISON_GLOB_PATTERNS__', { + eager: true, + query: '?raw', + import: 'default' +}) +const yamlComparisonSpecFiles: ComparisonSpecsSource[] = Object.entries(yamlComparisonFilesGlob).map(entry => { + return { + kind: 'yaml', + filename: entry[0], + content: entry[1] as string + } +}) + // If an output variable is renamed, define the mapping from the old key // to the new key here. If this step is omitted, the old variable will // appear as being removed and the new variable will appear as being added, @@ -53,8 +69,8 @@ export async function getConfigOptions( // - twice for each input // - once with single input at its minimum // - once with single input at its maximum - // TODO: Also read specs from comparison yaml files const baseComparisonSpecs = createBaseComparisonSpecs(bundleL, bundleR) + const comparisonSpecs = [baseComparisonSpecs, ...yamlComparisonSpecFiles] comparisonOptions = { baseline: { @@ -62,7 +78,7 @@ export async function getConfigOptions( bundle: bundleL }, thresholds: [1, 5, 10], - specs: [baseComparisonSpecs], + specs: comparisonSpecs, datasets: { renamedDatasetKeys } @@ -75,7 +91,7 @@ export async function getConfigOptions( bundle: bundleR }, check: { - tests + tests: yamlCheckSpecFiles }, comparison: comparisonOptions } From 66fa47683dc8aabb928e5582a8c7581b655078b5 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 2 Dec 2024 13:15:59 -0800 Subject: [PATCH 2/8] fix: include datasetKey in output variable spec --- packages/plugin-check/template-bundle/src/outputs.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/plugin-check/template-bundle/src/outputs.ts b/packages/plugin-check/template-bundle/src/outputs.ts index 280d88e0..2dd93da9 100644 --- a/packages/plugin-check/template-bundle/src/outputs.ts +++ b/packages/plugin-check/template-bundle/src/outputs.ts @@ -22,6 +22,7 @@ export function getOutputVars(outputSpecs: OutputSpec[]): Map Date: Mon, 2 Dec 2024 13:17:27 -0800 Subject: [PATCH 3/8] fix: update name and path for checks.yaml file in sir example --- examples/sir/model/checks/checks.yaml | 20 ++++++++++++++++++++ examples/sir/model/sir.check.yaml | 14 -------------- 2 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 examples/sir/model/checks/checks.yaml delete mode 100644 examples/sir/model/sir.check.yaml diff --git a/examples/sir/model/checks/checks.yaml b/examples/sir/model/checks/checks.yaml new file mode 100644 index 00000000..b9fa1594 --- /dev/null +++ b/examples/sir/model/checks/checks.yaml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=../../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/check.schema.json + +# +# This is a simple example of "check" tests that exercise the model under different +# input scenarios. For more guidance, consult the following wiki page: +# https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model +# + +- describe: Population Variables + tests: + - it: should be between 0 and 10000 for all input scenarios + scenarios: + - preset: matrix + datasets: + - name: Infectious Population I + - name: Recovered Population R + - name: Susceptible Population S + predicates: + - gte: 0 + lte: 10000 diff --git a/examples/sir/model/sir.check.yaml b/examples/sir/model/sir.check.yaml deleted file mode 100644 index 520e4f63..00000000 --- a/examples/sir/model/sir.check.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# yaml-language-server: $schema=../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/check.schema.json - -- describe: Population Variables - tests: - - it: should be between 0 and 10000 for all input scenarios - scenarios: - - preset: matrix - datasets: - - name: Infectious Population I - - name: Recovered Population R - - name: Susceptible Population S - predicates: - - gte: 0 - lte: 10000 From c191e8419d7cfb43af45e0632906c541dfb917ca Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 2 Dec 2024 13:17:43 -0800 Subject: [PATCH 4/8] fix: include custom comparison scenarios for sir example --- .../sir/model/comparisons/comparisons.yaml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 examples/sir/model/comparisons/comparisons.yaml diff --git a/examples/sir/model/comparisons/comparisons.yaml b/examples/sir/model/comparisons/comparisons.yaml new file mode 100644 index 00000000..9bd96472 --- /dev/null +++ b/examples/sir/model/comparisons/comparisons.yaml @@ -0,0 +1,26 @@ +# yaml-language-server: $schema=../../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/comparison.schema.json + +# +# This is a simple example of defining custom comparison scenarios, which allow you to see +# how the behavior of the model compares to that of previous versions. For more guidance, +# consult the following wiki page: +# https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model +# + +- scenario: + title: Custom scenario + subtitle: with avg duration=4 and contact rate=2 + with: + - input: Average Duration of Illness d + at: 4 + - input: Initial contact rate + at: 2 + +- scenario: + title: Custom scenario + subtitle: with avg duration=4 and contact rate=4 + with: + - input: Average Duration of Illness d + at: 4 + - input: Initial contact rate + at: 4 From 2992092db4287cf46d2106151cc98c56f2f51981 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 2 Dec 2024 13:19:40 -0800 Subject: [PATCH 5/8] fix: update hello-world example to use RAMP + improve custom checks and comparisons --- examples/hello-world/model/checks/checks.yaml | 27 ++++++++++++ .../model/comparisons/comparisons.yaml | 41 +++++++++++++++++++ examples/hello-world/model/sample.check.yaml | 12 ------ examples/hello-world/model/sample.mdl | 19 +++++++-- examples/hello-world/sde.config.js | 8 +++- 5 files changed, 89 insertions(+), 18 deletions(-) create mode 100644 examples/hello-world/model/checks/checks.yaml create mode 100644 examples/hello-world/model/comparisons/comparisons.yaml delete mode 100644 examples/hello-world/model/sample.check.yaml diff --git a/examples/hello-world/model/checks/checks.yaml b/examples/hello-world/model/checks/checks.yaml new file mode 100644 index 00000000..08ddeb79 --- /dev/null +++ b/examples/hello-world/model/checks/checks.yaml @@ -0,0 +1,27 @@ +# yaml-language-server: $schema=../../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/check.schema.json + +# +# This is a simple example of "check" tests that exercise the model under different +# input scenarios. For more guidance, consult the following wiki page: +# https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model +# + +- describe: Total inventory + tests: + - it: should be constant for years <= 2020 for all input scenarios + scenarios: + - preset: matrix + datasets: + - name: Total inventory + predicates: + - eq: 1000 + time: + before_incl: 2020 + - it: should be in the range [1000,1300] for all input scenarios + scenarios: + - preset: matrix + datasets: + - name: Total inventory + predicates: + - gte: 1000 + lte: 1300 diff --git a/examples/hello-world/model/comparisons/comparisons.yaml b/examples/hello-world/model/comparisons/comparisons.yaml new file mode 100644 index 00000000..67b52bb0 --- /dev/null +++ b/examples/hello-world/model/comparisons/comparisons.yaml @@ -0,0 +1,41 @@ +# yaml-language-server: $schema=../../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/comparison.schema.json + +# +# This is a simple example of defining custom comparison scenarios, which allow you to see +# how the behavior of the model compares to that of previous versions. For more guidance, +# consult the following wiki page: +# https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model +# + +- scenario: + title: Custom production scenario + subtitle: early/gradual ramp-up + with: + - input: Production start year + at: 2020 + - input: Production years + at: 25 + - input: Production slope + at: 2 + +- scenario: + title: Custom production scenario + subtitle: delayed/faster ramp-up + with: + - input: Production start year + at: 2040 + - input: Production years + at: 30 + - input: Production slope + at: 5 + +- scenario: + title: Custom production scenario + subtitle: late/fast ramp-up + with: + - input: Production start year + at: 2070 + - input: Production years + at: 10 + - input: Production slope + at: 10 diff --git a/examples/hello-world/model/sample.check.yaml b/examples/hello-world/model/sample.check.yaml deleted file mode 100644 index 73b30bad..00000000 --- a/examples/hello-world/model/sample.check.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# yaml-language-server: $schema=../node_modules/@sdeverywhere/plugin-check/node_modules/@sdeverywhere/check-core/schema/check.schema.json - -- describe: Z - tests: - - it: should be in the range [1990,2110] for all input scenarios - scenarios: - - preset: matrix - datasets: - - name: Z - predicates: - - gte: 1990 - lte: 2110 diff --git a/examples/hello-world/model/sample.mdl b/examples/hello-world/model/sample.mdl index b32e783f..918b2378 100644 --- a/examples/hello-world/model/sample.mdl +++ b/examples/hello-world/model/sample.mdl @@ -1,13 +1,24 @@ {UTF-8} -X = TIME ~~| +Production slope = 1 + ~ [0,10,1] + ~ + | -Y = 0 - ~ [-10,10,0.1] +Production start year = 2020 + ~ [2020,2070,1] ~ | -Z = X + Y +Production years = 30 + ~ [0,30,1] + ~ + | + +Initial inventory = 1000 + ~~| + +Total inventory = Initial inventory + RAMP(Production slope, Production start year, Production start year + Production years) ~~| INITIAL TIME = 2000 ~~| diff --git a/examples/hello-world/sde.config.js b/examples/hello-world/sde.config.js index bd6240e7..61ef1866 100644 --- a/examples/hello-world/sde.config.js +++ b/examples/hello-world/sde.config.js @@ -7,8 +7,12 @@ export async function config() { modelSpec: async () => { return { - inputs: [{ varName: 'Y', defaultValue: 0, minValue: -10, maxValue: 10 }], - outputs: [{ varName: 'Z' }] + inputs: [ + { varName: 'Production slope', defaultValue: 1, minValue: 1, maxValue: 10 }, + { varName: 'Production start year', defaultValue: 2020, minValue: 2020, maxValue: 2070 }, + { varName: 'Production years', defaultValue: 10, minValue: 0, maxValue: 30 } + ], + outputs: [{ varName: 'Total inventory' }] } }, From 996886d235fa2a51887d14b8e527a81356490ace Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 2 Dec 2024 13:50:27 -0800 Subject: [PATCH 6/8] fix: update create package to generate a sample comparisons.yaml file + change location of sample checks.yaml file --- packages/create/src/index.ts | 6 +-- packages/create/src/step-config.ts | 66 +++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/packages/create/src/index.ts b/packages/create/src/index.ts index f63f0081..04b9bbed 100644 --- a/packages/create/src/index.ts +++ b/packages/create/src/index.ts @@ -10,7 +10,7 @@ import detectPackageManager from 'which-pm-runs' import yargs from 'yargs-parser' import { chooseCodeFormat } from './step-code-format' -import { chooseGenConfig, generateCheckYaml, updateSdeConfig } from './step-config' +import { chooseGenConfig, generateSampleYamlFiles, updateSdeConfig } from './step-config' import { chooseInstallDeps } from './step-deps' import { chooseProjectDir } from './step-directory' import { chooseInstallEmsdk } from './step-emsdk' @@ -54,10 +54,10 @@ export async function main(): Promise { const genFormat = await chooseCodeFormat() // Update the `sde.config.js` file to use the chosen mdl file and - // generate a sample `.check.yaml` file + // generate sample `checks.yaml` and `comparisons.yaml` files if (!args.dryRun) { await updateSdeConfig(projDir, mdlPath, genFormat) - await generateCheckYaml(projDir, mdlPath) + await generateSampleYamlFiles(projDir) } console.log() diff --git a/packages/create/src/step-config.ts b/packages/create/src/step-config.ts index 2e3e562d..9f865746 100644 --- a/packages/create/src/step-config.ts +++ b/packages/create/src/step-config.ts @@ -30,12 +30,17 @@ interface MdlLevelVariable { type MdlVariable = MdlConstVariable | MdlAuxVariable | MdlLevelVariable -const sampleCheckContent = `\ +const sampleChecksContent = `\ # yaml-language-server: $schema=SCHEMA_PATH -# NOTE: This is just a simple check to get you started. Replace "Some output" with -# the name of some variable you'd like to test. Additional tests can be developed -# in the "playground" (beta) inside the model-check report. +# +# This file contains "check" tests that exercise your model under different input +# scenarios. For more guidance, consult this wiki page: +# https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model +# + +# NOTE: The following is an example of a simple check just to get you started. +# Replace "Some output" with the name of some variable you'd like to test. - describe: Some output tests: - it: should be > 0 for all input scenarios @@ -47,6 +52,29 @@ const sampleCheckContent = `\ - gt: 0 ` +const sampleComparisonsContent = `\ +# yaml-language-server: $schema=SCHEMA_PATH + +# +# This file contains definitions of custom comparison scenarios, which allow you to see +# how the behavior of the model compares to that of previous versions. For more guidance, +# consult the following wiki page: +# https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model +# + +# NOTE: The following is an example of a custom scenario just to get you started. +# Replace "Some input" and "Another input" with the names of some variables you'd +# like to test. +- scenario: + title: Custom scenario + subtitle: gradual ramp-up + with: + - input: Some input + at: 10 + - input: Another input + at: 20 +` + export async function updateSdeConfig(projDir: string, mdlPath: string, genFormat: string): Promise { // Read the `sde.config.js` file from the template const configPath = joinPath(projDir, 'sde.config.js') @@ -62,18 +90,19 @@ export async function updateSdeConfig(projDir: string, mdlPath: string, genForma await writeFile(configPath, configText) } -export async function generateCheckYaml(projDir: string, mdlPath: string): Promise { - // Generate a sample `{mdl}.check.yaml` file if one doesn't already exist - // TODO: Make this optional (ask user first)? - const checkYamlFile = mdlPath.replace('.mdl', '.check.yaml') - const checkYamlPath = joinPath(projDir, checkYamlFile) - if (!existsSync(checkYamlPath)) { +async function generateYaml(projDir: string, kind: 'checks' | 'comparisons', template: string): Promise { + const yamlDir = joinPath(projDir, 'model', kind) + const yamlPath = joinPath(yamlDir, `${kind}.yaml`) + if (!existsSync(yamlPath)) { // Get relative path from yaml file parent dir to project dir - let relProjPath = relative(dirname(checkYamlPath), projDir) + let relProjPath = relative(dirname(yamlPath), projDir) if (relProjPath.length === 0) { relProjPath = './' } + // Create the directory for the yaml file, if needed + await mkdir(yamlDir, { recursive: true }) + // TODO: This path is normally different depending on whether using npm/yarn or // pnpm. For npm/yarn, `check-core` is hoisted under top-level `node_modules`, // but for pnpm, it is nested under `node_modules/.pnpm`. As an ugly workaround @@ -82,13 +111,22 @@ export async function generateCheckYaml(projDir: string, mdlPath: string): Promi // suffice). This allows us to use the same path here that works for all // three package managers. const nodeModulesPart = joinPath(relProjPath, 'node_modules') - const checkCorePart = '@sdeverywhere/check-core/schema/check.schema.json' + const schemaName = kind === 'checks' ? 'check' : 'comparison' + const checkCorePart = `@sdeverywhere/check-core/schema/${schemaName}.schema.json` const schemaPath = `${nodeModulesPart}/${checkCorePart}` - const checkContent = sampleCheckContent.replace('SCHEMA_PATH', schemaPath) - await writeFile(checkYamlPath, checkContent) + const yamlContent = template.replace('SCHEMA_PATH', schemaPath) + await writeFile(yamlPath, yamlContent) } } +export async function generateSampleYamlFiles(projDir: string): Promise { + // Generate a sample `checks.yaml` file if one doesn't already exist + await generateYaml(projDir, 'checks', sampleChecksContent) + + // Generate a sample `comparisons.yaml` file if one doesn't already exist + await generateYaml(projDir, 'comparisons', sampleComparisonsContent) +} + export async function chooseGenConfig(projDir: string, mdlPath: string): Promise { // TODO: For now we eagerly read the mdl file; maybe change this to only load it if // the user chooses to generate graph and/or slider config From 83f7544b797ca139b2e12ae641994a4e0c76c6a1 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 2 Dec 2024 14:01:26 -0800 Subject: [PATCH 7/8] docs: replace references to `*.check.yaml` --- examples/hello-world/README.md | 8 +++++--- examples/sir/README.md | 7 ++++--- examples/template-default/README.md | 5 +++-- examples/template-minimal/README.md | 5 +++-- packages/plugin-check/src/options.ts | 8 -------- packages/plugin-check/src/plugin.ts | 8 ++++---- 6 files changed, 19 insertions(+), 22 deletions(-) diff --git a/examples/hello-world/README.md b/examples/hello-world/README.md index a0c0a52c..7918c3ae 100644 --- a/examples/hello-world/README.md +++ b/examples/hello-world/README.md @@ -24,9 +24,11 @@ cd ./hello-world npm install # Enter development mode for the sample model. This will start a live -# development environment that will build a WebAssembly version of the -# sample model and run checks on it any time you make changes to the -# Vensim model file (sample.mdl) or the checks file (sample.check.yaml). +# development environment that will build a JavaScript version of the +# sample model and run checks on it any time you make changes to: +# - the Vensim model file (sample.mdl) +# - the model check definitions (model/checks/*.yaml) +# - the model comparison definitions (model/comparisons/*.yaml) npm run dev ``` diff --git a/examples/sir/README.md b/examples/sir/README.md index fec6e8f4..e8045e5d 100644 --- a/examples/sir/README.md +++ b/examples/sir/README.md @@ -29,11 +29,12 @@ cd ./sir npm create @sdeverywhere@latest # Enter development mode for the sample model. This will start a live -# development environment that will build a WebAssembly version of the +# development environment that will build a JavaScript version of the # sample model and run checks on it any time you make changes to: # - the config files -# - the Vensim model file (sir.mdl) -# - the checks file (sir.check.yaml) +# - the Vensim model file (model/sir.mdl) +# - the model check definitions (model/checks/*.yaml) +# - the model comparison definitions (model/comparisons/*.yaml) npm run dev ``` diff --git a/examples/template-default/README.md b/examples/template-default/README.md index e9764200..bbec731e 100644 --- a/examples/template-default/README.md +++ b/examples/template-default/README.md @@ -26,11 +26,12 @@ The project includes: npm create @sdeverywhere@latest # Enter development mode for your model. This will start a live -# development environment that will build a WebAssembly version of the +# development environment that will build a JavaScript version of the # model and run checks on it any time you make changes to: # - the config files # - the Vensim model file (.mdl) -# - the checks file (.check.yaml) +# - the model check definitions (model/checks/*.yaml) +# - the model comparison definitions (model/comparisons/*.yaml) npm run dev ``` diff --git a/examples/template-minimal/README.md b/examples/template-minimal/README.md index 9ec2b9d3..4998e322 100644 --- a/examples/template-minimal/README.md +++ b/examples/template-minimal/README.md @@ -22,10 +22,11 @@ to add the `@sdeverywhere/plugin-config` package to your project.) npm create @sdeverywhere@latest # Enter development mode for your model. This will start a live -# development environment that will build a WebAssembly version of the +# development environment that will build a JavaScript version of the # model and run checks on it any time you make changes to: # - the Vensim model file (.mdl) -# - the checks file (.check.yaml) +# - the model check definitions (model/checks/*.yaml) +# - the model comparison definitions (model/comparisons/*.yaml) npm run dev ``` diff --git a/packages/plugin-check/src/options.ts b/packages/plugin-check/src/options.ts index d07817c3..be43e483 100644 --- a/packages/plugin-check/src/options.ts +++ b/packages/plugin-check/src/options.ts @@ -15,14 +15,6 @@ export interface CheckPluginOptions { /** The current bundle, i.e., the bundle that is being developed and checked. */ current?: CheckBundle - // TODO - // /** - // * The glob-style path to the check yaml files to be included. If undefined, - // * a default pattern will be used that finds all `*.check.yaml` files under - // * the configured `rootDir`. - // */ - // yamlPath?: string - /** * The absolute path to the JS file containing the test configuration. If undefined, * a default test configuration will be used. diff --git a/packages/plugin-check/src/plugin.ts b/packages/plugin-check/src/plugin.ts index 44fe3ecf..97cb78bf 100644 --- a/packages/plugin-check/src/plugin.ts +++ b/packages/plugin-check/src/plugin.ts @@ -40,7 +40,7 @@ class CheckPlugin implements Plugin { if (this.options?.testConfigPath === undefined) { // Test config was not provided, so generate a default config in watch mode. // The test template uses import.meta.glob so that checks are re-run - // automatically when the *.check.yaml files are changed. + // automatically when the `{checks/comparisons}/*.yaml` files are changed. await this.genTestConfig(config, 'watch') } @@ -56,9 +56,9 @@ class CheckPlugin implements Plugin { // in the baselines directory (Vite's import.meta.glob handling doesn't // seem to do this automatically), so as a workaround, watch the baselines // directory and restart the server if files are added/removed - // TODO: The same problem also applies to the glob for `.check.yaml` files - // in the test config, so we should also reload if files match/unmatch - // the `.check.yaml` glob + // TODO: The same problem also applies to the glob for `checks/*.yaml` and + // `comparisons/*.yaml` files in the test config, so we should also reload + // if files match/unmatch those glob patterns // TODO: Use the baselines directory from the config const baselinesDir = 'baselines' const watcher = chokidar.watch(baselinesDir, { From 0eb8e769aadad73bc5611277d1cd9786b70212d7 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Mon, 2 Dec 2024 14:41:02 -0800 Subject: [PATCH 8/8] docs: add comment --- packages/plugin-check/template-tests/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/plugin-check/template-tests/src/index.ts b/packages/plugin-check/template-tests/src/index.ts index 69cc0bbe..f8e78bb2 100644 --- a/packages/plugin-check/template-tests/src/index.ts +++ b/packages/plugin-check/template-tests/src/index.ts @@ -70,6 +70,8 @@ export async function getConfigOptions( // - once with single input at its minimum // - once with single input at its maximum const baseComparisonSpecs = createBaseComparisonSpecs(bundleL, bundleR) + + // Also include custom scenarios defined in the `comparisons/*.yaml` files const comparisonSpecs = [baseComparisonSpecs, ...yamlComparisonSpecFiles] comparisonOptions = {