feat: allow for defining model comparison scenarios in YAML files (#330)

Fixes #315

examples/sample-check-app/index.html

@@ -9,7 +9,7 @@
     <link rel="preconnect" href="https://fonts.googleapis.com" />
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
     <link
-      href="https://fonts.googleapis.com/css2?family=Roboto+Condensed:wght@400&family=Roboto:wght@500;700&display=swap"
+      href="https://fonts.googleapis.com/css2?family=Roboto+Condensed:wght@400;700&family=Roboto:wght@500;700&display=swap"
       rel="stylesheet"
     />
 

examples/sample-check-app/src/index.ts

@@ -21,12 +21,42 @@ if (suiteSummaryJson) {
   suiteSummary = JSON.parse(suiteSummaryJson) as SuiteSummary
 }
 
-// Load the bundles and build the model check/compare configuration
-const checkOptions = getConfigOptions(baselineBundle() as Bundle, currentBundle() as Bundle)
+// In local development mode, the app header contains a "Simplify Scenarios" checkbox; if checked,
+// we can create a configuration that includes a simpler set of specs so that the tests run faster
+function loadSimplifyScenariosFlag(): boolean {
+  if (import.meta.hot) {
+    return localStorage.getItem('sde-check-simplify-scenarios') === '1'
+  } else {
+    return false
+  }
+}
+
+async function initBundlesAndUI() {
+  // Before switching bundles, clear out the app-shell-container element
+  const container = document.getElementById('app-shell-container')
+  while (container.firstChild) {
+    container.removeChild(container.firstChild)
+  }
 
-// Initialize the root Svelte component
-const appShell = initAppShell(checkOptions, {
-  suiteSummary
-})
+  // TODO: Release resources associated with active bundles
 
-export default appShell
+  // Load the bundles and build the model check/compare configuration
+  const simplifyScenarios = loadSimplifyScenariosFlag()
+  const configOptions = getConfigOptions(baselineBundle() as Bundle, currentBundle() as Bundle, { simplifyScenarios })
+
+  // Initialize the root Svelte component
+  initAppShell(configOptions, {
+    suiteSummary
+  })
+}
+
+// Initialize the bundles and user interface
+initBundlesAndUI()
+
+if (import.meta.hot) {
+  // Reload everything when the user toggles the "Simplify Scenarios" checkbox
+  document.addEventListener('sde-check-simplify-scenarios-toggled', () => {
+    // Reinitialize using the new state
+    initBundlesAndUI()
+  })
+}

examples/sample-check-bundle/src/bundle.ts

@@ -7,17 +7,15 @@ import type {
   BundleModel as CheckBundleModel,
   DatasetKey,
   DatasetsResult,
-  ImplVar,
-  InputVar,
   LinkItem,
   ModelSpec,
-  Scenario
+  ScenarioSpec
 } from '@sdeverywhere/check-core'
 
 import { getGraphDataForScenario, getGraphLinksForScenario, getGraphSpecs } from './graphs'
-import { getInputVars } from './inputs'
+import { getInputs } from './inputs'
 import { getDatasetsForScenario } from './model-data'
-import { getOutputVars } from './outputs'
+import { getOutputs } from './outputs'
 
 // import modelWorkerJs from './worker.iife.js?raw'
 
@@ -47,18 +45,18 @@ export class BundleModel implements CheckBundleModel {
   constructor(public readonly modelSpec: ModelSpec) {}
 
   // from CheckBundleModel interface
-  async getDatasetsForScenario(scenario: Scenario, datasetKeys: DatasetKey[]): Promise<DatasetsResult> {
-    return getDatasetsForScenario(modelVersion, this.modelSpec, scenario, datasetKeys)
+  async getDatasetsForScenario(scenarioSpec: ScenarioSpec, datasetKeys: DatasetKey[]): Promise<DatasetsResult> {
+    return getDatasetsForScenario(modelVersion, this.modelSpec, scenarioSpec, datasetKeys)
   }
 
   // from CheckBundleModel interface
-  async getGraphDataForScenario(scenario: Scenario, graphId: BundleGraphId): Promise<BundleGraphData> {
-    return getGraphDataForScenario(scenario, graphId)
+  async getGraphDataForScenario(scenarioSpec: ScenarioSpec, graphId: BundleGraphId): Promise<BundleGraphData> {
+    return getGraphDataForScenario(scenarioSpec, graphId)
   }
 
   // from CheckBundleModel interface
-  getGraphLinksForScenario(scenario: Scenario, graphId: BundleGraphId): LinkItem[] {
-    return getGraphLinksForScenario(scenario, graphId)
+  getGraphLinksForScenario(scenarioSpec: ScenarioSpec, graphId: BundleGraphId): LinkItem[] {
+    return getGraphLinksForScenario(scenarioSpec, graphId)
   }
 }
 
@@ -84,61 +82,21 @@ async function initBundleModel(modelSpec: ModelSpec): Promise<BundleModel> {
  */
 export function createBundle(): Bundle {
   // Gather information about the input and output variables used in the model
-  const inputVars = getInputVars()
-  const outputVars = getOutputVars(modelVersion)
-
-  // TODO: For an SDEverywhere-generated model, you could use the JSON file
-  // produced by `sde generate --listjson` to create an `ImplVar` for each
-  // internal variable used in the model.  For the purposes of this sample
-  // bundle, we will synthesize `ImplVar` instances for the model outputs.
-  const implVarArray = [...outputVars.values()]
-    .filter(outputVar => outputVar.sourceName === undefined)
-    .map((outputVar, index) => {
-      const implVar: ImplVar = {
-        varId: outputVar.varId,
-        varName: outputVar.varName,
-        varIndex: index,
-        dimensions: [],
-        varType: 'aux'
-      }
-      return implVar
-    })
-  const implVars: Map<DatasetKey, ImplVar> = new Map()
-  implVarArray.forEach(implVar => {
-    implVars.set(`ModelImpl${implVar.varId}`, implVar)
-  })
-
-  // Configure input groups
-  const inputGroups: Map<string, InputVar[]> = new Map([
-    ['All Inputs', [...inputVars.values()]],
-    ['Input Group 1', [inputVars.get('_input_a'), inputVars.get('_input_b')]],
-    ['Empty Input Group', []]
-  ])
-
-  // Configure dataset groups
-  const keyForVarWithName = (name: string) => {
-    return [...outputVars.entries()].find(e => e[1].varName === name)[0]
-  }
-  const keysForVarsWithSource = (sourceName?: string) => {
-    return [...outputVars.entries()].filter(e => e[1].sourceName === sourceName).map(([k]) => k)
-  }
-  const datasetGroups: Map<string, DatasetKey[]> = new Map([
-    ['All Outputs', keysForVarsWithSource(undefined)],
-    ['Basic Outputs', [keyForVarWithName('Output X'), keyForVarWithName('Output Y'), keyForVarWithName('Output Z')]],
-    ['Static', keysForVarsWithSource('StaticData')]
-  ])
+  const inputs = getInputs(modelVersion)
+  const outputs = getOutputs(modelVersion)
 
   // Configure graphs
-  const graphSpecs = getGraphSpecs(modelVersion, outputVars)
+  const graphSpecs = getGraphSpecs(modelVersion, outputs.outputVars)
 
   const modelSpec: ModelSpec = {
     modelSizeInBytes,
     dataSizeInBytes,
-    inputVars,
-    outputVars,
-    implVars,
-    inputGroups,
-    datasetGroups,
+    inputVars: inputs.inputVars,
+    inputGroups: inputs.inputGroups,
+    inputAliases: inputs.inputAliases,
+    outputVars: outputs.outputVars,
+    implVars: outputs.implVars,
+    datasetGroups: outputs.datasetGroups,
     startTime: 1850,
     endTime: 2100,
     graphSpecs

examples/sample-check-bundle/src/graphs.ts

@@ -10,7 +10,7 @@ import type {
   LegendItem,
   LinkItem,
   OutputVar,
-  Scenario
+  ScenarioSpec
 } from '@sdeverywhere/check-core'
 
 const barWidths = [200, 150, 120]
@@ -76,7 +76,7 @@ class SampleGraphData implements BundleGraphData {
  */
 export function getGraphDataForScenario(
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  _scenario: Scenario,
+  _scenarioSpec: ScenarioSpec,
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   _graphId: BundleGraphId
 ): BundleGraphData {
@@ -87,7 +87,7 @@ export function getGraphDataForScenario(
  * Return the links for the given graph.
  */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export function getGraphLinksForScenario(_scenario: Scenario, _graphId: BundleGraphId): LinkItem[] {
+export function getGraphLinksForScenario(_scenarioSpec: ScenarioSpec, _graphId: BundleGraphId): LinkItem[] {
   return [
     {
       kind: 'url',

examples/sample-check-bundle/src/inputs.ts

@@ -1,18 +1,28 @@
 // Copyright (c) 2022 Climate Interactive / New Venture Fund
 
-import type { InputVar, VarId } from '@sdeverywhere/check-core'
+import type { InputId, InputVar, VarId } from '@sdeverywhere/check-core'
+
+export interface Inputs {
+  inputVars: Map<VarId, InputVar>
+  inputGroups: Map<string, InputVar[]>
+  inputAliases: Map<string, VarId>
+}
 
 /**
  * Gather the list of input variables used in this version of the model.
  */
-export function getInputVars(): Map<VarId, InputVar> {
+export function getInputs(modelVersion: number): Inputs {
   const inputVars: Map<VarId, InputVar> = new Map()
+  const inputAliases: Map<string, VarId> = new Map()
 
   // TODO: Typically you would return the actual list of model inputs (for
   // example, the list used to configure an SDEverywhere-generated model),
   // but for now we will use a hardcoded list of inputs
-  const addSlider = (varId: VarId, varName: string) => {
+  const addSlider = (inputId: InputId, varName: string) => {
+    const varId = `_${varName.toLowerCase().replace(/\s/g, '_')}`
+
     inputVars.set(varId, {
+      inputId,
       varId,
       varName,
       defaultValue: 50,
@@ -23,12 +33,41 @@ export function getInputVars(): Map<VarId, InputVar> {
         locationPath: ['Assumptions', varName]
       }
     })
+
+    const sliderName = `Main Sliders > Slider ${inputId}`
+    inputAliases.set(sliderName, varId)
+  }
+
+  // Slider id=1 is defined with input var 'Input A' in both v1 and v2
+  addSlider('1', 'Input A')
+
+  // Slider id=2 is defined with input var 'Input B' in v1, but pretend it was renamed
+  // to 'Input B prime' in v2
+  addSlider('2', modelVersion === 1 ? 'Input B' : 'Input B prime')
+
+  // Slider id=3 is defined with input var 'Input C' in v1 only (pretend slider was
+  // removed in v2)
+  if (modelVersion === 1) {
+    addSlider('3', 'Input C')
+  }
+
+  // Slider id=4 is defined with input var 'Input D' in v2 only (pretend slider was
+  // added in v2)
+  if (modelVersion === 2) {
+    addSlider('4', 'Input D')
   }
-  addSlider('_input_a', 'Input A')
-  addSlider('_input_b', 'Input B')
-  // XXX
-  addSlider('_carbon_tax_initial_target', 'Carbon tax initial target')
-  addSlider('_global_population_in_2100', 'Global population in 2100')
 
-  return inputVars
+  // Configure input groups
+  const inputGroups: Map<string, InputVar[]> = new Map([
+    ['All Inputs', [...inputVars.values()]],
+    // ['Input Group 1', [inputVars.get('_input_a'), inputVars.get('_input_b')]],
+    ['Input Group 1', [inputVars.get('_input_a')]],
+    ['Empty Input Group', []]
+  ])
+
+  return {
+    inputVars,
+    inputGroups,
+    inputAliases
+  }
 }

examples/sample-check-bundle/src/model-data.ts

@@ -1,22 +1,32 @@
 // Copyright (c) 2022 Climate Interactive / New Venture Fund
 
-import type { Dataset, DatasetKey, DatasetsResult, ModelSpec, Scenario, ScenarioKey } from '@sdeverywhere/check-core'
+import type {
+  Dataset,
+  DatasetKey,
+  DatasetsResult,
+  ModelSpec,
+  ScenarioSpec,
+  ScenarioSpecUid
+} from '@sdeverywhere/check-core'
 
 // The deltas that are included for a certain output variable to
 // simulate differences between two versions of the model.
-const deltas: Map<ScenarioKey, number> = new Map()
+const deltas: Map<ScenarioSpecUid, number> = new Map()
 
 export function getDatasetsForScenario(
   modelVersion: number,
   modelSpec: ModelSpec,
-  scenario: Scenario,
+  scenarioSpec: ScenarioSpec,
   datasetKeys: DatasetKey[]
 ): DatasetsResult {
   const datasetMap: Map<DatasetKey, Dataset> = new Map()
 
-  // TODO: Set the model inputs according to the given scenario
+  // TODO: With a real JS model, you would configure the model inputs based on
+  // the scenario that is passed in.  For this sample bundle, we tweak the fake
+  // output values below based on which scenario is passed in.
 
-  // TODO: Run the JS model
+  // TODO: With a real JS model, you would run the model here.  For this sample
+  // bundle, we simulate a non-zero elapsed time.
   const modelRunTime = 30 - 5 * modelVersion + Math.random() * 2
 
   // Extract the data for each requested output variable and put it into a map
@@ -30,19 +40,19 @@ export function getDatasetsForScenario(
 
     // TODO: With a real model, you would get the actual data for the
     // requested variable here (from the set of model outputs).  For this
-    // sample bundle, we generate a few data points.  For one variable,
+    // sample bundle, we generate a few data points.  For a couple variables,
     // we generate different values depending on the model version to
     // demonstrate how differences are shown in the report.
+    const isDefaultScenario = scenarioSpec.kind === 'all-inputs' && scenarioSpec.position === 'at-default'
+    const hasInputA =
+      scenarioSpec.kind === 'input-settings' && scenarioSpec.settings.find(s => s.inputVarId.includes('input_a'))
+
     let delta: number
-    const hasInputA = scenario.key.includes('input_a') || scenario.key.includes('input_group')
-    if (
-      (datasetKey === 'Model__output_x' && hasInputA) ||
-      (datasetKey === 'Model__output_z' && scenario.key === 'all_inputs_at_default')
-    ) {
-      delta = deltas.get(scenario.key)
+    if ((datasetKey === 'Model__output_x' && hasInputA) || (datasetKey === 'Model__output_z' && isDefaultScenario)) {
+      delta = deltas.get(scenarioSpec.uid)
       if (delta === undefined) {
         delta = Math.random() * modelVersion
-        deltas.set(scenario.key, delta)
+        deltas.set(scenarioSpec.uid, delta)
       }
     } else if (datasetKey === 'Model__historical_x') {
       delta = -0.5
@@ -60,6 +70,7 @@ export function getDatasetsForScenario(
       // }
       delta = 0
     }
+
     const dataset: Dataset = new Map()
     dataset.set(1850, 5 + delta)
     dataset.set(1900, 6 + delta)