Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: update default plugin-check configuration to include comparison yaml files #574

Merged
merged 8 commits into from
Dec 2, 2024
Prev Previous commit
Next Next commit
fix: update create package to generate a sample comparisons.yaml file…
… + change location of sample checks.yaml file
chrispcampbell committed Dec 2, 2024
commit 996886d235fa2a51887d14b8e527a81356490ace
6 changes: 3 additions & 3 deletions packages/create/src/index.ts
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@ import detectPackageManager from 'which-pm-runs'
import yargs from 'yargs-parser'

import { chooseCodeFormat } from './step-code-format'
import { chooseGenConfig, generateCheckYaml, updateSdeConfig } from './step-config'
import { chooseGenConfig, generateSampleYamlFiles, updateSdeConfig } from './step-config'
import { chooseInstallDeps } from './step-deps'
import { chooseProjectDir } from './step-directory'
import { chooseInstallEmsdk } from './step-emsdk'
@@ -54,10 +54,10 @@ export async function main(): Promise<void> {
const genFormat = await chooseCodeFormat()

// Update the `sde.config.js` file to use the chosen mdl file and
// generate a sample `.check.yaml` file
// generate sample `checks.yaml` and `comparisons.yaml` files
if (!args.dryRun) {
await updateSdeConfig(projDir, mdlPath, genFormat)
await generateCheckYaml(projDir, mdlPath)
await generateSampleYamlFiles(projDir)
}
console.log()

66 changes: 52 additions & 14 deletions packages/create/src/step-config.ts
Original file line number Diff line number Diff line change
@@ -30,12 +30,17 @@ interface MdlLevelVariable {

type MdlVariable = MdlConstVariable | MdlAuxVariable | MdlLevelVariable

const sampleCheckContent = `\
const sampleChecksContent = `\
# yaml-language-server: $schema=SCHEMA_PATH

# NOTE: This is just a simple check to get you started. Replace "Some output" with
# the name of some variable you'd like to test. Additional tests can be developed
# in the "playground" (beta) inside the model-check report.
#
# This file contains "check" tests that exercise your model under different input
# scenarios. For more guidance, consult this wiki page:
# https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model
#

# NOTE: The following is an example of a simple check just to get you started.
# Replace "Some output" with the name of some variable you'd like to test.
- describe: Some output
tests:
- it: should be > 0 for all input scenarios
@@ -47,6 +52,29 @@ const sampleCheckContent = `\
- gt: 0
`

const sampleComparisonsContent = `\
# yaml-language-server: $schema=SCHEMA_PATH

#
# This file contains definitions of custom comparison scenarios, which allow you to see
# how the behavior of the model compares to that of previous versions. For more guidance,
# consult the following wiki page:
# https://github.com/climateinteractive/SDEverywhere/wiki/Testing-and-Comparing-Your-Model
#

# NOTE: The following is an example of a custom scenario just to get you started.
# Replace "Some input" and "Another input" with the names of some variables you'd
# like to test.
- scenario:
title: Custom scenario
subtitle: gradual ramp-up
with:
- input: Some input
at: 10
- input: Another input
at: 20
`

export async function updateSdeConfig(projDir: string, mdlPath: string, genFormat: string): Promise<void> {
// Read the `sde.config.js` file from the template
const configPath = joinPath(projDir, 'sde.config.js')
@@ -62,18 +90,19 @@ export async function updateSdeConfig(projDir: string, mdlPath: string, genForma
await writeFile(configPath, configText)
}

export async function generateCheckYaml(projDir: string, mdlPath: string): Promise<void> {
// Generate a sample `{mdl}.check.yaml` file if one doesn't already exist
// TODO: Make this optional (ask user first)?
const checkYamlFile = mdlPath.replace('.mdl', '.check.yaml')
const checkYamlPath = joinPath(projDir, checkYamlFile)
if (!existsSync(checkYamlPath)) {
async function generateYaml(projDir: string, kind: 'checks' | 'comparisons', template: string): Promise<void> {
const yamlDir = joinPath(projDir, 'model', kind)
const yamlPath = joinPath(yamlDir, `${kind}.yaml`)
if (!existsSync(yamlPath)) {
// Get relative path from yaml file parent dir to project dir
let relProjPath = relative(dirname(checkYamlPath), projDir)
let relProjPath = relative(dirname(yamlPath), projDir)
if (relProjPath.length === 0) {
relProjPath = './'
}

// Create the directory for the yaml file, if needed
await mkdir(yamlDir, { recursive: true })

// TODO: This path is normally different depending on whether using npm/yarn or
// pnpm. For npm/yarn, `check-core` is hoisted under top-level `node_modules`,
// but for pnpm, it is nested under `node_modules/.pnpm`. As an ugly workaround
@@ -82,13 +111,22 @@ export async function generateCheckYaml(projDir: string, mdlPath: string): Promi
// suffice). This allows us to use the same path here that works for all
// three package managers.
const nodeModulesPart = joinPath(relProjPath, 'node_modules')
const checkCorePart = '@sdeverywhere/check-core/schema/check.schema.json'
const schemaName = kind === 'checks' ? 'check' : 'comparison'
const checkCorePart = `@sdeverywhere/check-core/schema/${schemaName}.schema.json`
const schemaPath = `${nodeModulesPart}/${checkCorePart}`
const checkContent = sampleCheckContent.replace('SCHEMA_PATH', schemaPath)
await writeFile(checkYamlPath, checkContent)
const yamlContent = template.replace('SCHEMA_PATH', schemaPath)
await writeFile(yamlPath, yamlContent)
}
}

export async function generateSampleYamlFiles(projDir: string): Promise<void> {
// Generate a sample `checks.yaml` file if one doesn't already exist
await generateYaml(projDir, 'checks', sampleChecksContent)

// Generate a sample `comparisons.yaml` file if one doesn't already exist
await generateYaml(projDir, 'comparisons', sampleComparisonsContent)
}

export async function chooseGenConfig(projDir: string, mdlPath: string): Promise<void> {
// TODO: For now we eagerly read the mdl file; maybe change this to only load it if
// the user chooses to generate graph and/or slider config