diff --git a/packages/build/docs/classes/BuildContext.md b/packages/build/docs/classes/BuildContext.md index 6f7e24f5..10744511 100644 --- a/packages/build/docs/classes/BuildContext.md +++ b/packages/build/docs/classes/BuildContext.md @@ -107,3 +107,25 @@ Spawn a child process that runs the given command. `Promise`<`ProcessOutput`\> The output of the process. + +___ + +### canonicalVarId + +**canonicalVarId**(`name`): `string` + +Format a (subscripted or non-subscripted) model variable name into a canonical +identifier (with special characters converted to underscore, and subscript/dimension +parts separated by commas). + +#### Parameters + +| Name | Type | Description | +| :------ | :------ | :------ | +| `name` | `string` | The name of the variable in the source model, e.g., `Variable name[DimA, B2]`. | + +#### Returns + +`string` + +The canonical identifier for the given name, e.g., `_variable_name[_dima,_b2]`. diff --git a/packages/build/package.json b/packages/build/package.json index 7b114117..d9105951 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -30,6 +30,7 @@ "ci:build": "run-s clean lint prettier:check type-check build test:ci docs" }, "dependencies": { + "@sdeverywhere/parse": "^0.1.1", "chokidar": "^3.5.3", "cross-spawn": "^7.0.3", "folder-hash": "^4.0.2", diff --git a/packages/build/src/context/context.ts b/packages/build/src/context/context.ts index fd5df924..762b297d 100644 --- a/packages/build/src/context/context.ts +++ b/packages/build/src/context/context.ts @@ -1,5 +1,6 @@ // Copyright (c) 2022 Climate Interactive / New Venture Fund +import { canonicalVarId } from '@sdeverywhere/parse' import type { LogLevel } from '../_shared/log' import { log } from '../_shared/log' @@ -84,4 +85,16 @@ export class BuildContext { spawnChild(cwd: string, command: string, args: string[], opts?: ProcessOptions): Promise { return spawnChild(cwd, command, args, this.abortSignal, opts) } + + /** + * Format a (subscripted or non-subscripted) model variable name into a canonical + * identifier (with special characters converted to underscore, and subscript/dimension + * parts separated by commas). + * + * @param name The name of the variable in the source model, e.g., `Variable name[DimA, B2]`. + * @returns The canonical identifier for the given name, e.g., `_variable_name[_dima,_b2]`. + */ + canonicalVarId(name: string): string { + return canonicalVarId(name) + } } diff --git a/packages/compile/package.json b/packages/compile/package.json index d0e08909..ad0b8c21 100644 --- a/packages/compile/package.json +++ b/packages/compile/package.json @@ -22,7 +22,6 @@ "csv-parse": "^5.3.3", "js-yaml": "^3.13.1", "ramda": "^0.27.0", - "split-string": "^6.0.0", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" }, "author": "Climate Interactive", diff --git a/packages/compile/src/_shared/helpers.js b/packages/compile/src/_shared/helpers.js index af74dae1..5c41de3b 100644 --- a/packages/compile/src/_shared/helpers.js +++ b/packages/compile/src/_shared/helpers.js @@ -3,9 +3,10 @@ import util from 'util' import B from 'bufx' import { parse as parseCsv } from 'csv-parse/sync' import * as R from 'ramda' -import split from 'split-string' import XLSX from 'xlsx' +import { canonicalId, canonicalVarId } from '@sdeverywhere/parse' + // Set true to print a stack trace in vlog export const PRINT_VLOG_TRACE = false @@ -40,25 +41,11 @@ export function resetHelperState() { } export let canonicalName = name => { - // Format a model variable name into a valid C identifier. - return ( - '_' + - name - .trim() - .replace(/"/g, '_') - .replace(/\s+!$/g, '!') - .replace(/\s/g, '_') - .replace(/,/g, '_') - .replace(/-/g, '_') - .replace(/\./g, '_') - .replace(/\$/g, '_') - .replace(/'/g, '_') - .replace(/&/g, '_') - .replace(/%/g, '_') - .replace(/\//g, '_') - .replace(/\|/g, '_') - .toLowerCase() - ) + // Format a model variable or subscript/dimension name into a valid C identifier. + // In the case where you have a full variable name that includes subscripts/dimensions + // (e.g., 'Variable name[DimA,B2]'), use `canonicalVensimName` to convert the + // base variable name and subscript/dimension parts to canonical form indepdendently. + return canonicalId(name) } export let decanonicalize = name => { // Decanonicalize the var name. @@ -71,9 +58,6 @@ export let decanonicalize = name => { } return name } -export let cFunctionName = name => { - return canonicalName(name).toUpperCase() -} export let isSeparatedVar = v => { return v.separationDims.length > 0 } @@ -244,22 +228,7 @@ export let readCsv = (pathname, delimiter = ',') => { } // Convert the var name and subscript names to canonical form separately. export let canonicalVensimName = vname => { - let result = vname - let m = vname.match(/([^[]+)(?:\[([^\]]+)\])?/) - if (m) { - result = canonicalName(m[1]) - if (m[2]) { - let subscripts = m[2].split(',').map(x => canonicalName(x)) - result += `[${subscripts.join(',')}]` - } - } - return result -} -// Split a model string into an array of equations without the "|" terminator. -// Allow "|" to occur in quoted variable names across line breaks. -// Retain the backslash character. -export let splitEquations = mdl => { - return split(mdl, { separator: '|', quotes: ['"'], keep: () => true }) + return canonicalVarId(vname) } // Function to map over lists's value and index export let mapIndexed = R.addIndex(R.map) diff --git a/packages/parse/src/_shared/canonical-id.js b/packages/parse/src/_shared/canonical-id.js new file mode 100644 index 00000000..e90c4f98 --- /dev/null +++ b/packages/parse/src/_shared/canonical-id.js @@ -0,0 +1,79 @@ +// Copyright (c) 2023 Climate Interactive / New Venture Fund + +// Detect '!' at the end of a marked dimension when preceded by whitespace +const reTrailingMark = new RegExp('\\s+!$', 'g') + +// Detect one or more consecutive whitespace or underscore characters +const reWhitespace = new RegExp('(\\s|_)+', 'g') + +// Detect special punctuation characters +// TODO: We do not currently include '!' characters in this set; we should only replace these +// when they don't appear at the end of a (marked) dimension +const reSpecialChars = new RegExp(`['"\\.,\\-\\$&%\\/\\|]`, 'g') + +/** + * Format a model variable or subscript/dimension name into a valid C identifier (with + * special characters converted to underscore). + * + * Note that this should only be called with an individual variable base name (e.g., + * 'Variable name') or a subscript/dimension name (e.g., 'DimA'). In the case where + * you have a full variable name that includes subscripts/dimensions (e.g., + * 'Variable name[DimA,B2]'), use `canonicalVarId` to convert the base variable name + * and subscript/dimension parts to canonical form indepdendently. + * + * @param {string} name The name of the variable in the source model, e.g., "Variable name". + * @returns {string} The C identifier for the given name, e.g., "_variable_name". + */ +export function canonicalId(name) { + return ( + '_' + + name + // Ignore any leading or trailing whitespace + .trim() + // When a '!' character appears at the end of a marked dimension, preserve the mark + // but remove any preceding whitespace + .replace(reTrailingMark, '!') + // Replace one or more consecutive whitespace or underscore characters with a single + // underscore character; this matches the behavior of Vensim documented here: + // https://www.vensim.com/documentation/ref_variable_names.html + .replace(reWhitespace, '_') + // Replace each special punctuation character with an underscore + .replace(reSpecialChars, '_') + // Convert to lower case + .toLowerCase() + ) +} + +/** + * Format a (subscripted or non-subscripted) model variable name into a canonical identifier, + * (with special characters converted to underscore, and subscript/dimension parts separated + * by commas). + * + * @param {string} name The name of the variable in the source model, e.g., "Variable name[DimA, B2]". + * @returns {string} The canonical identifier for the given name, e.g., "_variable_name[_dima,_b2]". + */ +export function canonicalVarId(name) { + const m = name.match(/([^[]+)(?:\[([^\]]+)\])?/) + if (!m) { + throw new Error(`Invalid variable name: ${name}`) + } + + let id = canonicalId(m[1]) + if (m[2]) { + const subscripts = m[2].split(',').map(x => canonicalId(x)) + id += `[${subscripts.join(',')}]` + } + + return id +} + +/** + * Format a model function name into a valid C identifier (with special characters + * converted to underscore, and the ID converted to uppercase). + * + * @param {string} name The name of the variable in the source model, e.g., "FUNCTION name". + * @returns {string} The C identifier for the given name, e.g., "_FUNCTION_NAME". + */ +export function canonicalFunctionId(name) { + return canonicalId(name).toUpperCase() +} diff --git a/packages/parse/src/_shared/canonical-id.spec.ts b/packages/parse/src/_shared/canonical-id.spec.ts new file mode 100644 index 00000000..46a29a34 --- /dev/null +++ b/packages/parse/src/_shared/canonical-id.spec.ts @@ -0,0 +1,85 @@ +// Copyright (c) 2024 Climate Interactive / New Venture Fund + +import { describe, expect, it } from 'vitest' + +import { canonicalFunctionId, canonicalId, canonicalVarId } from './canonical-id' + +describe('canonicalId', () => { + it('should collapse multiple consecutive whitespace or underscore characters to a single underscore', () => { + // The following examples are taken from the Vensim documentation under "Rules for Variable Names": + // https://www.vensim.com/documentation/ref_variable_names.html + expect(canonicalId('Hello There')).toBe('_hello_there') + expect(canonicalId('Hello_There')).toBe('_hello_there') + expect(canonicalId('Hello __ ___ There')).toBe('_hello_there') + }) + + it('should replace each special character with a single underscore', () => { + let input = '"Special' + let expected = '__special' + function add(name: string, char: string) { + input += ` ${name}${char}` + expected += `_${name}_` + } + add('period', '.') + add('comma', ',') + add('dash', '-') + add('dollar', '$') + add('amp', '&') + add('pct', '%') + add('slash', '/') + // TODO: Handle backslashes + // add('bslash', '\\') + // TODO: Handle parentheses + // add('lparen', '(') + // add('rparen', ')') + input += ' characters"' + expected += '_characters_' + expect(canonicalId(input)).toBe(expected) + + // The following examples are taken from the Vensim documentation under "Rules for Variable Names": + // https://www.vensim.com/documentation/ref_variable_names.html + expect(canonicalId('"HiRes TV/Web Sets"')).toBe('__hires_tv_web_sets_') + // TODO: Handle backslashes + // expect(canonicalId('"The \\"Final\\" Frontier"')).toBe('') + expect(canonicalId("érosion d'action")).toBe('_érosion_d_action') + }) + + it('should preserve mark when preceded by whitespace', () => { + expect(canonicalVarId(`DimA !`)).toBe('_dima!') + }) + + it('should preserve mark when split over multiple lines', () => { + const name = `DimA +! +` + expect(canonicalVarId(name)).toBe('_dima!') + }) +}) + +describe('canonicalVarId', () => { + it('should work for non-subscripted variable', () => { + expect(canonicalVarId('Hello There')).toBe('_hello_there') + }) + + it('should work for variable with 1 subscript', () => { + expect(canonicalVarId('Variable name[A1]')).toBe('_variable_name[_a1]') + }) + + it('should work for variable with 2 subscripts', () => { + expect(canonicalVarId('Variable name[A1, DimB]')).toBe('_variable_name[_a1,_dimb]') + }) + + it('should work for variable with 3 subscripts', () => { + expect(canonicalVarId('Variable name[A1, DimB,C2]')).toBe('_variable_name[_a1,_dimb,_c2]') + }) +}) + +describe('canonicalFunctionId', () => { + it('should work for uppercase function name', () => { + expect(canonicalFunctionId('FUNCTION NAME')).toBe('_FUNCTION_NAME') + }) + + it('should work for mixed case function name', () => { + expect(canonicalFunctionId('function name')).toBe('_FUNCTION_NAME') + }) +}) diff --git a/packages/parse/src/_shared/names.js b/packages/parse/src/_shared/names.js deleted file mode 100644 index 9649c55f..00000000 --- a/packages/parse/src/_shared/names.js +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2023 Climate Interactive / New Venture Fund - -/** - * Format a model variable name into a valid C identifier (with special characters - * converted to underscore). - * - * @param {string} name The name of the variable in the source model, e.g., "Variable name". - * @returns {string} The C identifier for the given name, e.g., "_variable_name". - */ -export function canonicalName(name) { - // TODO: This is also defined in the compile package. Would be good to - // define it in one place to reduce the chance of them getting out of sync. - return ( - '_' + - name - .trim() - .replace(/"/g, '_') - .replace(/\s+!$/g, '!') - .replace(/\s/g, '_') - .replace(/,/g, '_') - .replace(/-/g, '_') - .replace(/\./g, '_') - .replace(/\$/g, '_') - .replace(/'/g, '_') - .replace(/&/g, '_') - .replace(/%/g, '_') - .replace(/\//g, '_') - .replace(/\|/g, '_') - .toLowerCase() - ) -} - -/** - * Format a model function name into a valid C identifier (with special characters - * converted to underscore). - * - * @param {string} name The name of the variable in the source model, e.g., "FUNCTION NAME". - * @returns {string} The C identifier for the given name, e.g., "_FUNCTION_NAME". - */ -export function cFunctionName(name) { - return canonicalName(name).toUpperCase() -} diff --git a/packages/parse/src/ast/ast-builders.ts b/packages/parse/src/ast/ast-builders.ts index c8de4b03..c56f97ba 100644 --- a/packages/parse/src/ast/ast-builders.ts +++ b/packages/parse/src/ast/ast-builders.ts @@ -1,6 +1,6 @@ // Copyright (c) 2023 Climate Interactive / New Venture Fund -import { canonicalName, cFunctionName } from '../_shared/names' +import { canonicalFunctionId, canonicalId } from '../_shared/canonical-id' import type { BinaryOp, @@ -44,14 +44,14 @@ import type { export function subRef(dimOrSubName: DimOrSubName): SubscriptRef { return { subName: dimOrSubName, - subId: canonicalName(dimOrSubName) + subId: canonicalId(dimOrSubName) } } export function subMapping(toDimName: DimName, dimOrSubNames: DimOrSubName[] = []): SubscriptMapping { return { toDimName, - toDimId: canonicalName(toDimName), + toDimId: canonicalId(toDimName), subscriptRefs: dimOrSubNames.map(subRef) } } @@ -66,9 +66,9 @@ export function dimDef( ): DimensionDef { return { dimName, - dimId: canonicalName(dimName), + dimId: canonicalId(dimName), familyName, - familyId: canonicalName(familyName), + familyId: canonicalId(familyName), subscriptRefs: dimOrSubNames.map(subRef), subscriptMappings, comment, @@ -106,7 +106,7 @@ export function varRef(varName: VariableName, subscriptNames?: DimOrSubName[]): return { kind: 'variable-ref', varName, - varId: canonicalName(varName), + varId: canonicalId(varName), subscriptRefs: subscriptNames?.map(subRef) } } @@ -155,7 +155,7 @@ export function call(fnName: FunctionName, ...args: Expr[]): FunctionCall { return { kind: 'function-call', fnName, - fnId: cFunctionName(fnName), + fnId: canonicalFunctionId(fnName), args } } @@ -172,7 +172,7 @@ export function varDef( return { kind: 'variable-def', varName, - varId: canonicalName(varName), + varId: canonicalId(varName), subscriptRefs: subscriptNames?.map(subRef), exceptSubscriptRefSets: exceptSubscriptNames?.map(namesForSet => namesForSet.map(subRef)) } diff --git a/packages/parse/src/index.ts b/packages/parse/src/index.ts index 012e449e..8330a1c4 100644 --- a/packages/parse/src/index.ts +++ b/packages/parse/src/index.ts @@ -1,5 +1,7 @@ // Copyright (c) 2023 Climate Interactive / New Venture Fund +export * from './_shared/canonical-id' + export * from './ast/ast-types' export * from './ast/print-expr' export * from './ast/reduce-expr' diff --git a/packages/parse/src/vensim/impl/equation-reader.js b/packages/parse/src/vensim/impl/equation-reader.js index 857dd724..33febc99 100644 --- a/packages/parse/src/vensim/impl/equation-reader.js +++ b/packages/parse/src/vensim/impl/equation-reader.js @@ -2,7 +2,7 @@ import { ModelVisitor } from 'antlr4-vensim' -import { canonicalName } from '../../_shared/names' +import { canonicalId } from '../../_shared/canonical-id' import { createAntlrParser } from './antlr-parser' import { ExprReader } from './expr-reader' @@ -101,7 +101,7 @@ export class EquationReader extends ModelVisitor { visitLhs(ctx) { // Process the variable name const lhsVarName = ctx.Id().getText() - const lhsVarId = canonicalName(lhsVarName) + const lhsVarId = canonicalId(lhsVarName) // Process any subscripts that follow the variable name super.visitLhs(ctx) @@ -109,7 +109,7 @@ export class EquationReader extends ModelVisitor { const subscriptRefs = subscriptNames?.map(name => { return { subName: name, - subId: canonicalName(name) + subId: canonicalId(name) } }) @@ -119,7 +119,7 @@ export class EquationReader extends ModelVisitor { return subscriptSet.map(name => { return { subName: name, - subId: canonicalName(name) + subId: canonicalId(name) } }) }) diff --git a/packages/parse/src/vensim/impl/expr-reader.js b/packages/parse/src/vensim/impl/expr-reader.js index e50951d7..524ddf63 100644 --- a/packages/parse/src/vensim/impl/expr-reader.js +++ b/packages/parse/src/vensim/impl/expr-reader.js @@ -2,7 +2,7 @@ import { ModelLexer, ModelVisitor } from 'antlr4-vensim' -import { canonicalName, cFunctionName } from '../../_shared/names' +import { canonicalFunctionId, canonicalId } from '../../_shared/canonical-id' import { createAntlrParser } from './antlr-parser' @@ -82,7 +82,7 @@ export class ExprReader extends ModelVisitor { visitCall(ctx) { // Convert the function name from Vensim to C format const vensimFnName = ctx.Id().getText() - const fnId = cFunctionName(vensimFnName) + const fnId = canonicalFunctionId(vensimFnName) this.callStack.push({ fn: fnId, args: [] }) super.visitCall(ctx) const callInfo = this.callStack.pop() @@ -109,7 +109,7 @@ export class ExprReader extends ModelVisitor { visitVar(ctx) { // Convert the variable name from Vensim to C format const vensimVarName = ctx.Id().getText().trim() - const varId = canonicalName(vensimVarName) + const varId = canonicalId(vensimVarName) // Process the subscripts (if any) that follow the variable name this.subscripts = undefined @@ -118,7 +118,7 @@ export class ExprReader extends ModelVisitor { const subscriptRefs = subscriptNames?.map(name => { return { subName: name, - subId: canonicalName(name) + subId: canonicalId(name) } }) this.subscripts = undefined @@ -178,7 +178,7 @@ export class ExprReader extends ModelVisitor { visitLookupCall(ctx) { // Process the lookup variable name const lookupVarName = ctx.Id().getText() - const lookupVarId = canonicalName(lookupVarName) + const lookupVarId = canonicalId(lookupVarName) // Process any subscripts that follow the variable name if (ctx.subscriptList()) { @@ -188,7 +188,7 @@ export class ExprReader extends ModelVisitor { const subscriptRefs = subscriptNames?.map(name => { return { subName: name, - subId: canonicalName(name) + subId: canonicalId(name) } }) this.subscripts = undefined diff --git a/packages/parse/src/vensim/impl/subscript-range-reader.js b/packages/parse/src/vensim/impl/subscript-range-reader.js index 7d8d73c7..b078c71f 100644 --- a/packages/parse/src/vensim/impl/subscript-range-reader.js +++ b/packages/parse/src/vensim/impl/subscript-range-reader.js @@ -2,7 +2,7 @@ import { ModelParser, ModelVisitor } from 'antlr4-vensim' -import { canonicalName, cFunctionName } from '../../_shared/names' +import { canonicalFunctionId, canonicalId } from '../../_shared/canonical-id' import { createAntlrParser } from './antlr-parser' @@ -54,7 +54,7 @@ export class SubscriptRangeReader extends ModelVisitor { if (ids.length === 1) { // This is a regular subscript range definition, which begins with the dimension name const dimName = ids[0].getText() - const dimId = canonicalName(dimName) + const dimId = canonicalId(dimName) // Visit children to fill in the subscript range definition super.visitSubscriptRange(ctx) @@ -72,7 +72,7 @@ export class SubscriptRangeReader extends ModelVisitor { subscriptRefs: this.subscriptNames.map(subName => { return { subName, - subId: canonicalName(subName) + subId: canonicalId(subName) } }), subscriptMappings: this.subscriptMappings, @@ -81,9 +81,9 @@ export class SubscriptRangeReader extends ModelVisitor { } else if (ids.length === 2) { // This is a dimension alias (`DimA <-> DimB`) const dimName = ids[0].getText() - const dimId = canonicalName(dimName) + const dimId = canonicalId(dimName) const familyName = ids[1].getText() - const familyId = canonicalName(familyName) + const familyId = canonicalId(familyName) return { dimName, dimId, @@ -138,11 +138,11 @@ export class SubscriptRangeReader extends ModelVisitor { // Add the mappings this.subscriptMappings.push({ toDimName, - toDimId: canonicalName(toDimName), + toDimId: canonicalId(toDimName), subscriptRefs: this.mappedSubscriptNames.map(subName => { return { subName, - subId: canonicalName(subName) + subId: canonicalId(subName) } }) }) @@ -157,7 +157,7 @@ export class SubscriptRangeReader extends ModelVisitor { visitCall(ctx) { // A subscript range can have a `GET DIRECT SUBSCRIPT` call on the RHS const fnName = ctx.Id().getText() - const fnId = cFunctionName(fnName) + const fnId = canonicalFunctionId(fnName) if (fnId === '_GET_DIRECT_SUBSCRIPT') { super.visitCall(ctx) } else { diff --git a/packages/plugin-check/src/plugin.ts b/packages/plugin-check/src/plugin.ts index 97cb78bf..a641ed51 100644 --- a/packages/plugin-check/src/plugin.ts +++ b/packages/plugin-check/src/plugin.ts @@ -98,7 +98,7 @@ class CheckPlugin implements Plugin { await this.copyPreviousBundle(context.config) } context.log('info', 'Generating model check bundle...') - await this.genCurrentBundle(context.config, modelSpec) + await this.genCurrentBundle(context, modelSpec) } // For production builds (and for the initial build in local development mode), @@ -137,9 +137,8 @@ class CheckPlugin implements Plugin { } } - private async genCurrentBundle(config: ResolvedConfig, modelSpec: ResolvedModelSpec): Promise { - const prepDir = config.prepDir - const viteConfig = await createViteConfigForBundle(prepDir, modelSpec) + private async genCurrentBundle(context: BuildContext, modelSpec: ResolvedModelSpec): Promise { + const viteConfig = await createViteConfigForBundle(context, modelSpec) await build(viteConfig) } diff --git a/packages/plugin-check/src/var-names.ts b/packages/plugin-check/src/var-names.ts deleted file mode 100644 index a27a5527..00000000 --- a/packages/plugin-check/src/var-names.ts +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2022 Climate Interactive / New Venture Fund - -/** - * Helper function that converts a Vensim variable or subscript name - * into a valid C identifier as used by SDE. - * TODO: Import helper function from `compile` package instead - */ -function sdeNameForVensimName(name: string): string { - return ( - '_' + - name - .trim() - .replace(/"/g, '_') - .replace(/\s+!$/g, '!') - .replace(/\s/g, '_') - .replace(/,/g, '_') - .replace(/-/g, '_') - .replace(/\./g, '_') - .replace(/\$/g, '_') - .replace(/'/g, '_') - .replace(/&/g, '_') - .replace(/%/g, '_') - .replace(/\//g, '_') - .replace(/\|/g, '_') - .toLowerCase() - ) -} - -/** - * Helper function that converts a Vensim variable name (possibly containing - * subscripts) into a valid C identifier as used by SDE. - * TODO: Import helper function from `compile` package instead - */ -export function sdeNameForVensimVarName(varName: string): string { - const m = varName.match(/([^[]+)(?:\[([^\]]+)\])?/) - if (!m) { - throw new Error(`Invalid Vensim name: ${varName}`) - } - let id = sdeNameForVensimName(m[1]) - if (m[2]) { - const subscripts = m[2].split(',').map(x => sdeNameForVensimName(x)) - id += `[${subscripts.join(',')}]` - } - - return id -} diff --git a/packages/plugin-check/src/vite-config-for-bundle.ts b/packages/plugin-check/src/vite-config-for-bundle.ts index 726daa9e..a4eb7d1a 100644 --- a/packages/plugin-check/src/vite-config-for-bundle.ts +++ b/packages/plugin-check/src/vite-config-for-bundle.ts @@ -7,9 +7,7 @@ import { fileURLToPath } from 'url' import type { InlineConfig, ResolvedConfig, Plugin as VitePlugin } from 'vite' import { nodeResolve } from '@rollup/plugin-node-resolve' -import type { ResolvedModelSpec } from '@sdeverywhere/build' - -import { sdeNameForVensimVarName } from './var-names' +import type { BuildContext, ResolvedModelSpec } from '@sdeverywhere/build' const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) @@ -24,7 +22,7 @@ const __dirname = dirname(__filename) * TODO: This could be simplified by using `vite-plugin-virtual` but that * doesn't seem to be working correctly in an ESM setting */ -function injectModelSpec(prepDir: string, modelSpec: ResolvedModelSpec): VitePlugin { +function injectModelSpec(context: BuildContext, modelSpec: ResolvedModelSpec): VitePlugin { // Include the SDE variable ID with each spec const inputSpecs = [] for (const modelInputSpec of modelSpec.inputs) { @@ -52,7 +50,7 @@ function injectModelSpec(prepDir: string, modelSpec: ResolvedModelSpec): VitePlu // a stable `inputId` for each row in the `inputs.csv`, and that is the most // common way to configure a `ModelSpec`, so it will be uncommon for `inputId` // to be undefined here. - const varId = sdeNameForVensimVarName(modelInputSpec.varName) + const varId = context.canonicalVarId(modelInputSpec.varName) const inputId = modelInputSpec.inputId || varId inputSpecs.push({ inputId, @@ -62,12 +60,13 @@ function injectModelSpec(prepDir: string, modelSpec: ResolvedModelSpec): VitePlu } const outputSpecs = modelSpec.outputs.map(o => { return { - varId: sdeNameForVensimVarName(o.varName), + varId: context.canonicalVarId(o.varName), ...o } }) function stagedFileSize(filename: string): number { + const prepDir = context.config.prepDir const path = joinPath(prepDir, 'staged', 'model', filename) if (existsSync(path)) { return statSync(path).size @@ -161,12 +160,16 @@ function overrideViteResolvePlugin(viteConfig: ResolvedConfig) { } } -export async function createViteConfigForBundle(prepDir: string, modelSpec: ResolvedModelSpec): Promise { +export async function createViteConfigForBundle( + context: BuildContext, + modelSpec: ResolvedModelSpec +): Promise { // Use `template-bundle` as the root directory for the bundle project const root = resolvePath(__dirname, '..', 'template-bundle') // Calculate output directory relative to the template root // TODO: For now we write it to `prepDir`; make this configurable? + const prepDir = context.config.prepDir const outDir = relative(root, prepDir) // Use the model worker from the staged directory @@ -228,7 +231,7 @@ export async function createViteConfigForBundle(prepDir: string, modelSpec: Reso plugins: [ // Use a virtual module plugin to inject the model spec values - injectModelSpec(prepDir, modelSpec), + injectModelSpec(context, modelSpec), // XXX: Install a wrapper around the built-in `vite:resolve` plugin so that we can // override the default resolver behavior that tries to resolve the `browser` section diff --git a/packages/plugin-config/src/context.ts b/packages/plugin-config/src/context.ts index 2dee9c80..045dd7fb 100644 --- a/packages/plugin-config/src/context.ts +++ b/packages/plugin-config/src/context.ts @@ -7,10 +7,8 @@ import { parse as parseCsv } from 'csv-parse/sync' import type { BuildContext, InputSpec, LogLevel, OutputSpec } from '@sdeverywhere/build' -import type { HexColor } from './spec-types' +import type { HexColor, InputVarId, OutputVarId } from './spec-types' import { Strings } from './strings' -import type { InputVarId, OutputVarId } from './var-names' -import { sdeNameForVensimVarName } from './var-names' export type CsvRow = { [key: string]: string } export type ColorId = string @@ -56,6 +54,18 @@ export class ConfigContext { this.buildContext.log(level, msg) } + /** + * Format a (subscripted or non-subscripted) model variable name into a canonical + * identifier (with special characters converted to underscore, and subscript/dimension + * parts separated by commas). + * + * @param name The name of the variable in the source model, e.g., `Variable name[DimA, B2]`. + * @returns The canonical identifier for the given name, e.g., `_variable_name[_dima,_b2]`. + */ + canonicalVarId(name: string): string { + return this.buildContext.canonicalVarId(name) + } + /** * Write a file to the staged directory. * @@ -85,7 +95,7 @@ export class ConfigContext { ): void { // We use the C name as the key to avoid redundant entries in cases where // the csv file refers to variables with different capitalization - const varId = sdeNameForVensimVarName(inputVarName) + const varId = this.buildContext.canonicalVarId(inputVarName) if (this.inputSpecs.get(varId)) { // Fail if the variable was already added (there should only be one spec // per input variable) @@ -103,7 +113,7 @@ export class ConfigContext { addOutputVariable(outputVarName: string): void { // We use the C name as the key to avoid redundant entries in cases where // the csv file refers to variables with different capitalization - const varId = sdeNameForVensimVarName(outputVarName) + const varId = this.buildContext.canonicalVarId(outputVarName) this.outputVarNames.set(varId, outputVarName) } diff --git a/packages/plugin-config/src/gen-graphs.ts b/packages/plugin-config/src/gen-graphs.ts index f9af954b..abd61951 100644 --- a/packages/plugin-config/src/gen-graphs.ts +++ b/packages/plugin-config/src/gen-graphs.ts @@ -16,7 +16,6 @@ import type { UnitSystem } from './spec-types' import { genStringKey, htmlToUtf8 } from './strings' -import { sdeNameForVensimVarName } from './var-names' /** * Convert the `config/graphs.csv` file to config specs that can be used in @@ -181,7 +180,7 @@ function graphSpecFromCsv(g: CsvRow, context: ConfigContext): GraphSpec | undefi return } - const varId = sdeNameForVensimVarName(varName) + const varId = context.canonicalVarId(varName) const externalSourceName = overrides?.sourceName || optionalString(g[plotKey('source')]) const datasetLabel = optionalString(g[plotKey('label')]) let labelKey: StringKey diff --git a/packages/plugin-config/src/gen-inputs.ts b/packages/plugin-config/src/gen-inputs.ts index 9d723400..451caf89 100644 --- a/packages/plugin-config/src/gen-inputs.ts +++ b/packages/plugin-config/src/gen-inputs.ts @@ -4,7 +4,6 @@ import type { ConfigContext, CsvRow } from './context' import { optionalNumber, optionalString } from './read-config' import type { InputId, InputSpec, SliderSpec, StringKey, SwitchSpec } from './spec-types' import { genStringKey, htmlToUtf8 } from './strings' -import { sdeNameForVensimVarName } from './var-names' // TODO: For now, all strings use the same "layout" specifier; this could be customized // to provide a "maximum length" hint for a group of strings to the translation tool @@ -116,7 +115,7 @@ function inputSpecFromCsv(r: CsvRow, context: ConfigContext): InputSpec | undefi // Converts a slider row in `inputs.csv` to a `SliderSpec` function sliderSpecFromCsv(): SliderSpec { const varName = requiredString('varname') - const varId = sdeNameForVensimVarName(varName) + const varId = context.canonicalVarId(varName) const defaultValue = requiredNumber('slider/switch default') const minValue = requiredNumber('slider min') @@ -166,7 +165,7 @@ function inputSpecFromCsv(r: CsvRow, context: ConfigContext): InputSpec | undefi // Converts a switch row in `inputs.csv` to a `SwitchSpec` function switchSpecFromCsv(): SwitchSpec { const varName = requiredString('varname') - const varId = sdeNameForVensimVarName(varName) + const varId = context.canonicalVarId(varName) const onValue = requiredNumber('enabled value') const offValue = requiredNumber('disabled value') diff --git a/packages/plugin-config/src/gen-model-spec.ts b/packages/plugin-config/src/gen-model-spec.ts index e2c1776e..d781357c 100644 --- a/packages/plugin-config/src/gen-model-spec.ts +++ b/packages/plugin-config/src/gen-model-spec.ts @@ -1,15 +1,14 @@ // Copyright (c) 2022 Climate Interactive / New Venture Fund import type { ConfigContext } from './context' -import { sdeNameForVensimVarName } from './var-names' /** * Write the `model-spec.ts` file used by the core package to initialize the model. */ export function writeModelSpec(context: ConfigContext, dstDir: string): void { // Create ordered arrays of inputs and outputs - const inputVarIds = context.getOrderedInputs().map(i => sdeNameForVensimVarName(i.varName)) - const outputVarIds = context.getOrderedOutputs().map(o => sdeNameForVensimVarName(o.varName)) + const inputVarIds = context.getOrderedInputs().map(i => context.canonicalVarId(i.varName)) + const outputVarIds = context.getOrderedOutputs().map(o => context.canonicalVarId(o.varName)) // Generate the `model-spec.ts` file let tsContent = '' diff --git a/packages/plugin-config/src/var-names.ts b/packages/plugin-config/src/var-names.ts deleted file mode 100644 index 711f34a2..00000000 --- a/packages/plugin-config/src/var-names.ts +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2022 Climate Interactive / New Venture Fund - -export type InputVarId = string -export type OutputVarId = string - -/** - * Helper function that converts a Vensim variable or subscript name - * into a valid C identifier as used by SDE. - * TODO: Import helper function from `sdeverywhere` package instead - */ -function sdeNameForVensimName(name: string): string { - return ( - '_' + - name - .trim() - .replace(/"/g, '_') - .replace(/\s+!$/g, '!') - .replace(/\s/g, '_') - .replace(/,/g, '_') - .replace(/-/g, '_') - .replace(/\./g, '_') - .replace(/\$/g, '_') - .replace(/'/g, '_') - .replace(/&/g, '_') - .replace(/%/g, '_') - .replace(/\//g, '_') - .replace(/\|/g, '_') - .toLowerCase() - ) -} - -/** - * Helper function that converts a Vensim variable name (possibly containing - * subscripts) into a valid C identifier as used by SDE. - * TODO: Import helper function from `sdeverywhere` package instead - */ -export function sdeNameForVensimVarName(varName: string): string { - const m = varName.match(/([^[]+)(?:\[([^\]]+)\])?/) - if (!m) { - throw new Error(`Invalid Vensim name: ${varName}`) - } - let id = sdeNameForVensimName(m[1]) - if (m[2]) { - const subscripts = m[2].split(',').map(x => sdeNameForVensimName(x)) - id += `[${subscripts.join(',')}]` - } - - return id -} diff --git a/packages/plugin-wasm/src/plugin.ts b/packages/plugin-wasm/src/plugin.ts index 689c6380..a89319f0 100644 --- a/packages/plugin-wasm/src/plugin.ts +++ b/packages/plugin-wasm/src/plugin.ts @@ -10,7 +10,6 @@ import { findUp } from 'find-up' import type { BuildContext, ResolvedModelSpec, Plugin } from '@sdeverywhere/build' import type { WasmPluginOptions } from './options' -import { sdeNameForVensimVarName } from './var-names' export function wasmPlugin(options?: WasmPluginOptions): Plugin { return new WasmPlugin(options) @@ -23,11 +22,11 @@ class WasmPlugin implements Plugin { constructor(private readonly options?: WasmPluginOptions) {} - async preGenerate(_context: BuildContext, modelSpec: ResolvedModelSpec): Promise { + async preGenerate(context: BuildContext, modelSpec: ResolvedModelSpec): Promise { // Save some properties for later processing. This is a workaround for the fact // that `modelSpec` is not passed to `postGenerateCode`, so we need to capture // these values here. - this.outputVarIds = modelSpec.outputs.map(o => sdeNameForVensimVarName(o.varName)) + this.outputVarIds = modelSpec.outputs.map(o => context.canonicalVarId(o.varName)) this.bundleListing = modelSpec.bundleListing } diff --git a/packages/plugin-wasm/src/var-names.ts b/packages/plugin-wasm/src/var-names.ts deleted file mode 100644 index b4bb649e..00000000 --- a/packages/plugin-wasm/src/var-names.ts +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2024 Climate Interactive / New Venture Fund - -/** - * Helper function that converts a Vensim variable or subscript name - * into a valid C identifier as used by SDE. - * TODO: Import helper function from `compile` package instead - */ -function sdeNameForVensimName(name: string): string { - return ( - '_' + - name - .trim() - .replace(/"/g, '_') - .replace(/\s+!$/g, '!') - .replace(/\s/g, '_') - .replace(/,/g, '_') - .replace(/-/g, '_') - .replace(/\./g, '_') - .replace(/\$/g, '_') - .replace(/'/g, '_') - .replace(/&/g, '_') - .replace(/%/g, '_') - .replace(/\//g, '_') - .replace(/\|/g, '_') - .toLowerCase() - ) -} - -/** - * Helper function that converts a Vensim variable name (possibly containing - * subscripts) into a valid C identifier as used by SDE. - * TODO: Import helper function from `compile` package instead - */ -export function sdeNameForVensimVarName(varName: string): string { - const m = varName.match(/([^[]+)(?:\[([^\]]+)\])?/) - if (!m) { - throw new Error(`Invalid Vensim name: ${varName}`) - } - let id = sdeNameForVensimName(m[1]) - if (m[2]) { - const subscripts = m[2].split(',').map(x => sdeNameForVensimName(x)) - id += `[${subscripts.join(',')}]` - } - - return id -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9ad9ac74..fd3c94b6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -192,6 +192,9 @@ importers: packages/build: dependencies: + '@sdeverywhere/parse': + specifier: ^0.1.1 + version: link:../parse chokidar: specifier: ^3.5.3 version: 3.5.3 @@ -344,9 +347,6 @@ importers: ramda: specifier: ^0.27.0 version: 0.27.2 - split-string: - specifier: ^6.0.0 - version: 6.1.0 xlsx: specifier: https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz version: '@cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz'