From 697e94397ca00b33d2e6216d63f4cfbc2f160fa0 Mon Sep 17 00:00:00 2001 From: Chris Campbell Date: Fri, 16 Aug 2024 18:34:11 -0700 Subject: [PATCH] fix: change encoding of variable and lookup indices to allow for arbitrary number of subscripts (#507) Fixes #506 --- packages/cli/src/c/model.c | 34 +-- packages/cli/src/c/sde.h | 3 +- packages/compile/src/generate/gen-code-c.js | 12 +- .../compile/src/generate/gen-code-c.spec.ts | 14 +- packages/runtime/src/_shared/types.ts | 2 +- .../runtime/src/_shared/var-indices.spec.ts | 123 +++++++++ packages/runtime/src/_shared/var-indices.ts | 242 ++++++++++++++++-- packages/runtime/src/js-model/js-model.ts | 38 ++- .../synchronous-model-runner.spec.ts | 10 +- .../buffered-run-model-params.spec.ts | 69 +++-- .../buffered-run-model-params.ts | 148 +---------- .../referenced-run-model-params.spec.ts | 57 ++++- .../referenced-run-model-params.ts | 9 +- 13 files changed, 498 insertions(+), 263 deletions(-) create mode 100644 packages/runtime/src/_shared/var-indices.spec.ts diff --git a/packages/cli/src/c/model.c b/packages/cli/src/c/model.c index 1854fd23..7c48a785 100644 --- a/packages/cli/src/c/model.c +++ b/packages/cli/src/c/model.c @@ -8,14 +8,6 @@ struct timespec startTime, finishTime; #endif -// For each output variable specified in the indices buffer, there -// are 4 index values: -// varIndex -// subIndex0 -// subIndex1 -// subIndex2 -#define INDICES_PER_OUTPUT 4 - // The special _time variable is not included in .mdl files. double _time; @@ -149,22 +141,20 @@ void run() { } outputVarIndex = 0; if (outputIndexBuffer != NULL) { - // Store the outputs as specified in the current output index buffer. This - // iterates over the output indices buffer until we reach the first zero index. - size_t i = 0; - while (true) { - size_t indexBufferOffset = i * INDICES_PER_OUTPUT; - size_t varIndex = (size_t)outputIndexBuffer[indexBufferOffset]; - if (varIndex > 0) { - size_t subIndex0 = (size_t)outputIndexBuffer[indexBufferOffset + 1]; - size_t subIndex1 = (size_t)outputIndexBuffer[indexBufferOffset + 2]; - size_t subIndex2 = (size_t)outputIndexBuffer[indexBufferOffset + 3]; - storeOutput(varIndex, subIndex0, subIndex1, subIndex2); + // Store the outputs as specified in the current output index buffer + size_t indexBufferOffset = 0; + size_t outputCount = (size_t)outputIndexBuffer[indexBufferOffset++]; + for (size_t i = 0; i < outputCount; i++) { + size_t varIndex = (size_t)outputIndexBuffer[indexBufferOffset++]; + size_t subCount = (size_t)outputIndexBuffer[indexBufferOffset++]; + size_t* subIndices; + if (subCount > 0) { + subIndices = (size_t*)(outputIndexBuffer + indexBufferOffset); } else { - // Stop when we reach the first zero index - break; + subIndices = NULL; } - i++; + indexBufferOffset += subCount; + storeOutput(varIndex, subIndices); } } else { // Store the normal outputs diff --git a/packages/cli/src/c/sde.h b/packages/cli/src/c/sde.h index ae96e45e..4d3e583c 100644 --- a/packages/cli/src/c/sde.h +++ b/packages/cli/src/c/sde.h @@ -66,10 +66,11 @@ void initConstants(void); void initLevels(void); void setInputs(const char* inputData); void setInputsFromBuffer(double *inputData); +void setLookup(size_t varIndex, size_t* subIndices, double* points, size_t numPoints); void evalAux(void); void evalLevels(void); void storeOutputData(void); -void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t subIndex2); +void storeOutput(size_t varIndex, size_t* subIndices); const char* getHeader(void); #ifdef __cplusplus diff --git a/packages/compile/src/generate/gen-code-c.js b/packages/compile/src/generate/gen-code-c.js index 451b4564..5a3eb062 100644 --- a/packages/compile/src/generate/gen-code-c.js +++ b/packages/compile/src/generate/gen-code-c.js @@ -225,7 +225,7 @@ void storeOutputData() { ${specOutputSection(outputVarIds)} } -void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t subIndex2) { +void storeOutput(size_t varIndex, size_t* subIndices) { ${storeOutputBody} } ` @@ -396,14 +396,8 @@ ${section(chunk)} }) const code = R.map(info => { let varAccess = info.varName - if (info.subscriptCount > 0) { - varAccess += '[subIndex0]' - } - if (info.subscriptCount > 1) { - varAccess += '[subIndex1]' - } - if (info.subscriptCount > 2) { - varAccess += '[subIndex2]' + for (let i = 0; i < info.subscriptCount; i++) { + varAccess += `[subIndices[${i}]]` } return `\ case ${info.varIndex}: diff --git a/packages/compile/src/generate/gen-code-c.spec.ts b/packages/compile/src/generate/gen-code-c.spec.ts index 28b3aa70..61f7f7f2 100644 --- a/packages/compile/src/generate/gen-code-c.spec.ts +++ b/packages/compile/src/generate/gen-code-c.spec.ts @@ -312,7 +312,7 @@ void storeOutputData() { outputVar(_w); } -void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t subIndex2) { +void storeOutput(size_t varIndex, size_t* subIndices) { switch (varIndex) { case 1: outputVar(_final_time); @@ -330,10 +330,10 @@ void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t sub outputVar(_input); break; case 10: - outputVar(_a[subIndex0]); + outputVar(_a[subIndices[0]]); break; case 11: - outputVar(_b[subIndex0][subIndex1]); + outputVar(_b[subIndices[0]][subIndices[1]]); break; case 12: outputVar(_c); @@ -345,7 +345,7 @@ void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t sub outputVar(_w); break; case 15: - outputVar(_d[subIndex0]); + outputVar(_d[subIndices[0]]); break; case 16: outputVar(_y); @@ -446,7 +446,7 @@ void setLookup(size_t varIndex, size_t* subIndices, double* points, size_t numPo outputVarNames: ['y'] }) expect(code).toMatch(`\ -void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t subIndex2) { +void storeOutput(size_t varIndex, size_t* subIndices) { fprintf(stderr, "The storeOutput function was not enabled for the generated model. Set the customOutputs property in the spec/config file to allow for capturing arbitrary variables at runtime.\\n"); }`) }) @@ -469,10 +469,10 @@ void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t sub customOutputs: ['u[A1]', 'x'] }) expect(code).toMatch(`\ -void storeOutput(size_t varIndex, size_t subIndex0, size_t subIndex1, size_t subIndex2) { +void storeOutput(size_t varIndex, size_t* subIndices) { switch (varIndex) { case 5: - outputVar(_u[subIndex0]); + outputVar(_u[subIndices[0]]); break; case 6: outputVar(_x); diff --git a/packages/runtime/src/_shared/types.ts b/packages/runtime/src/_shared/types.ts index bd3e36d8..08520b79 100644 --- a/packages/runtime/src/_shared/types.ts +++ b/packages/runtime/src/_shared/types.ts @@ -22,7 +22,7 @@ export interface VarSpec { /** The variable index as used in the generated C/JS code. */ varIndex: number /** The subscript index values as used in the generated C/JS code. */ - subscriptIndices?: number[] + subscriptIndices?: number[] | Int32Array } /** diff --git a/packages/runtime/src/_shared/var-indices.spec.ts b/packages/runtime/src/_shared/var-indices.spec.ts new file mode 100644 index 00000000..34ba8cce --- /dev/null +++ b/packages/runtime/src/_shared/var-indices.spec.ts @@ -0,0 +1,123 @@ +// Copyright (c) 2024 Climate Interactive / New Venture Fund + +import { describe, expect, it } from 'vitest' + +import { createLookupDef, type LookupDef } from './lookup-def' +import type { VarSpec } from './types' +import { + decodeLookups, + encodeLookups, + encodeVarIndices, + getEncodedLookupBufferLengths, + getEncodedVarIndicesLength +} from './var-indices' + +const varSpecs: VarSpec[] = [ + { varIndex: 1 }, + { varIndex: 2 }, + { varIndex: 3, subscriptIndices: [1, 2, 3, 4] }, + { varIndex: 4, subscriptIndices: [1, 2] } +] + +describe('getEncodedVarIndicesLength', () => { + it('should return the correct length', () => { + expect(getEncodedVarIndicesLength(varSpecs)).toBe(15) + }) +}) + +describe('encodeVarIndices', () => { + it('should encode the correct values', () => { + const array = new Int32Array(20) + encodeVarIndices(varSpecs, array) + expect(array).toEqual( + new Int32Array([ + 4, // variable count + + 1, // var0 index + 0, // var0 subscript count + + 2, // var1 index + 0, // var1 subscript count + + 3, // var2 index + 4, // var2 subscript count + 1, // var2 sub0 index + 2, // var2 sub1 index + 3, // var2 sub2 index + 4, // var2 sub3 index + + 4, // var3 index + 2, // var2 subscript count + 1, // var3 sub0 index + 2, // var3 sub1 index + + // zero padding + 0, + 0, + 0, + 0, + 0 + ]) + ) + }) +}) + +const p = (x: number, y: number) => ({ x, y }) +const lookupDefs: LookupDef[] = [ + createLookupDef({ varSpec: { varIndex: 1 } }, [p(0, 0), p(1, 1)]), + createLookupDef({ varSpec: { varIndex: 2, subscriptIndices: [1, 2] } }, [p(0, 0), p(1, 1)]) +] + +describe('getEncodedLookupBufferLengths', () => { + it('should return the correct length', () => { + const { lookupIndicesLength, lookupsLength } = getEncodedLookupBufferLengths(lookupDefs) + expect(lookupIndicesLength).toBe(11) + expect(lookupsLength).toBe(8) + }) +}) + +describe('encodeLookups and decodeLookups', () => { + it('should encode and decode the correct values', () => { + const lookupIndices = new Int32Array(13) + const lookupValues = new Float64Array(10) + encodeLookups(lookupDefs, lookupIndices, lookupValues) + + expect(lookupIndices).toEqual( + new Int32Array([ + 2, // variable count + + 1, // var0 index + 0, // var0 subscript count + 0, // var0 data offset + 4, // var0 data length + + 2, // var1 index + 2, // var1 subscript count + 1, // var1 sub0 index + 2, // var1 sub1 index + 4, // var1 data offset + 4, // var1 data length + + // zero padding + 0, + 0 + ]) + ) + + expect(lookupValues).toEqual( + new Float64Array([ + // var0 data + 0, 0, 1, 1, + + // var1 data + 0, 0, 1, 1, + + // zero padding + 0, 0 + ]) + ) + + const decodedLookupDefs = decodeLookups(lookupIndices, lookupValues) + expect(decodedLookupDefs).toEqual(lookupDefs) + }) +}) diff --git a/packages/runtime/src/_shared/var-indices.ts b/packages/runtime/src/_shared/var-indices.ts index 59806dc9..e182c52c 100644 --- a/packages/runtime/src/_shared/var-indices.ts +++ b/packages/runtime/src/_shared/var-indices.ts @@ -1,38 +1,238 @@ // Copyright (c) 2024 Climate Interactive / New Venture Fund +import type { LookupDef } from './lookup-def' import type { VarSpec } from './types' /** - * For each variable specified in an indices buffer, there are 4 index values: - * varIndex - * subIndex0 - * subIndex1 - * subIndex2 - * NOTE: This value needs to match `INDICES_PER_VARIABLE` as defined in SDE's `model.c`. - * @hidden This is not part of the public API. + * Return the length of the array that is required to store the variable + * indices for the given `VarSpec` instances. + * + * @hidden This is not part of the public API; it is exposed here for use by + * the synchronous and asynchronous model runner implementations. + * + * @param varSpecs The `VarSpec` instances to encode. */ -export const indicesPerVariable = 4 +export function getEncodedVarIndicesLength(varSpecs: VarSpec[]): number { + // The indices buffer has the following format: + // variable count + // varN index + // varN subscript count + // varN sub1 index + // varN sub2 index + // ... + // varN subM index + // ... (repeat for each var spec) + + // Start with one element for the total variable count + let length = 1 + + for (const varSpec of varSpecs) { + // Include one element for the variable index and one for the subscript count + length += 2 + + // Include one element for each subscript + const subCount = varSpec.subscriptIndices?.length || 0 + length += subCount + } + + return length +} /** + * Encode variable indices to the given array. + * * @hidden This is not part of the public API; it is exposed here for use by * the synchronous and asynchronous model runner implementations. + * + * @param varSpecs The `VarSpec` instances to encode. */ -export function updateVarIndices(indicesArray: Int32Array, varSpecs: VarSpec[]): void { - if (indicesArray.length < varSpecs.length * indicesPerVariable) { - throw new Error('Length of indicesArray must be large enough to accommodate the given varSpecs') - } - - // Write the indices to the buffer +export function encodeVarIndices(varSpecs: VarSpec[], indicesArray: Int32Array): void { + // Write the variable count let offset = 0 + indicesArray[offset++] = varSpecs.length + + // Write the indices for each variable for (const varSpec of varSpecs) { + // Write the variable index + indicesArray[offset++] = varSpec.varIndex + + // Write the subscript count + const subs = varSpec.subscriptIndices + const subCount = subs?.length || 0 + indicesArray[offset++] = subCount + + // Write the subscript indices + for (let i = 0; i < subCount; i++) { + indicesArray[offset++] = subs[i] + } + } +} + +/** + * Return the lengths of the arrays that are required to store the lookup data + * and indices for the given `LookupDef` instances. + * + * @hidden This is not part of the public API; it is exposed here for use by + * the synchronous and asynchronous model runner implementations. + * + * @param lookupDefs The `LookupDef` instances to encode. + */ +export function getEncodedLookupBufferLengths(lookupDefs: LookupDef[]): { + lookupIndicesLength: number + lookupsLength: number +} { + // The lookups buffer includes all data points for the provided lookup overrides + // (added sequentially, with no padding between datasets). The lookup indices + // buffer has the following format: + // lookup count + // lookupN var index + // lookupN subscript count + // lookupN sub1 index + // lookupN sub2 index + // ... + // lookupN subM index + // lookupN data offset (relative to the start of the lookups buffer, in float64 elements) + // lookupN data length (in float64 elements) + // ... (repeat for each lookup) + + // Start with one element for the total lookup variable count + let lookupIndicesLength = 1 + let lookupsLength = 0 + + for (const lookupDef of lookupDefs) { + // Ensure that the var spec has already been resolved + const varSpec = lookupDef.varRef.varSpec + if (varSpec === undefined) { + throw new Error('Cannot compute lookup buffer lengths until all lookup var specs are defined') + } + + // Include one element for the variable index and one for the subscript count + lookupIndicesLength += 2 + + // Include one element for each subscript const subCount = varSpec.subscriptIndices?.length || 0 - indicesArray[offset + 0] = varSpec.varIndex - indicesArray[offset + 1] = subCount > 0 ? varSpec.subscriptIndices[0] : 0 - indicesArray[offset + 2] = subCount > 1 ? varSpec.subscriptIndices[1] : 0 - indicesArray[offset + 3] = subCount > 2 ? varSpec.subscriptIndices[2] : 0 - offset += indicesPerVariable + lookupIndicesLength += subCount + + // Include one element for the data offset and one element for the data length + lookupIndicesLength += 2 + + // Add the length of the lookup points array + lookupsLength += lookupDef.points.length + } + + return { + lookupIndicesLength, + lookupsLength + } +} + +/** + * Encode lookup data and indices to the given arrays. + * + * @hidden This is not part of the public API; it is exposed here for use by + * the synchronous and asynchronous model runner implementations. + * + * @param lookupDefs The `LookupDef` instances to encode. + * @param lookupIndicesArray The view on the lookup indices buffer. + * @param lookupsArray The view on the lookup data buffer. This can be undefined in + * the case where the data for the lookup(s) is empty. + */ +export function encodeLookups( + lookupDefs: LookupDef[], + lookupIndicesArray: Int32Array, + lookupsArray: Float64Array | undefined +): void { + // Write the lookup variable count + let li = 0 + lookupIndicesArray[li++] = lookupDefs.length + + // Write the indices and data for each lookup + let lookupDataOffset = 0 + for (const lookupDef of lookupDefs) { + // Write the lookup variable index + const varSpec = lookupDef.varRef.varSpec + lookupIndicesArray[li++] = varSpec.varIndex + + // Write the subscript count + const subs = varSpec.subscriptIndices + const subCount = subs?.length || 0 + lookupIndicesArray[li++] = subCount + + // Write the subscript indices + for (let i = 0; i < subCount; i++) { + lookupIndicesArray[li++] = subs[i] + } + + // Write the lookup data offset and length for this variable + lookupIndicesArray[li++] = lookupDataOffset + lookupIndicesArray[li++] = lookupDef.points.length + + // Write the lookup data. Note that `lookupsView` can be undefined in the case + // where the lookup data is empty. + lookupsArray?.set(lookupDef.points, lookupDataOffset) + lookupDataOffset += lookupDef.points.length + } +} + +/** + * Decode lookup data and indices from the given buffer views and return the + * reconstructed `LookupDef` instances. + * + * @hidden This is not part of the public API; it is exposed here for use by + * the synchronous and asynchronous model runner implementations. + * + * @param lookupIndicesArray The view on the lookup indices buffer. + * @param lookupsArray The view on the lookup data buffer. This can be undefined in + * the case where the data for the lookup(s) is empty. + */ +export function decodeLookups(lookupIndicesArray: Int32Array, lookupsArray: Float64Array | undefined): LookupDef[] { + const lookupDefs: LookupDef[] = [] + let li = 0 + + // Read the lookup variable count + const lookupCount = lookupIndicesArray[li++] + + // Read the metadata for each variable from the lookup indices buffer + for (let i = 0; i < lookupCount; i++) { + // Read the lookup variable index + const varIndex = lookupIndicesArray[li++] + + // Read the subscript count + const subCount = lookupIndicesArray[li++] + + // Read the subscript indices + const subscriptIndices: number[] = subCount > 0 ? Array(subCount) : undefined + for (let subIndex = 0; subIndex < subCount; subIndex++) { + subscriptIndices[subIndex] = lookupIndicesArray[li++] + } + + // Read the lookup data offset and length for this variable + const lookupDataOffset = lookupIndicesArray[li++] + const lookupDataLength = lookupIndicesArray[li++] + + // Create a `VarSpec` for the variable + const varSpec: VarSpec = { + varIndex, + subscriptIndices + } + + // Copy the data from the lookup data buffer. Note that `lookupsArray` can be undefined + // in the case where the lookup data is empty. + // TODO: We can use `subarray` here instead of `slice` and let the model implementations + // copy the data if needed on their side + let points: Float64Array + if (lookupsArray) { + points = lookupsArray.slice(lookupDataOffset, lookupDataOffset + lookupDataLength) + } else { + points = new Float64Array(0) + } + lookupDefs.push({ + varRef: { + varSpec + }, + points + }) } - // Fill the remainder of the buffer with zeros - indicesArray.fill(0, offset) + return lookupDefs } diff --git a/packages/runtime/src/js-model/js-model.ts b/packages/runtime/src/js-model/js-model.ts index 36bc1931..b8789561 100644 --- a/packages/runtime/src/js-model/js-model.ts +++ b/packages/runtime/src/js-model/js-model.ts @@ -1,6 +1,6 @@ // Copyright (c) 2024 Climate Interactive / New Venture Fund -import { indicesPerVariable, type LookupDef, type VarSpec } from '../_shared' +import { type LookupDef, type VarSpec } from '../_shared' import type { RunnableModel } from '../runnable-model' import { BaseRunnableModel } from '../runnable-model/base-runnable-model' @@ -183,28 +183,22 @@ function runJsModel( outputVarIndex++ } if (outputIndices !== undefined) { - // Store the outputs as specified in the current output indices buffer. This - // iterates over the output indices buffer until we reach the first zero index. - let i = 0 - // eslint-disable-next-line no-constant-condition - while (true) { - const indexBufferOffset = i * indicesPerVariable - const varIndex = outputIndices[indexBufferOffset] - if (varIndex > 0) { - const subscriptIndices: number[] = Array(3) - subscriptIndices[0] = outputIndices[indexBufferOffset + 1] - subscriptIndices[1] = outputIndices[indexBufferOffset + 2] - subscriptIndices[2] = outputIndices[indexBufferOffset + 3] - const varSpec: VarSpec = { - varIndex, - subscriptIndices - } - model.storeOutput(varSpec, storeValue) - } else { - // Stop when we reach the first zero index - break + // Store the outputs as specified in the current output indices buffer + let indexBufferOffset = 0 + const outputCount = outputIndices[indexBufferOffset++] + for (let i = 0; i < outputCount; i++) { + const varIndex = outputIndices[indexBufferOffset++] + const subCount = outputIndices[indexBufferOffset++] + let subscriptIndices: Int32Array + if (subCount > 0) { + subscriptIndices = outputIndices.subarray(indexBufferOffset, indexBufferOffset + subCount) + indexBufferOffset += subCount } - i++ + const varSpec: VarSpec = { + varIndex, + subscriptIndices + } + model.storeOutput(varSpec, storeValue) } } else { // Store the normal outputs diff --git a/packages/runtime/src/model-runner/synchronous-model-runner.spec.ts b/packages/runtime/src/model-runner/synchronous-model-runner.spec.ts index b95bff32..84ba456a 100644 --- a/packages/runtime/src/model-runner/synchronous-model-runner.spec.ts +++ b/packages/runtime/src/model-runner/synchronous-model-runner.spec.ts @@ -98,14 +98,14 @@ function createMockWasmModule(): MockWasmModule { // Verify output indices expect(outputIndices).toEqual( new Int32Array([ + // variable count + 3, // _x - 5, 0, 0, 0, + 5, 0, // _output_2 - 3, 0, 0, 0, + 3, 0, // _output_1 - 1, 0, 0, 0, - // (zero terminator) - 0, 0, 0, 0 + 1, 0 ]) ) // Store 3 values for each of the three variables diff --git a/packages/runtime/src/runnable-model/buffered-run-model-params.spec.ts b/packages/runtime/src/runnable-model/buffered-run-model-params.spec.ts index 54846ab4..b217fa65 100644 --- a/packages/runtime/src/runnable-model/buffered-run-model-params.spec.ts +++ b/packages/runtime/src/runnable-model/buffered-run-model-params.spec.ts @@ -10,6 +10,21 @@ import { ModelListing } from '../model-listing' const listingJson = ` { "dimensions": [ + { + "id": "_dima", + "subIds": [ + "_a1", + "_a2" + ] + }, + { + "id": "_dimb", + "subIds": [ + "_b1", + "_b2", + "_b3" + ] + } ], "variables": [ { @@ -27,6 +42,14 @@ const listingJson = ` { "id": "_y", "index": 4 + }, + { + "id": "_z", + "dimIds": [ + "_dima", + "_dimb" + ], + "index": 5 } ] } @@ -75,7 +98,7 @@ describe('BufferedRunModelParams', () => { const inputs = [1, 2, 3] const normalOutputs = new Outputs(['_x', '_y'], 2000, 2002, 1) - const implOutputs = listing.deriveOutputs(normalOutputs, ['_x', '_a', '_b']) + const implOutputs = listing.deriveOutputs(normalOutputs, ['_x', '_z[_a2,_b1]', '_b']) // Update using the normal outputs (which includes only 2 variables) const params1 = new BufferedRunModelParams() @@ -108,14 +131,14 @@ describe('BufferedRunModelParams', () => { expect(params1.getOutputsLength()).toEqual(9) expect(params1.getOutputIndices()).toEqual( new Int32Array([ + // variable count + 3, // _x - 3, 0, 0, 0, - // _a - 1, 0, 0, 0, + 3, 0, + // _z[_a2,_b1] + 5, 2, 1, 0, // _b - 2, 0, 0, 0, - // (zero terminator) - 0, 0, 0, 0 + 2, 0 ]) ) @@ -179,7 +202,7 @@ describe('BufferedRunModelParams', () => { const listing = new ModelListing(JSON.parse(listingJson)) const inputs = [1, 2, 3] const normalOutputs = new Outputs(['_x', '_y'], 2000, 2002, 1) - const implOutputs = listing.deriveOutputs(normalOutputs, ['_x', '_a', '_b']) + const implOutputs = listing.deriveOutputs(normalOutputs, ['_x', '_z[_a2,_b3]', '_z[_a1,_b1]', '_b']) const runnerParams = new BufferedRunModelParams() const workerParams = new BufferedRunModelParams() @@ -188,14 +211,16 @@ describe('BufferedRunModelParams', () => { workerParams.updateFromEncodedBuffer(runnerParams.getEncodedBuffer()) const expectedIndices = new Int32Array([ + // variable count + 4, // _x - 3, 0, 0, 0, - // _a - 1, 0, 0, 0, + 3, 0, + // _z[_a2,_b3] + 5, 2, 1, 2, + // _z[_a1,_b1] + 5, 2, 0, 0, // _b - 2, 0, 0, 0, - // (zero terminator) - 0, 0, 0, 0 + 2, 0 ]) let array: Int32Array @@ -214,18 +239,20 @@ describe('BufferedRunModelParams', () => { expect(array).toEqual(expectedIndices) // Verify case where existing array is large enough - array = new Int32Array(20).fill(6) + array = new Int32Array(17).fill(6) workerParams.copyOutputIndices(array, create) expect(array).toEqual( new Int32Array([ + // variable count + 4, // _x - 3, 0, 0, 0, - // _a - 1, 0, 0, 0, + 3, 0, + // _z[_a2,_b3] + 5, 2, 1, 2, + // _z[_a1,_b1] + 5, 2, 0, 0, // _b - 2, 0, 0, 0, - // (zero terminator) - 0, 0, 0, 0, + 2, 0, // (existing data) 6, 6, 6, 6 ]) diff --git a/packages/runtime/src/runnable-model/buffered-run-model-params.ts b/packages/runtime/src/runnable-model/buffered-run-model-params.ts index a14654ee..c5e28b8d 100644 --- a/packages/runtime/src/runnable-model/buffered-run-model-params.ts +++ b/packages/runtime/src/runnable-model/buffered-run-model-params.ts @@ -1,7 +1,13 @@ // Copyright (c) 2024 Climate Interactive / New Venture Fund -import { indicesPerVariable, updateVarIndices } from '../_shared' -import type { InputValue, LookupDef, Outputs, VarSpec } from '../_shared' +import { + decodeLookups, + encodeLookups, + encodeVarIndices, + getEncodedLookupBufferLengths, + getEncodedVarIndicesLength +} from '../_shared' +import type { InputValue, LookupDef, Outputs } from '../_shared' import type { ModelListing } from '../model-listing' import { resolveVarRef } from './resolve-var-ref' import type { RunModelOptions } from './run-model-options' @@ -189,7 +195,7 @@ export class BufferedRunModelParams implements RunModelParams { // Reconstruct the `LookupDef` instances using the data from the lookup data and // indices buffers - return decodeLookups(this.lookups.view, this.lookupIndices.view) + return decodeLookups(this.lookupIndices.view, this.lookups.view) } // from RunModelParams interface @@ -236,9 +242,8 @@ export class BufferedRunModelParams implements RunModelParams { let outputIndicesLengthInElements: number const outputVarSpecs = outputs.varSpecs if (outputVarSpecs !== undefined && outputVarSpecs.length > 0) { - // The output indices buffer needs to include N elements for each var spec plus one - // additional "zero" element as a terminator - outputIndicesLengthInElements = (outputVarSpecs.length + 1) * indicesPerVariable + // Compute the required length of the output indices buffer + outputIndicesLengthInElements = getEncodedVarIndicesLength(outputVarSpecs) } else { // Don't use the output indices buffer when output var specs are not provided outputIndicesLengthInElements = 0 @@ -345,12 +350,12 @@ export class BufferedRunModelParams implements RunModelParams { // Copy the the output indices into the internal buffer, if needed if (this.outputIndices.view) { - updateVarIndices(this.outputIndices.view, outputVarSpecs) + encodeVarIndices(outputVarSpecs, this.outputIndices.view) } // Copy the lookup data and indices into the internal buffers, if needed if (lookupIndicesLengthInElements > 0) { - encodeLookups(options.lookups, this.lookups.view, this.lookupIndices.view) + encodeLookups(options.lookups, this.lookupIndices.view, this.lookups.view) } } @@ -418,130 +423,3 @@ export class BufferedRunModelParams implements RunModelParams { this.lookupIndices.update(this.encoded, lookupIndicesOffsetInBytes, lookupIndicesLengthInElements) } } - -/** - * Return the lengths of the arrays that are required to store the lookup data and indices for - * the given `LookupDef` instances. - * - * @param lookupDefs The `LookupDef` instances to encode. - */ -function getEncodedLookupBufferLengths(lookupDefs: LookupDef[]): { - lookupsLength: number - lookupIndicesLength: number -} { - // The lookups buffer includes all data points for the provided lookup overrides - // (added sequentially, with no padding between datasets). The lookup indices - // buffer has the following format: - // lookup count - // lookupN var index - // lookupN sub0 index - // lookupN sub1 index - // lookupN sub2 index - // lookupN data offset (relative to the start of the lookups buffer, in float64 elements) - // lookupN data length (in float64 elements) - // ... (repeat for each lookup) - const numIndexElementsForTotalCount = 1 - // TODO: Update this once we support > 3 subscripts - const numIndexElementsPerLookup = 6 - let lookupsLength = 0 - let lookupIndicesLength = numIndexElementsForTotalCount - for (const lookupDef of lookupDefs) { - lookupsLength += lookupDef.points.length - lookupIndicesLength += numIndexElementsPerLookup - } - return { - lookupsLength, - lookupIndicesLength - } -} - -/** - * Encode lookup data and indices to the given buffer views. - * - * @param lookupDefs The `LookupDef` instances to encode. - * @param lookupsView The view on the lookup data buffer. This can be undefined in - * the case where the data for the lookup(s) is empty. - * @param lookupIndicesView The view on the lookup indices buffer. - */ -function encodeLookups( - lookupDefs: LookupDef[], - lookupsView: Float64Array | undefined, - lookupIndicesView: Int32Array -): void { - // Store the total lookup count - let li = 0 - lookupIndicesView[li++] = lookupDefs.length - - // Store the data and indices for each lookup - let lookupDataOffset = 0 - for (const lookupDef of lookupDefs) { - // Store lookup indices - const varSpec = lookupDef.varRef.varSpec - const subs = varSpec.subscriptIndices - const subCount = varSpec.subscriptIndices?.length || 0 - lookupIndicesView[li++] = varSpec.varIndex - // TODO: Update this once we support > 3 subscripts - lookupIndicesView[li++] = subCount > 0 ? subs[0] : -1 - lookupIndicesView[li++] = subCount > 1 ? subs[1] : -1 - lookupIndicesView[li++] = subCount > 2 ? subs[2] : -1 - lookupIndicesView[li++] = lookupDataOffset - lookupIndicesView[li++] = lookupDef.points.length - - // Store lookup data. Note that `lookupsView` can be undefined in the case - // where the lookup data is empty. - lookupsView?.set(lookupDef.points, lookupDataOffset) - lookupDataOffset += lookupDef.points.length - } -} - -/** - * Decode lookup data and indices from the given buffer views and return the - * reconstruct `LookupDef` instances. - * - * @param lookupsView The view on the lookup data buffer. This can be undefined in - * the case where the data for the lookup(s) is empty. - * @param lookupIndicesView The view on the lookup indices buffer. - */ -function decodeLookups(lookupsView: Float64Array | undefined, lookupIndicesView: Int32Array): LookupDef[] { - const lookupDefs: LookupDef[] = [] - - let li = 0 - const lookupCount = lookupIndicesView[li++] - for (let i = 0; i < lookupCount; i++) { - // Read the metadata from the lookup indices buffer - const varIndex = lookupIndicesView[li++] - // TODO: Update this once we support > 3 subscripts - const subIndex0 = lookupIndicesView[li++] - const subIndex1 = lookupIndicesView[li++] - const subIndex2 = lookupIndicesView[li++] - const lookupDataOffset = lookupIndicesView[li++] - const lookupDataLength = lookupIndicesView[li++] - const subscriptIndices: number[] = subIndex0 >= 0 ? [] : undefined - if (subIndex0 >= 0) subscriptIndices.push(subIndex0) - if (subIndex1 >= 0) subscriptIndices.push(subIndex1) - if (subIndex2 >= 0) subscriptIndices.push(subIndex2) - const varSpec: VarSpec = { - varIndex, - subscriptIndices - } - - // Copy the data from the lookup data buffer. Note that `lookupsView` can be undefined - // in the case where the lookup data is empty. - // TODO: We can use `subarray` here instead of `slice` and let the model implementations - // copy the data if needed on their side - let points: Float64Array - if (lookupsView) { - points = lookupsView.slice(lookupDataOffset, lookupDataOffset + lookupDataLength) - } else { - points = new Float64Array(0) - } - lookupDefs.push({ - varRef: { - varSpec - }, - points - }) - } - - return lookupDefs -} diff --git a/packages/runtime/src/runnable-model/referenced-run-model-params.spec.ts b/packages/runtime/src/runnable-model/referenced-run-model-params.spec.ts index c957f0e1..d6439823 100644 --- a/packages/runtime/src/runnable-model/referenced-run-model-params.spec.ts +++ b/packages/runtime/src/runnable-model/referenced-run-model-params.spec.ts @@ -10,6 +10,21 @@ import { ModelListing } from '../model-listing' const listingJson = ` { "dimensions": [ + { + "id": "_dima", + "subIds": [ + "_a1", + "_a2" + ] + }, + { + "id": "_dimb", + "subIds": [ + "_b1", + "_b2", + "_b3" + ] + } ], "variables": [ { @@ -27,6 +42,14 @@ const listingJson = ` { "id": "_y", "index": 4 + }, + { + "id": "_z", + "dimIds": [ + "_dima", + "_dimb" + ], + "index": 5 } ] } @@ -96,20 +119,22 @@ describe('ReferencedRunModelParams', () => { const listing = new ModelListing(JSON.parse(listingJson)) const inputs = [1, 2, 3] const normalOutputs = new Outputs(['_x', '_y'], 2000, 2002, 1) - const implOutputs = listing.deriveOutputs(normalOutputs, ['_x', '_a', '_b']) + const implOutputs = listing.deriveOutputs(normalOutputs, ['_x', '_z[_a2,_b3]', '_z[_a1,_b1]', '_b']) const params = new ReferencedRunModelParams() params.updateFromParams(inputs, implOutputs) const expectedIndices = new Int32Array([ + // variable count + 4, // _x - 3, 0, 0, 0, - // _a - 1, 0, 0, 0, + 3, 0, + // _z[_a2,_b3] + 5, 2, 1, 2, + // _z[_a1,_b1] + 5, 2, 0, 0, // _b - 2, 0, 0, 0, - // (zero terminator) - 0, 0, 0, 0 + 2, 0 ]) let array: Int32Array @@ -128,18 +153,22 @@ describe('ReferencedRunModelParams', () => { expect(array).toEqual(expectedIndices) // Verify case where existing array is large enough - array = new Int32Array(20).fill(6) + array = new Int32Array(17).fill(6) params.copyOutputIndices(array, create) expect(array).toEqual( new Int32Array([ + // variable count + 4, // _x - 3, 0, 0, 0, - // _a - 1, 0, 0, 0, + 3, 0, + // _z[_a2,_b3] + 5, 2, 1, 2, + // _z[_a1,_b1] + 5, 2, 0, 0, // _b - 2, 0, 0, 0, - // (zero terminators) - 0, 0, 0, 0, 0, 0, 0, 0 + 2, 0, + // (existing data) + 6, 6, 6, 6 ]) ) }) diff --git a/packages/runtime/src/runnable-model/referenced-run-model-params.ts b/packages/runtime/src/runnable-model/referenced-run-model-params.ts index 7487f572..b29dce21 100644 --- a/packages/runtime/src/runnable-model/referenced-run-model-params.ts +++ b/packages/runtime/src/runnable-model/referenced-run-model-params.ts @@ -1,7 +1,7 @@ // Copyright (c) 2024 Climate Interactive / New Venture Fund import type { InputValue, LookupDef, Outputs } from '../_shared' -import { indicesPerVariable, updateVarIndices } from '../_shared' +import { encodeVarIndices, getEncodedVarIndicesLength } from '../_shared' import type { ModelListing } from '../model-listing' import { resolveVarRef } from './resolve-var-ref' import type { RunModelOptions } from './run-model-options' @@ -81,7 +81,7 @@ export class ReferencedRunModelParams implements RunModelParams { } // Copy the output indices to the provided array - updateVarIndices(array, this.outputs.varSpecs) + encodeVarIndices(this.outputs.varSpecs, array) } // from RunModelParams interface @@ -159,9 +159,8 @@ export class ReferencedRunModelParams implements RunModelParams { // See if the output indices are needed const outputVarSpecs = outputs.varSpecs if (outputVarSpecs !== undefined && outputVarSpecs.length > 0) { - // The output indices buffer needs to include N elements for each var spec plus one - // additional "zero" element as a terminator - this.outputIndicesLengthInElements = (outputVarSpecs.length + 1) * indicesPerVariable + // Compute the required length of the output indices buffer + this.outputIndicesLengthInElements = getEncodedVarIndicesLength(outputVarSpecs) } else { // Don't use the output indices buffer when output var specs are not provided this.outputIndicesLengthInElements = 0