diff --git a/.gitignore b/.gitignore index 55fdff59..dd4b3cbd 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,9 @@ node_modules *.launch .settings/ *.sublime-workspace +.nvim.lua +.nvimrc +.exrc # IDE - VSCode .vscode/* @@ -58,4 +61,4 @@ Thumbs.db .cache-loader/ .nx/cache -.nx/workspace-data \ No newline at end of file +.nx/workspace-data diff --git a/libs/execution/src/lib/blocks/block-execution-util.ts b/libs/execution/src/lib/blocks/block-execution-util.ts index f18a6d1d..1d1f234b 100644 --- a/libs/execution/src/lib/blocks/block-execution-util.ts +++ b/libs/execution/src/lib/blocks/block-execution-util.ts @@ -62,11 +62,15 @@ export async function executeBlocks( executionContext.enterNode(block); + await executionContext.executeHooks(inputValue); + const executionResult = await executeBlock( inputValue, block, executionContext, ); + await executionContext.executeHooks(inputValue, executionResult); + if (R.isErr(executionResult)) { return executionResult; } diff --git a/libs/execution/src/lib/execution-context.ts b/libs/execution/src/lib/execution-context.ts index 8c6c9a1b..5463ea62 100644 --- a/libs/execution/src/lib/execution-context.ts +++ b/libs/execution/src/lib/execution-context.ts @@ -4,6 +4,7 @@ // eslint-disable-next-line unicorn/prefer-node-protocol import { strict as assert } from 'assert'; +import { inspect } from 'node:util'; import { type BlockDefinition, @@ -26,13 +27,16 @@ import { } from '@jvalue/jayvee-language-server'; import { assertUnreachable, isReference } from 'langium'; +import { type Result } from './blocks'; import { type JayveeConstraintExtension } from './constraints'; import { type DebugGranularity, type DebugTargets, } from './debugging/debug-configuration'; import { type JayveeExecExtension } from './extension'; +import { type HookContext } from './hooks'; import { type Logger } from './logging/logger'; +import { type IOTypeImplementation } from './types'; export type StackNode = | BlockDefinition @@ -55,6 +59,7 @@ export class ExecutionContext { debugTargets: DebugTargets; }, public readonly evaluationContext: EvaluationContext, + public readonly hookContext: HookContext, ) { logger.setLoggingContext(pipeline.name); } @@ -133,6 +138,34 @@ export class ExecutionContext { return property; } + public executeHooks( + input: IOTypeImplementation | null, + output?: Result, + ) { + const node = this.getCurrentNode(); + assert( + isBlockDefinition(node), + `Expected node to be \`BlockDefinition\`: ${inspect(node)}`, + ); + + const blocktype = node.type.ref?.name; + assert( + blocktype !== undefined, + `Expected block definition to have a blocktype: ${inspect(node)}`, + ); + + if (output === undefined) { + return this.hookContext.executePreBlockHooks(blocktype, input, this); + } + + return this.hookContext.executePostBlockHooks( + blocktype, + input, + this, + output, + ); + } + private getDefaultPropertyValue( propertyName: string, valueType: ValueType, diff --git a/libs/execution/src/lib/hooks/hook-context.ts b/libs/execution/src/lib/hooks/hook-context.ts new file mode 100644 index 00000000..adfe9c26 --- /dev/null +++ b/libs/execution/src/lib/hooks/hook-context.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: 2025 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { type Result } from '../blocks'; +import { type ExecutionContext } from '../execution-context'; +import { type IOTypeImplementation } from '../types'; + +import { + type HookOptions, + type HookPosition, + type PostBlockHook, + type PreBlockHook, + isPreBlockHook, +} from './hook'; + +const AllBlocks = '*'; + +interface HookSpec { + blocking: boolean; + hook: H; +} + +// eslint-disable-next-line @typescript-eslint/no-empty-function +function noop() {} + +async function executePreBlockHooks( + hooks: HookSpec[], + blocktype: string, + input: IOTypeImplementation | null, + context: ExecutionContext, +) { + await Promise.all( + hooks.map(async ({ blocking, hook }) => { + if (blocking) { + await hook(blocktype, input, context); + } else { + hook(blocktype, input, context).catch(noop); + } + }), + ); +} + +async function executePostBlockHooks( + hooks: HookSpec[], + blocktype: string, + input: IOTypeImplementation | null, + context: ExecutionContext, + output: Result, +) { + await Promise.all( + hooks.map(async ({ blocking, hook }) => { + if (blocking) { + await hook(blocktype, input, output, context); + } else { + hook(blocktype, input, output, context).catch(noop); + } + }), + ); +} + +export class HookContext { + private hooks: { + pre: Record[]>; + post: Record[]>; + } = { pre: {}, post: {} }; + + public addHook( + position: 'preBlock', + hook: PreBlockHook, + opts: HookOptions, + ): void; + public addHook( + position: 'postBlock', + hook: PostBlockHook, + opts: HookOptions, + ): void; + public addHook( + position: HookPosition, + hook: PreBlockHook | PostBlockHook, + opts: HookOptions, + ): void; + public addHook( + position: HookPosition, + hook: PreBlockHook | PostBlockHook, + opts: HookOptions, + ) { + for (const blocktype of opts.blocktypes ?? [AllBlocks]) { + if (isPreBlockHook(hook, position)) { + if (this.hooks.pre[blocktype] === undefined) { + this.hooks.pre[blocktype] = []; + } + this.hooks.pre[blocktype].push({ + blocking: opts.blocking ?? false, + hook, + }); + } else { + if (this.hooks.post[blocktype] === undefined) { + this.hooks.post[blocktype] = []; + } + this.hooks.post[blocktype].push({ + blocking: opts.blocking ?? false, + hook, + }); + } + } + } + + public async executePreBlockHooks( + blocktype: string, + input: IOTypeImplementation | null, + context: ExecutionContext, + ) { + context.logger.logInfo(`Executing general pre-block-hooks`); + const general = executePreBlockHooks( + this.hooks.pre[AllBlocks] ?? [], + blocktype, + input, + context, + ); + context.logger.logInfo( + `Executing pre-block-hooks for blocktype ${blocktype}`, + ); + const blockSpecific = executePreBlockHooks( + this.hooks.pre[blocktype] ?? [], + blocktype, + input, + context, + ); + + await Promise.all([general, blockSpecific]); + } + + public async executePostBlockHooks( + blocktype: string, + input: IOTypeImplementation | null, + context: ExecutionContext, + output: Result, + ) { + context.logger.logInfo(`Executing general post-block-hooks`); + const general = executePostBlockHooks( + this.hooks.post[AllBlocks] ?? [], + blocktype, + input, + context, + output, + ); + context.logger.logInfo( + `Executing post-block-hooks for blocktype ${blocktype}`, + ); + const blockSpecific = executePostBlockHooks( + this.hooks.post[blocktype] ?? [], + blocktype, + input, + context, + output, + ); + + await Promise.all([general, blockSpecific]); + } +} diff --git a/libs/execution/src/lib/hooks/hook.ts b/libs/execution/src/lib/hooks/hook.ts new file mode 100644 index 00000000..64803658 --- /dev/null +++ b/libs/execution/src/lib/hooks/hook.ts @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2025 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { type Result } from '../blocks'; +import { type ExecutionContext } from '../execution-context'; +import { type IOTypeImplementation } from '../types'; + +/** When to execute the hook.*/ +export type HookPosition = 'preBlock' | 'postBlock'; + +export interface HookOptions { + /** Whether the pipeline should await the hooks completion. `false` if omitted.*/ + blocking?: boolean; + /** Optionally specify one or more blocks to limit this hook to. If omitted, the hook will be executed on all blocks*/ + blocktypes?: string[]; +} + +/** This function will be executed before a block.*/ +export type PreBlockHook = ( + blocktype: string, + input: IOTypeImplementation | null, + context: ExecutionContext, +) => Promise; + +export function isPreBlockHook( + hook: PreBlockHook | PostBlockHook, + position: HookPosition, +): hook is PreBlockHook { + return position === 'preBlock'; +} + +/** This function will be executed before a block.*/ +export type PostBlockHook = ( + blocktype: string, + input: IOTypeImplementation | null, + output: Result, + context: ExecutionContext, +) => Promise; + +export function isPostBlockHook( + hook: PreBlockHook | PostBlockHook, + position: HookPosition, +): hook is PostBlockHook { + return position === 'postBlock'; +} diff --git a/libs/execution/src/lib/hooks/index.ts b/libs/execution/src/lib/hooks/index.ts new file mode 100644 index 00000000..20bb5603 --- /dev/null +++ b/libs/execution/src/lib/hooks/index.ts @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: 2025 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +export * from './hook'; +export * from './hook-context'; diff --git a/libs/execution/src/lib/index.ts b/libs/execution/src/lib/index.ts index cd267ae5..5c4a03ab 100644 --- a/libs/execution/src/lib/index.ts +++ b/libs/execution/src/lib/index.ts @@ -13,3 +13,4 @@ export * from './types/value-types/visitors'; export * from './execution-context'; export * from './extension'; export * from './logging'; +export * from './hooks'; diff --git a/libs/interpreter-lib/src/interpreter.spec.ts b/libs/interpreter-lib/src/interpreter.spec.ts index 08a534a2..401e30f4 100644 --- a/libs/interpreter-lib/src/interpreter.spec.ts +++ b/libs/interpreter-lib/src/interpreter.spec.ts @@ -2,10 +2,12 @@ // // SPDX-License-Identifier: AGPL-3.0-only +import { Table, isOk } from '@jvalue/jayvee-execution'; +import { type JayveeModel } from '@jvalue/jayvee-language-server'; import { readJvTestAssetHelper } from '@jvalue/jayvee-language-server/test'; import { DefaultJayveeInterpreter } from './interpreter'; -import { ExitCode } from './parsing-util'; +import { ExitCode, extractAstNodeFromString } from './parsing-util'; describe('Interpreter', () => { const readJvTestAsset = readJvTestAssetHelper(__dirname, '../../../'); @@ -26,4 +28,240 @@ describe('Interpreter', () => { expect(exitCode).toEqual(ExitCode.SUCCESS); }); }); + + describe('hooks', () => { + it('should execute a general hook on every block', async () => { + const exampleFilePath = 'example/cars.jv'; + const model = readJvTestAsset(exampleFilePath); + + const interpreter = new DefaultJayveeInterpreter({ + pipelineMatcher: () => true, + debug: true, + debugGranularity: 'peek', + debugTarget: 'all', + env: new Map(), + }); + + const program = await interpreter.parseModel( + async (services, loggerFactory) => + await extractAstNodeFromString( + model, + services, + loggerFactory.createLogger(), + ), + ); + expect(program).toBeDefined(); + assert(program !== undefined); + + const spy = vi + .fn>() + .mockResolvedValue(undefined); + + program.addHook( + 'preBlock', + async () => { + return spy(); + }, + { blocking: true }, + ); + + const exitCode = await interpreter.interpretProgram(program); + expect(exitCode).toEqual(ExitCode.SUCCESS); + + expect(spy).toHaveBeenCalledTimes(6); + }); + + it('should execute a block specific hook only on that blocktype', async () => { + const exampleFilePath = + 'libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv'; + const model = readJvTestAsset(exampleFilePath); + + const interpreter = new DefaultJayveeInterpreter({ + pipelineMatcher: () => true, + debug: true, + debugGranularity: 'peek', + debugTarget: 'all', + env: new Map(), + }); + + const program = await interpreter.parseModel( + async (services, loggerFactory) => + await extractAstNodeFromString( + model, + services, + loggerFactory.createLogger(), + ), + ); + expect(program).toBeDefined(); + assert(program !== undefined); + + const sqlite_spy = vi + .fn>() + .mockResolvedValue(undefined); + + program.addHook( + 'preBlock', + async (blocktype) => { + return sqlite_spy(blocktype); + }, + { blocking: true, blocktypes: ['SQLiteLoader'] }, + ); + + const interpreter_spy = vi + .fn>() + .mockResolvedValue(undefined); + + program.addHook( + 'postBlock', + async (blocktype) => { + return interpreter_spy(blocktype); + }, + { blocking: true, blocktypes: ['CSVFileInterpreter'] }, + ); + + const exitCode = await interpreter.interpretProgram(program); + expect(exitCode).toEqual(ExitCode.SUCCESS); + + expect(sqlite_spy).toHaveBeenCalledTimes(1); + expect(sqlite_spy).toHaveBeenCalledWith('SQLiteLoader'); + expect(interpreter_spy).toHaveBeenCalledTimes(1); + expect(interpreter_spy).toHaveBeenCalledWith('CSVFileInterpreter'); + }); + + it('should be called with the correct parameters', async () => { + const exampleFilePath = + 'libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv'; + const model = readJvTestAsset(exampleFilePath); + + const interpreter = new DefaultJayveeInterpreter({ + pipelineMatcher: () => true, + debug: true, + debugGranularity: 'peek', + debugTarget: 'all', + env: new Map(), + }); + + const program = await interpreter.parseModel( + async (services, loggerFactory) => + await extractAstNodeFromString( + model, + services, + loggerFactory.createLogger(), + ), + ); + expect(program).toBeDefined(); + assert(program !== undefined); + + const parameter_spy = vi + .fn>() + .mockResolvedValue(undefined); + + const EXPECTED_NAMES = [ + 'Mazda RX4', + 'Mazda RX4 Wag', + 'Datsun 710', + 'Hornet 4 Drive', + 'Hornet Sportabout', + 'Valiant', + 'Duster 360', + 'Merc 240D', + 'Merc 230', + 'Merc 280', + 'Merc 280C', + 'Merc 450SE', + 'Merc 450SL', + 'Merc 450SLC', + 'Cadillac Fleetwood', + 'Lincoln Continental', + 'Chrysler Imperial', + 'Fiat 128', + 'Honda Civic', + 'Toyota Corolla', + 'Toyota Corona', + 'Dodge Challenger', + 'AMC Javelin', + 'Camaro Z28', + 'Pontiac Firebird', + 'Fiat X1-9', + 'Porsche 914-2', + 'Lotus Europa', + 'Ford Pantera L', + 'Ferrari Dino', + 'Maserati Bora', + 'Volvo 142E', + ]; + + program.addHook( + 'postBlock', + async (blocktype, input, output) => { + expect(blocktype).toBe('TableTransformer'); + + expect(input).not.toBeNull(); + assert(input != null); + assert(input instanceof Table); + + expect(input.getNumberOfColumns()).toBe(1); + expect(input.getColumn('name')?.values).toStrictEqual(EXPECTED_NAMES); + + expect(isOk(output)).toBe(true); + assert(isOk(output)); + const out = output.right; + expect(out).not.toBeNull(); + assert(out != null); + assert(out instanceof Table); + + expect(out.getNumberOfColumns()).toBe(2); + expect(out.getColumn('name')?.values).toStrictEqual(EXPECTED_NAMES); + expect(out.getColumn('nameCopy')?.values).toStrictEqual( + EXPECTED_NAMES, + ); + + return parameter_spy(); + }, + { blocking: true, blocktypes: ['TableTransformer'] }, + ); + + const exitCode = await interpreter.interpretProgram(program); + expect(exitCode).toEqual(ExitCode.SUCCESS); + + expect(parameter_spy).toHaveBeenCalledTimes(1); + }); + + it('should not wait for non-blocking hooks', async () => { + const exampleFilePath = 'example/cars.jv'; + const model = readJvTestAsset(exampleFilePath); + + const interpreter = new DefaultJayveeInterpreter({ + pipelineMatcher: () => true, + debug: true, + debugGranularity: 'peek', + debugTarget: 'all', + env: new Map(), + }); + + const program = await interpreter.parseModel( + async (services, loggerFactory) => + await extractAstNodeFromString( + model, + services, + loggerFactory.createLogger(), + ), + ); + expect(program).toBeDefined(); + assert(program !== undefined); + + program.addHook( + 'postBlock', + (): Promise => { + return new Promise((resolve) => { + setTimeout(resolve, 30000); + }); + }, + { blocking: false }, + ); + + const exitCode = await interpreter.interpretProgram(program); + expect(exitCode).toEqual(ExitCode.SUCCESS); + }, 10000); + }); }); diff --git a/libs/interpreter-lib/src/interpreter.ts b/libs/interpreter-lib/src/interpreter.ts index 19140ad2..a5979e57 100644 --- a/libs/interpreter-lib/src/interpreter.ts +++ b/libs/interpreter-lib/src/interpreter.ts @@ -11,9 +11,14 @@ import { type DebugTargets, DefaultConstraintExtension, ExecutionContext, + HookContext, + type HookOptions, + type HookPosition, type JayveeConstraintExtension, type JayveeExecExtension, type Logger, + type PostBlockHook, + type PreBlockHook, executeBlocks, isErr, logExecutionDuration, @@ -52,6 +57,35 @@ export interface InterpreterOptions { debugTarget: DebugTargets; } +export class JayveeProgram { + private _hooks = new HookContext(); + + constructor(public model: JayveeModel) {} + + /** Add a hook to one or more blocks in the pipeline.*/ + public addHook( + position: 'preBlock', + hook: PreBlockHook, + opts?: HookOptions, + ): void; + public addHook( + position: 'postBlock', + hook: PostBlockHook, + opts?: HookOptions, + ): void; + public addHook( + position: HookPosition, + hook: PreBlockHook | PostBlockHook, + opts?: HookOptions, + ) { + this._hooks.addHook(position, hook, opts ?? {}); + } + + public get hooks() { + return this._hooks; + } +} + export interface JayveeInterpreter { /** * Interprets a parsed Jayvee model. @@ -59,7 +93,7 @@ export interface JayveeInterpreter { * @param extractAstNodeFn the Jayvee model. * @returns the exit code indicating whether interpretation was successful or not. */ - interpretModel(model: JayveeModel): Promise; + interpretProgram(program: JayveeProgram): Promise; /** * Interprets a file as a Jayvee model. @@ -80,18 +114,18 @@ export interface JayveeInterpreter { interpretString(modelString: string): Promise; /** - * Parses a model without executing it. + * Parses a program without executing it. * Also sets up the environment so that the model can be properly executed. * * @param extractAstNodeFn method that extracts the AST node - * @returns the parsed Jayvee model, or undefined on failure. + * @returns the parsed Jayvee program, or undefined on failure. */ parseModel( extractAstNodeFn: ( services: JayveeServices, loggerFactory: LoggerFactory, ) => Promise, - ): Promise; + ): Promise; } export class DefaultJayveeInterpreter implements JayveeInterpreter { @@ -116,11 +150,11 @@ export class DefaultJayveeInterpreter implements JayveeInterpreter { return this; } - async interpretModel(model: JayveeModel): Promise { + async interpretProgram(program: JayveeProgram): Promise { await this.prepareInterpretation(); - const interpretationExitCode = await this.interpretJayveeModel( - model, + const interpretationExitCode = await this.interpretJayveeProgram( + program, new StdExecExtension(), new DefaultConstraintExtension(), ); @@ -145,7 +179,7 @@ export class DefaultJayveeInterpreter implements JayveeInterpreter { return ExitCode.FAILURE; } - return await this.interpretModel(model); + return await this.interpretProgram(model); } async interpretString(modelString: string): Promise { @@ -166,7 +200,7 @@ export class DefaultJayveeInterpreter implements JayveeInterpreter { return ExitCode.FAILURE; } - return await this.interpretModel(model); + return await this.interpretProgram(model); } async parseModel( @@ -174,12 +208,12 @@ export class DefaultJayveeInterpreter implements JayveeInterpreter { services: JayveeServices, loggerFactory: LoggerFactory, ) => Promise, - ): Promise { + ): Promise { await this.prepareInterpretation(); try { const model = await extractAstNodeFn(this.services, this.loggerFactory); - return model; + return new JayveeProgram(model); } catch (e) { this.loggerFactory .createLogger() @@ -213,11 +247,12 @@ export class DefaultJayveeInterpreter implements JayveeInterpreter { } } - private async interpretJayveeModel( - model: JayveeModel, + private async interpretJayveeProgram( + program: JayveeProgram, executionExtension: JayveeExecExtension, constraintExtension: JayveeConstraintExtension, ): Promise { + const model = program.model; const selectedPipelines = model.pipelines.filter((pipeline) => this.options.pipelineMatcher(pipeline), ); @@ -237,6 +272,7 @@ export class DefaultJayveeInterpreter implements JayveeInterpreter { pipeline, executionExtension, constraintExtension, + program.hooks, ); }, ); @@ -252,6 +288,7 @@ export class DefaultJayveeInterpreter implements JayveeInterpreter { pipeline: PipelineDefinition, executionExtension: JayveeExecExtension, constraintExtension: JayveeConstraintExtension, + hooks: HookContext, ): Promise { const executionContext = new ExecutionContext( pipeline, @@ -270,6 +307,7 @@ export class DefaultJayveeInterpreter implements JayveeInterpreter { this.services.operators.EvaluatorRegistry, this.services.ValueTypeProvider, ), + hooks, ); logPipelineOverview( diff --git a/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv b/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv new file mode 100644 index 00000000..3595e7b4 --- /dev/null +++ b/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: 2025 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline CarsPipeline { + + CarsExtractor + -> CarsInterpreter + -> NameHeaderWriter + -> CarsTableInterpreter + -> CarsTableTransformer + -> CarsLoader; + + + block CarsExtractor oftype HttpExtractor { + url: "https://gist.githubusercontent.com/noamross/e5d3e859aa0c794be10b/raw/b999fb4425b54c63cab088c0ce2c0d6ce961a563/cars.csv"; + } + + block CarsInterpreter oftype CSVFileInterpreter { + enclosing: '"'; + } + + block NameHeaderWriter oftype CellWriter { + at: cell A1; + + write: [ + "name" + ]; + } + + block CarsTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "name" oftype text, + ]; + } + + transform copy { + from s oftype text; + to t oftype text; + + t: s; + } + + block CarsTableTransformer oftype TableTransformer { + inputColumns: [ + "name", + ]; + + outputColumn: "nameCopy"; + + uses: copy; + } + + block CarsLoader oftype SQLiteLoader { + table: "Cars"; + file: "./cars.sqlite"; + } +}