feat: optionally generate code without runner integration (#836)

Closes #831 - allow not generating runner function calls (memoized function call, placeholder saving) - added test for eject functionality + tests that begin with "eject" are automatically tested without runner support --------- Co-authored-by: megalinter-bot <[email protected]>
Safe-DS · Feb 3, 2024 · 0ed9d6e · 0ed9d6e
1 parent 21298d0
commit 0ed9d6e
Show file tree

Hide file tree

Showing 14 changed files with 219 additions and 31 deletions.
diff --git a/docs/development/generation-testing.md b/docs/development/generation-testing.md
@@ -12,6 +12,10 @@ generation test.
 
         If you want to skip a test, add the prefix `skip-` to the folder name.
 
+    !!! tip "Tests without runner integration"
+
+        If you want to create a test without runner integration (memoization and placeholder saving), put it in the `eject` folder or use `eject` as a prefix for a new top level folder.
+
 2. Add files with the extension `.sdstest`, `.sdspipe`, or `.sdsstub` **directly inside the folder**. All files in a
    folder will be loaded into the same workspace, so they can reference each other. Files in different folders are
    loaded into different workspaces, so they cannot reference each other. Generation will be triggered for all files in

diff --git a/packages/safe-ds-cli/src/cli/generate.ts b/packages/safe-ds-cli/src/cli/generate.ts
@@ -23,6 +23,7 @@ export const generate = async (fsPaths: string[], options: GenerateOptions): Pro
             destination: URI.file(path.resolve(options.out)),
             createSourceMaps: options.sourcemaps,
             targetPlaceholder: undefined,
+            disableRunnerIntegration: false,
         });
 
         for (const file of generatedFiles) {

diff --git a/packages/safe-ds-lang/src/language/generation/safe-ds-python-generator.ts b/packages/safe-ds-lang/src/language/generation/safe-ds-python-generator.ts
@@ -105,12 +105,60 @@ const BLOCK_LAMBDA_PREFIX = `${CODEGEN_PREFIX}block_lambda_`;
 const BLOCK_LAMBDA_RESULT_PREFIX = `${CODEGEN_PREFIX}block_lambda_result_`;
 const YIELD_PREFIX = `${CODEGEN_PREFIX}yield_`;
 
-const RUNNER_CODEGEN_PACKAGE = 'safeds_runner.codegen';
 const RUNNER_SERVER_PIPELINE_MANAGER_PACKAGE = 'safeds_runner.server.pipeline_manager';
 const PYTHON_INDENT = '    ';
 
 const NLNL = new CompositeGeneratorNode(NL, NL);
 
+const UTILITY_EAGER_OR: UtilityFunction = {
+    code: expandToNode`def ${CODEGEN_PREFIX}eager_or(left_operand: bool, right_operand: bool) -> bool:`
+        .appendNewLine()
+        .indent({ indentedChildren: ['return left_operand or right_operand'], indentation: PYTHON_INDENT }),
+    name: `${CODEGEN_PREFIX}eager_or`,
+    imports: [],
+};
+
+const UTILITY_EAGER_AND: UtilityFunction = {
+    code: expandToNode`def ${CODEGEN_PREFIX}eager_and(left_operand: bool, right_operand: bool) -> bool:`
+        .appendNewLine()
+        .indent({ indentedChildren: ['return left_operand and right_operand'], indentation: PYTHON_INDENT }),
+    name: `${CODEGEN_PREFIX}eager_and`,
+    imports: [],
+};
+
+const UTILITY_EAGER_ELVIS: UtilityFunction = {
+    code: expandToNode`${CODEGEN_PREFIX}T = TypeVar("${CODEGEN_PREFIX}T")`
+        .appendNewLine()
+        .appendNewLine()
+        .append(
+            `def ${CODEGEN_PREFIX}eager_elvis(left_operand: ${CODEGEN_PREFIX}T, right_operand: ${CODEGEN_PREFIX}T) -> ${CODEGEN_PREFIX}T:`,
+        )
+        .appendNewLine()
+        .indent({
+            indentedChildren: ['return left_operand if left_operand is not None else right_operand'],
+            indentation: PYTHON_INDENT,
+        }),
+    name: `${CODEGEN_PREFIX}eager_elvis`,
+    imports: [{ importPath: 'typing', declarationName: 'TypeVar' }],
+};
+
+const UTILITY_SAFE_ACCESS: UtilityFunction = {
+    code: expandToNode`${CODEGEN_PREFIX}S = TypeVar("${CODEGEN_PREFIX}S")`
+        .appendNewLine()
+        .appendNewLine()
+        .append(`def ${CODEGEN_PREFIX}safe_access(receiver: Any, member_name: str) -> ${CODEGEN_PREFIX}S | None:`)
+        .appendNewLine()
+        .indent({
+            indentedChildren: ['return getattr(receiver, member_name) if receiver is not None else None'],
+            indentation: PYTHON_INDENT,
+        }),
+    name: `${CODEGEN_PREFIX}safe_access`,
+    imports: [
+        { importPath: 'typing', declarationName: 'TypeVar' },
+        { importPath: 'typing', declarationName: 'Any' },
+    ],
+};
+
 export class SafeDsPythonGenerator {
     private readonly builtinAnnotations: SafeDsAnnotations;
     private readonly nodeMapper: SafeDsNodeMapper;
@@ -138,7 +186,7 @@ export class SafeDsPythonGenerator {
         const parentDirectoryPath = path.join(generateOptions.destination!.fsPath, ...packagePath);
 
         const generatedFiles = new Map<string, string>();
-        const generatedModule = this.generateModule(node, generateOptions.targetPlaceholder);
+        const generatedModule = this.generateModule(node, generateOptions);
         const { text, trace } = toStringAndTrace(generatedModule);
         const pythonOutputPath = `${path.join(parentDirectoryPath, this.formatGeneratedFileName(name))}.py`;
         if (generateOptions.createSourceMaps) {
@@ -261,14 +309,15 @@ export class SafeDsPythonGenerator {
         return moduleName.replaceAll('%2520', '_').replaceAll(/[ .-]/gu, '_').replaceAll(/\\W/gu, '');
     }
 
-    private generateModule(module: SdsModule, targetPlaceholder: string | undefined): CompositeGeneratorNode {
+    private generateModule(module: SdsModule, generateOptions: GenerateOptions): CompositeGeneratorNode {
         const importSet = new Map<String, ImportData>();
+        const utilitySet = new Set<UtilityFunction>();
         const segments = getModuleMembers(module)
             .filter(isSdsSegment)
-            .map((segment) => this.generateSegment(segment, importSet));
+            .map((segment) => this.generateSegment(segment, importSet, utilitySet, generateOptions));
         const pipelines = getModuleMembers(module)
             .filter(isSdsPipeline)
-            .map((pipeline) => this.generatePipeline(pipeline, importSet, targetPlaceholder));
+            .map((pipeline) => this.generatePipeline(pipeline, importSet, utilitySet, generateOptions));
         const imports = this.generateImports(Array.from(importSet.values()));
         const output = new CompositeGeneratorNode();
         output.trace(module);
@@ -279,16 +328,24 @@ export class SafeDsPythonGenerator {
             output.append(joinToNode(imports, (importDecl) => importDecl, { separator: NL }));
             output.appendNewLine();
         }
-        if (segments.length > 0) {
+        if (utilitySet.size > 0) {
             output.appendNewLineIf(imports.length > 0);
+            output.append('# Utils ------------------------------------------------------------------------');
+            output.appendNewLine();
+            output.appendNewLine();
+            output.append(joinToNode(utilitySet, (importDecl) => importDecl.code, { separator: NLNL }));
+            output.appendNewLine();
+        }
+        if (segments.length > 0) {
+            output.appendNewLineIf(imports.length > 0 || utilitySet.size > 0);
             output.append('# Segments ---------------------------------------------------------------------');
             output.appendNewLine();
             output.appendNewLine();
             output.append(joinToNode(segments, (segment) => segment, { separator: NLNL }));
             output.appendNewLine();
         }
         if (pipelines.length > 0) {
-            output.appendNewLineIf(imports.length > 0 || segments.length > 0);
+            output.appendNewLineIf(imports.length > 0 || utilitySet.size > 0 || segments.length > 0);
             output.append('# Pipelines --------------------------------------------------------------------');
             output.appendNewLine();
             output.appendNewLine();
@@ -298,8 +355,19 @@ export class SafeDsPythonGenerator {
         return output;
     }
 
-    private generateSegment(segment: SdsSegment, importSet: Map<String, ImportData>): CompositeGeneratorNode {
-        const infoFrame = new GenerationInfoFrame(importSet);
+    private generateSegment(
+        segment: SdsSegment,
+        importSet: Map<String, ImportData>,
+        utilitySet: Set<UtilityFunction>,
+        generateOptions: GenerateOptions,
+    ): CompositeGeneratorNode {
+        const infoFrame = new GenerationInfoFrame(
+            importSet,
+            utilitySet,
+            false,
+            undefined,
+            generateOptions.disableRunnerIntegration,
+        );
         const segmentResult = segment.resultList?.results || [];
         const segmentBlock = this.generateBlock(segment.body, infoFrame);
         if (segmentResult.length !== 0) {
@@ -350,9 +418,16 @@ export class SafeDsPythonGenerator {
     private generatePipeline(
         pipeline: SdsPipeline,
         importSet: Map<String, ImportData>,
-        targetPlaceholder: string | undefined,
+        utilitySet: Set<UtilityFunction>,
+        generateOptions: GenerateOptions,
     ): CompositeGeneratorNode {
-        const infoFrame = new GenerationInfoFrame(importSet, true, targetPlaceholder);
+        const infoFrame = new GenerationInfoFrame(
+            importSet,
+            utilitySet,
+            true,
+            generateOptions.targetPlaceholder,
+            generateOptions.disableRunnerIntegration,
+        );
         return expandTracedToNode(pipeline)`def ${traceToNode(
             pipeline,
             'name',
@@ -538,7 +613,7 @@ export class SafeDsPythonGenerator {
                     )} = ${this.generateExpression(assignment.expression!, frame)}`,
                 );
             }
-            if (frame.isInsidePipeline && !generateLambda) {
+            if (frame.isInsidePipeline && !generateLambda && !frame.disableRunnerIntegration) {
                 for (const savableAssignment of assignees.filter(isSdsPlaceholder)) {
                     // should always be SdsPlaceholder
                     frame.addImport({ importPath: RUNNER_SERVER_PIPELINE_MANAGER_PACKAGE });
@@ -675,7 +750,7 @@ export class SafeDsPythonGenerator {
                         return this.generatePythonCall(expression, pythonCall, argumentsMap, frame, thisParam);
                     }
                 }
-                if (this.isMemoizableCall(expression)) {
+                if (this.isMemoizableCall(expression) && !frame.disableRunnerIntegration) {
                     let thisParam: CompositeGeneratorNode | undefined = undefined;
                     if (isSdsMemberAccess(expression.receiver)) {
                         thisParam = this.generateExpression(expression.receiver.receiver, frame);
@@ -694,23 +769,23 @@ export class SafeDsPythonGenerator {
             const rightOperand = this.generateExpression(expression.rightOperand, frame);
             switch (expression.operator) {
                 case 'or':
-                    frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE });
+                    frame.addUtility(UTILITY_EAGER_OR);
                     return expandTracedToNode(expression)`${traceToNode(
                         expression,
                         'operator',
-                    )(`${RUNNER_CODEGEN_PACKAGE}.eager_or`)}(${leftOperand}, ${rightOperand})`;
+                    )(UTILITY_EAGER_OR.name)}(${leftOperand}, ${rightOperand})`;
                 case 'and':
-                    frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE });
+                    frame.addUtility(UTILITY_EAGER_AND);
                     return expandTracedToNode(expression)`${traceToNode(
                         expression,
                         'operator',
-                    )(`${RUNNER_CODEGEN_PACKAGE}.eager_and`)}(${leftOperand}, ${rightOperand})`;
+                    )(UTILITY_EAGER_AND.name)}(${leftOperand}, ${rightOperand})`;
                 case '?:':
-                    frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE });
+                    frame.addUtility(UTILITY_EAGER_ELVIS);
                     return expandTracedToNode(expression)`${traceToNode(
                         expression,
                         'operator',
-                    )(`${RUNNER_CODEGEN_PACKAGE}.eager_elvis`)}(${leftOperand}, ${rightOperand})`;
+                    )(UTILITY_EAGER_ELVIS.name)}(${leftOperand}, ${rightOperand})`;
                 case '===':
                     return expandTracedToNode(expression)`(${leftOperand}) ${traceToNode(
                         expression,
@@ -751,11 +826,11 @@ export class SafeDsPythonGenerator {
             } else {
                 const memberExpression = this.generateExpression(expression.member!, frame);
                 if (expression.isNullSafe) {
-                    frame.addImport({ importPath: RUNNER_CODEGEN_PACKAGE });
+                    frame.addUtility(UTILITY_SAFE_ACCESS);
                     return expandTracedToNode(expression)`${traceToNode(
                         expression,
                         'isNullSafe',
-                    )(`${RUNNER_CODEGEN_PACKAGE}.safe_access`)}(${receiver}, '${memberExpression}')`;
+                    )(UTILITY_SAFE_ACCESS.name)}(${receiver}, '${memberExpression}')`;
                 } else {
                     return expandTracedToNode(expression)`${receiver}.${memberExpression}`;
                 }
@@ -817,7 +892,7 @@ export class SafeDsPythonGenerator {
             { separator: '' },
         )!;
         // Non-memoizable calls can be directly generated
-        if (!this.isMemoizableCall(expression)) {
+        if (!this.isMemoizableCall(expression) || frame.disableRunnerIntegration) {
             return generatedPythonCall;
         }
         frame.addImport({ importPath: RUNNER_SERVER_PIPELINE_MANAGER_PACKAGE });
@@ -1040,6 +1115,12 @@ export class SafeDsPythonGenerator {
     }
 }
 
+interface UtilityFunction {
+    readonly code: CompositeGeneratorNode;
+    readonly name: string;
+    readonly imports: ImportData[];
+}
+
 interface ImportData {
     readonly importPath: string;
     readonly declarationName?: string;
@@ -1049,18 +1130,24 @@ interface ImportData {
 class GenerationInfoFrame {
     private readonly blockLambdaManager: IdManager<SdsBlockLambda>;
     private readonly importSet: Map<String, ImportData>;
+    private readonly utilitySet: Set<UtilityFunction>;
     public readonly isInsidePipeline: boolean;
     public readonly targetPlaceholder: string | undefined;
+    public readonly disableRunnerIntegration: boolean;
 
     constructor(
         importSet: Map<String, ImportData> = new Map<String, ImportData>(),
+        utilitySet: Set<UtilityFunction> = new Set<UtilityFunction>(),
         insidePipeline: boolean = false,
         targetPlaceholder: string | undefined = undefined,
+        disableRunnerIntegration: boolean = false,
     ) {
         this.blockLambdaManager = new IdManager<SdsBlockLambda>();
         this.importSet = importSet;
+        this.utilitySet = utilitySet;
         this.isInsidePipeline = insidePipeline;
         this.targetPlaceholder = targetPlaceholder;
+        this.disableRunnerIntegration = disableRunnerIntegration;
     }
 
     addImport(importData: ImportData | undefined) {
@@ -1072,6 +1159,13 @@ class GenerationInfoFrame {
         }
     }
 
+    addUtility(utilityFunction: UtilityFunction) {
+        this.utilitySet.add(utilityFunction);
+        for (const importData of utilityFunction.imports) {
+            this.addImport(importData);
+        }
+    }
+
     getUniqueLambdaBlockName(lambda: SdsBlockLambda): string {
         return `${BLOCK_LAMBDA_PREFIX}${this.blockLambdaManager.assignId(lambda)}`;
     }
@@ -1081,4 +1175,5 @@ export interface GenerateOptions {
     destination: URI;
     createSourceMaps: boolean;
     targetPlaceholder: string | undefined;
+    disableRunnerIntegration: boolean;
 }
diff --git a/packages/safe-ds-lang/tests/language/generation/creator.ts b/packages/safe-ds-lang/tests/language/generation/creator.ts
@@ -82,6 +82,7 @@ const createGenerationTest = async (parentDirectory: URI, inputUris: URI[]): Pro
         actualOutputRoot,
         expectedOutputFiles,
         runUntil,
+        disableRunnerIntegration: shortenedResourceName.startsWith('eject'), // Tests in the "eject" top level folder are tested with disabled runner integration
     };
 };
 
@@ -115,6 +116,7 @@ const invalidTest = (level: 'FILE' | 'SUITE', error: TestDescriptionError): Gene
         actualOutputRoot: URI.file(''),
         expectedOutputFiles: [],
         error,
+        disableRunnerIntegration: false,
     };
 };
 
@@ -141,6 +143,11 @@ interface GenerationTest extends TestDescription {
      * Location after which execution should be stopped.
      */
     runUntil?: Location;
+
+    /**
+     * Whether the test should run with runner integration (memoization & placeholder saving) disabled
+     */
+    disableRunnerIntegration: boolean;
 }
 
 /**

diff --git a/packages/safe-ds-lang/tests/language/generation/safe-ds-python-generator.test.ts b/packages/safe-ds-lang/tests/language/generation/safe-ds-python-generator.test.ts
@@ -36,6 +36,7 @@ describe('generation', async () => {
                     destination: test.actualOutputRoot,
                     createSourceMaps: true,
                     targetPlaceholder: runUntilPlaceholderName,
+                    disableRunnerIntegration: test.disableRunnerIntegration,
                 }),
             )
             .map((textDocument) => [textDocument.uri, textDocument.getText()])

diff --git a/packages/safe-ds-lang/tests/resources/generation/eject/input.sdstest b/packages/safe-ds-lang/tests/resources/generation/eject/input.sdstest
@@ -0,0 +1,24 @@
+package tests.generator.eject
+
+@Impure([ImpurityReason.Other]) fun f(param: Any?)
+
+@Pure fun g() -> result: Boolean
+
+@Pure fun i() -> result: Int?
+
+@Pure fun factory() -> instance: C?
+
+class C() {
+    attr a: Int
+    @PythonName("c") attr b: Int
+}
+
+pipeline test {
+    f(g() or g());
+    f(g() and g());
+
+    f(factory()?.a);
+    f(factory()?.b);
+
+    f(i() ?: i());
+}
diff --git a/...s/safe-ds-lang/tests/resources/generation/eject/output/tests/generator/eject/gen_input.py b/...s/safe-ds-lang/tests/resources/generation/eject/output/tests/generator/eject/gen_input.py
@@ -0,0 +1,30 @@
+# Imports ----------------------------------------------------------------------
+
+from typing import Any, TypeVar
+
+# Utils ------------------------------------------------------------------------
+
+def __gen_eager_or(left_operand: bool, right_operand: bool) -> bool:
+    return left_operand or right_operand
+
+def __gen_eager_and(left_operand: bool, right_operand: bool) -> bool:
+    return left_operand and right_operand
+
+__gen_S = TypeVar("__gen_S")
+
+def __gen_safe_access(receiver: Any, member_name: str) -> __gen_S | None:
+    return getattr(receiver, member_name) if receiver is not None else None
+
+__gen_T = TypeVar("__gen_T")
+
+def __gen_eager_elvis(left_operand: __gen_T, right_operand: __gen_T) -> __gen_T:
+    return left_operand if left_operand is not None else right_operand
+
+# Pipelines --------------------------------------------------------------------
+
+def test():
+    f(__gen_eager_or(g(), g()))
+    f(__gen_eager_and(g(), g()))
+    f(__gen_safe_access(factory(), 'a'))
+    f(__gen_safe_access(factory(), 'c'))
+    f(__gen_eager_elvis(i(), i()))