feat: generation (#634)

Closes partially #542 

### Summary of Changes

- added an implementation of the generator
- enabled tests for the generator (and removed the dummy test)
- fixed an error related to constant expressions of integers being
interpreted as floats
- fixed an error where test files were read in an incorrect order
- added a generation test for lists and maps
- removed varargs from existing tests

---------

Co-authored-by: Lars Reimann <[email protected]>
Co-authored-by: megalinter-bot <[email protected]>

Author: 3 people

src/cli/generator.ts

@@ -36,26 +36,26 @@ import { isInPipelineFile, isInStubFile, isInTestFile } from '../helpers/fileExt
 import { declarationIsAllowedInPipelineFile, declarationIsAllowedInStubFile } from './other/modules.js';
 import { SafeDsServices } from '../safe-ds-module.js';
 import { listBuiltinFiles } from '../builtins/fileFinder.js';
+import { CODEGEN_PREFIX } from '../../cli/generator.js';
 
-export const CODE_NAME_BLOCK_LAMBDA_PREFIX = 'name/block-lambda-prefix';
+export const CODE_NAME_CODEGEN_PREFIX = 'name/codegen-prefix';
 export const CODE_NAME_CASING = 'name/casing';
 export const CODE_NAME_DUPLICATE = 'name/duplicate';
 
 // -----------------------------------------------------------------------------
-// Block lambda prefix
+// Codegen prefix
 // -----------------------------------------------------------------------------
 
-export const nameMustNotStartWithBlockLambdaPrefix = (node: SdsDeclaration, accept: ValidationAcceptor) => {
+export const nameMustNotStartWithCodegenPrefix = (node: SdsDeclaration, accept: ValidationAcceptor) => {
     const name = node.name ?? '';
-    const blockLambdaPrefix = '__block_lambda_';
-    if (name.startsWith(blockLambdaPrefix)) {
+    if (name.startsWith(CODEGEN_PREFIX)) {
         accept(
             'error',
-            "Names of declarations must not start with '__block_lambda_'. This is reserved for code generation of block lambdas.",
+            `Names of declarations must not start with '${CODEGEN_PREFIX}'. This is reserved for code generation.`,
             {
                 node,
                 property: 'name',
-                code: CODE_NAME_BLOCK_LAMBDA_PREFIX,
+                code: CODE_NAME_CODEGEN_PREFIX,
             },
         );
     }

src/language/validation/names.ts

@@ -12,7 +12,7 @@ import {
     functionMustContainUniqueNames,
     moduleMemberMustHaveNameThatIsUniqueInPackage,
     moduleMustContainUniqueNames,
-    nameMustNotStartWithBlockLambdaPrefix,
+    nameMustNotStartWithCodegenPrefix,
     nameShouldHaveCorrectCasing,
     pipelineMustContainUniqueNames,
     schemaMustContainUniqueNames,
@@ -188,7 +188,7 @@ export const registerValidationChecks = function (services: SafeDsServices) {
         SdsClassBody: [classBodyShouldNotBeEmpty],
         SdsConstraintList: [constraintListShouldNotBeEmpty],
         SdsDeclaration: [
-            nameMustNotStartWithBlockLambdaPrefix,
+            nameMustNotStartWithCodegenPrefix,
             nameShouldHaveCorrectCasing,
             pythonNameShouldDifferFromSafeDsName(services),
             singleUseAnnotationsMustNotBeRepeated(services),

src/language/validation/safe-ds-validator.ts

@@ -41,6 +41,18 @@ export const getErrors = async (services: LangiumServices, code: string): Promis
     return diagnostics.filter((d) => d.severity === DiagnosticSeverity.Error);
 };
 
+/**
+ * Get all errors from a loaded document.
+ *
+ * @param services The language services.
+ * @param uri The URI of the document to check.
+ * @returns The errors.
+ */
+export const getErrorsAtURI = (services: LangiumServices, uri: URI): Diagnostic[] => {
+    const diagnostics = getDiagnosticsAtURI(services, uri);
+    return diagnostics.filter((d) => d.severity === DiagnosticSeverity.Error);
+};
+
 /**
  * Get all diagnostics from a code snippet.
  *
@@ -57,6 +69,18 @@ const getDiagnostics = async (services: LangiumServices, code: string): Promise<
     return document.diagnostics ?? [];
 };
 
+/**
+ * Get all diagnostics from a loaded document.
+ *
+ * @param services The language services.
+ * @param uri The URI of the document to check.
+ * @returns The diagnostics.
+ */
+const getDiagnosticsAtURI = (services: LangiumServices, uri: URI): Diagnostic[] => {
+    const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(uri);
+    return document.diagnostics ?? [];
+};
+
 /**
  * The code contains syntax errors.
  */

tests/helpers/diagnostics.ts

@@ -2,7 +2,8 @@ import path from 'path';
 import { globSync } from 'glob';
 import { SAFE_DS_FILE_EXTENSIONS } from '../../src/language/helpers/fileExtensions.js';
 import { group } from 'radash';
-import { URI } from 'langium';
+import { BuildOptions, LangiumDocument, URI } from 'langium';
+import { SafeDsServices } from '../../src/language/safe-ds-module.js';
 
 const TEST_RESOURCES_PATH = path.join(__dirname, '..', 'resources');
 
@@ -92,3 +93,21 @@ const isNotSkipped = (pathRelativeToResources: string) => {
     const segments = pathRelativeToResources.split(path.sep);
     return !segments.some((segment) => segment.startsWith('skip'));
 };
+
+/**
+ * Load the documents at the specified URIs into the workspace managed by the given services.
+ *
+ * @param services The language services.
+ * @param uris The URIs of the documents to load.
+ * @param options The build options.
+ * @returns The loaded documents.
+ */
+export const loadDocuments = async (
+    services: SafeDsServices,
+    uris: URI[],
+    options: BuildOptions = {},
+): Promise<LangiumDocument[]> => {
+    const documents = uris.map((uri) => services.shared.workspace.LangiumDocuments.getOrCreateDocument(uri));
+    await services.shared.workspace.DocumentBuilder.build(documents, options);
+    return documents;
+};

tests/helpers/testResources.ts

@@ -1,12 +1,13 @@
 import {
     listTestPythonFiles,
     listTestSafeDsFilesGroupedByParentDirectory,
+    loadDocuments,
     uriToShortenedTestResourceName,
 } from '../../helpers/testResources.js';
 import path from 'path';
 import fs from 'fs';
 import { createSafeDsServices } from '../../../src/language/safe-ds-module.js';
-import { ErrorsInCodeError, getErrors } from '../../helpers/diagnostics.js';
+import { ErrorsInCodeError, getErrorsAtURI } from '../../helpers/diagnostics.js';
 import { findTestChecks } from '../../helpers/testChecks.js';
 import { Location } from 'vscode-languageserver';
 import { NodeFileSystem } from 'langium/node';
@@ -31,11 +32,14 @@ const createGenerationTest = async (parentDirectory: URI, inputUris: URI[]): Pro
     const expectedOutputFiles = readExpectedOutputFiles(expectedOutputRoot, actualOutputRoot);
     let runUntil: Location | undefined;
 
+    // Load all files, so they get linked
+    await loadDocuments(services, inputUris, { validation: true });
+
     for (const uri of inputUris) {
-        const code = fs.readFileSync(uri.fsPath).toString();
+        const code = services.shared.workspace.LangiumDocuments.getOrCreateDocument(uri).textDocument.getText();
 
         // File must not contain any errors
-        const errors = await getErrors(services, code);
+        const errors = getErrorsAtURI(services, uri);
         if (errors.length > 0) {
             return invalidTest('FILE', new ErrorsInCodeError(errors, uri));
         }

tests/language/generation/creator.ts

@@ -6,6 +6,7 @@ import { createGenerationTests } from './creator.js';
 import { SdsModule } from '../../../src/language/generated/ast.js';
 import { generatePython } from '../../../src/cli/generator.js';
 import fs from 'fs';
+import { loadDocuments } from '../../helpers/testResources.js';
 
 const services = createSafeDsServices(NodeFileSystem).SafeDs;
 const generationTests = createGenerationTests();
@@ -27,18 +28,15 @@ describe('generation', async () => {
         }
 
         // Load all documents
-        const documents = test.inputUris.map((uri) =>
-            services.shared.workspace.LangiumDocuments.getOrCreateDocument(uri),
-        );
-        await services.shared.workspace.DocumentBuilder.build(documents);
+        const documents = await loadDocuments(services, test.inputUris);
 
         // Generate code for all documents
         const actualOutputPaths: string[] = [];
 
         for (const document of documents) {
             const module = document.parseResult.value as SdsModule;
             const fileName = document.uri.fsPath;
-            const generatedFilePaths = generatePython(module, fileName, test.actualOutputRoot.fsPath);
+            const generatedFilePaths = generatePython(services, module, fileName, test.actualOutputRoot.fsPath);
             actualOutputPaths.push(...generatedFilePaths);
         }
 

tests/language/generation/testGeneration.test.ts

@@ -6,6 +6,7 @@ import { AssertionError } from 'assert';
 import { locationToString } from '../../helpers/location.js';
 import { createPartialEvaluationTests } from './creator.js';
 import { getNodeByLocation } from '../../helpers/nodeFinder.js';
+import { loadDocuments } from '../../helpers/testResources.js';
 
 const services = createSafeDsServices(NodeFileSystem).SafeDs;
 const partialEvaluator = services.evaluation.PartialEvaluator;
@@ -22,8 +23,7 @@ describe('partial evaluation', async () => {
         }
 
         // Load all documents
-        const documents = test.uris.map((uri) => services.shared.workspace.LangiumDocuments.getOrCreateDocument(uri));
-        await services.shared.workspace.DocumentBuilder.build(documents);
+        await loadDocuments(services, test.uris);
 
         // Ensure that partially evaluating nodes in the same equivalence class yields the same result
         for (const equivalenceClassAssertion of test.equivalenceClassAssertions) {

tests/language/partialEvaluation/testPartialEvaluation.test.ts

@@ -7,6 +7,7 @@ import { AssertionError } from 'assert';
 import { isLocationEqual, locationToString } from '../../helpers/location.js';
 import { createScopingTests, ExpectedReference } from './creator.js';
 import { Location } from 'vscode-languageserver';
+import { loadDocuments } from '../../helpers/testResources.js';
 
 const services = createSafeDsServices(NodeFileSystem).SafeDs;
 
@@ -27,8 +28,7 @@ describe('scoping', async () => {
         }
 
         // Load all documents
-        const documents = test.uris.map((uri) => services.shared.workspace.LangiumDocuments.getOrCreateDocument(uri));
-        await services.shared.workspace.DocumentBuilder.build(documents);
+        await loadDocuments(services, test.uris);
 
         // Ensure all expected references match
         for (const expectedReference of test.expectedReferences) {

tests/language/scoping/testScoping.test.ts

@@ -1,14 +1,14 @@
 import {
     listTestSafeDsFilesGroupedByParentDirectory,
     uriToShortenedTestResourceName,
-} from '../../helpers/testResources.js';
+} from '../../../helpers/testResources.js';
 import fs from 'fs';
-import { findTestChecks } from '../../helpers/testChecks.js';
+import { findTestChecks } from '../../../helpers/testChecks.js';
 import { Location } from 'vscode-languageserver';
-import { getSyntaxErrors, SyntaxErrorsInCodeError } from '../../helpers/diagnostics.js';
+import { getSyntaxErrors, SyntaxErrorsInCodeError } from '../../../helpers/diagnostics.js';
 import { EmptyFileSystem, URI } from 'langium';
-import { createSafeDsServices } from '../../../src/language/safe-ds-module.js';
-import { TestDescription, TestDescriptionError } from '../../helpers/testDescription.js';
+import { createSafeDsServices } from '../../../../src/language/safe-ds-module.js';
+import { TestDescription, TestDescriptionError } from '../../../helpers/testDescription.js';
 
 const services = createSafeDsServices(EmptyFileSystem).SafeDs;
 const rootResourceName = 'typing';

tests/language/typing/creator.ts renamed to tests/language/typing/type computer/creator.ts

@@ -1,11 +1,12 @@
 import { afterEach, beforeEach, describe, it } from 'vitest';
-import { createSafeDsServices } from '../../../src/language/safe-ds-module.js';
+import { createSafeDsServices } from '../../../../src/language/safe-ds-module.js';
 import { NodeFileSystem } from 'langium/node';
 import { clearDocuments } from 'langium/test';
 import { AssertionError } from 'assert';
-import { locationToString } from '../../helpers/location.js';
+import { locationToString } from '../../../helpers/location.js';
 import { createTypingTests } from './creator.js';
-import { getNodeByLocation } from '../../helpers/nodeFinder.js';
+import { getNodeByLocation } from '../../../helpers/nodeFinder.js';
+import { loadDocuments } from '../../../helpers/testResources.js';
 
 const services = createSafeDsServices(NodeFileSystem).SafeDs;
 const typeComputer = services.types.TypeComputer;
@@ -27,8 +28,7 @@ describe('typing', async () => {
         }
 
         // Load all documents
-        const documents = test.uris.map((uri) => services.shared.workspace.LangiumDocuments.getOrCreateDocument(uri));
-        await services.shared.workspace.DocumentBuilder.build(documents);
+        await loadDocuments(services, test.uris);
 
         // Ensure all nodes in the equivalence class have the same type
         for (const equivalenceClassAssertion of test.equivalenceClassAssertions) {

tests/language/typing/testTyping.test.ts renamed to tests/language/typing/type computer/testTyping.test.ts

@@ -6,6 +6,7 @@ import { Diagnostic, DiagnosticSeverity } from 'vscode-languageserver';
 import { AssertionError } from 'assert';
 import { clearDocuments, isRangeEqual } from 'langium/test';
 import { locationToString } from '../../helpers/location.js';
+import { loadDocuments } from '../../helpers/testResources.js';
 
 const services = createSafeDsServices(NodeFileSystem).SafeDs;
 
@@ -26,8 +27,7 @@ describe('validation', async () => {
         }
 
         // Load all documents
-        const documents = test.uris.map((uri) => services.shared.workspace.LangiumDocuments.getOrCreateDocument(uri));
-        await services.shared.workspace.DocumentBuilder.build(documents, { validation: true });
+        await loadDocuments(services, test.uris, { validation: true });
 
         // Ensure all expected issues match
         for (const expectedIssue of test.expectedIssues) {

tests/language/validation/testValidation.test.ts

@@ -0,0 +1,9 @@
+package tests.generator.parameterWithPythonName
+
+fun f1(param: (a: Int, b: Int, c: Int) -> r: Int)
+fun f2(param: (a: Int, b: Int, c: Int) -> ())
+
+segment test(param1: Int, @PythonName("param_2") param2: Int, @PythonName("param_3") param3: Int = 0) {
+    f1((param1: Int, param2: Int, param3: Int = 0) -> 1);
+    f2((param1: Int, param2: Int, param3: Int = 0) {});
+}

tests/resources/generation/skip-declarations/empty pipeline/input.sdstest renamed to tests/resources/generation/declarations/empty pipeline/input.sdstest

@@ -0,0 +1,7 @@
+# Steps ------------------------------------------------------------------------
+
+def test(param1, param_2, param_3=0):
+    f1(lambda param1, param2, param3=0: 1)
+    def __gen_block_lambda_0(param1, param2, param3=0):
+        pass
+    f2(__gen_block_lambda_0)

tests/resources/generation/skip-declarations/empty pipeline/output/tests/generator/emptyPipeline/gen_input.py renamed to tests/resources/generation/declarations/empty pipeline/output/tests/generator/emptyPipeline/gen_input.py

@@ -6,6 +6,6 @@ segment test1(a: Int, b: Int = 0) {
     f();
 }
 
-segment test2(a: Int, vararg c: Int) {
+segment test2(a: Int, c: Int) {
     f();
 }

tests/resources/generation/skip-declarations/empty pipeline/output/tests/generator/emptyPipeline/gen_input_test.py renamed to tests/resources/generation/declarations/empty pipeline/output/tests/generator/emptyPipeline/gen_input_test.py

@@ -3,5 +3,5 @@
 def test1(a, b=0):
     f()
 
-def test2(a, *c):
+def test2(a, c):
     f()

tests/resources/generation/skip-declarations/empty step/input.sdstest renamed to tests/resources/generation/declarations/empty step/input.sdstest

@@ -0,0 +1,26 @@
+package tests.generator.blockLambdaResult
+
+fun g() -> a: Int
+
+fun h(a: Int)
+
+segment f1(l: (a: Int, b: Int) -> d: Int) {
+    h(l(1, 2).d);
+}
+
+segment f2(l: (a: Int, b: Int) -> (d1: Int, e1: Int)) {
+    h(l(1, 2).e1);
+    h(l(1, 2).d1);
+}
+
+pipeline test {
+
+    f1((a: Int, b: Int) {
+        yield d = g();
+    });
+    f2((a: Int, b: Int) {
+        yield d = g();
+        yield e = g();
+    });
+
+}

tests/resources/generation/skip-declarations/empty step/output/tests/generator/emptyStep/gen_input.py renamed to tests/resources/generation/declarations/empty step/output/tests/generator/emptyStep/gen_input.py

@@ -0,0 +1,21 @@
+# Steps ------------------------------------------------------------------------
+
+def f1(l):
+    h(l(1, 2))
+
+def f2(l):
+    h(l(1, 2)[1])
+    h(l(1, 2)[0])
+
+# Pipelines --------------------------------------------------------------------
+
+def test():
+    def __gen_block_lambda_0(a, b):
+        __gen_block_lambda_result_d = g()
+        return __gen_block_lambda_result_d
+    f1(__gen_block_lambda_0)
+    def __gen_block_lambda_1(a, b):
+        __gen_block_lambda_result_d = g()
+        __gen_block_lambda_result_e = g()
+        return __gen_block_lambda_result_d, __gen_block_lambda_result_e
+    f2(__gen_block_lambda_1)

tests/resources/generation/declarations/parameter with python name/input.sdstest

@@ -1,17 +1,16 @@
 package tests.generator.blockLambda
 
 fun f1(param: (a: Int, b: Int) -> r: Int)
-fun f2(param: (a: Int, vararg b: Int) -> r: Int)
-fun f3(param: () -> ())
+fun f2(param: () -> ())
 
 fun g() -> a: Int
 
 pipeline test {
     f1((a: Int, b: Int = 2) {
         yield d = g();
     });
-    f2((a: Int, vararg c: Int) {
+    f1((a: Int, c: Int) {
         yield d = g();
     });
-    f3(() {});
+    f2(() {});
 }