diff --git a/libs/execution/src/lib/types/io-types/filesystem-node-file-text.ts b/libs/execution/src/lib/types/io-types/filesystem-node-file-text.ts index cebccb55..4b44a3b1 100644 --- a/libs/execution/src/lib/types/io-types/filesystem-node-file-text.ts +++ b/libs/execution/src/lib/types/io-types/filesystem-node-file-text.ts @@ -11,7 +11,7 @@ import { } from './io-type-implementation'; export class TextFile - extends FileSystemFile + extends FileSystemFile implements IOTypeImplementation { public readonly ioType = IOType.TEXT_FILE; diff --git a/libs/execution/src/lib/util/file-util.spec.ts b/libs/execution/src/lib/util/file-util.spec.ts index 5d307415..7db1bb1f 100644 --- a/libs/execution/src/lib/util/file-util.spec.ts +++ b/libs/execution/src/lib/util/file-util.spec.ts @@ -2,14 +2,25 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { FileExtension, MimeType } from '../types'; +import * as R from '../blocks'; +import { FileExtension, MimeType, TextFile } from '../types'; import { inferFileExtensionFromContentTypeString, inferFileExtensionFromFileExtensionString, inferMimeTypeFromFileExtensionString, + transformTextFileLines, } from './file-util'; +function exampleTextFile(content: string): TextFile { + return new TextFile( + 'exampleTextFile', + FileExtension.TXT, + MimeType.TEXT_PLAIN, + content, + ); +} + describe('Validation of file-util', () => { describe('Function inferMimeTypeFromContentTypeString', () => { it('should diagnose no error on known mimeType', () => { @@ -68,4 +79,56 @@ describe('Validation of file-util', () => { expect(result).toEqual(undefined); }); }); + describe('Function transformTextFileLines', () => { + it('should diagnose no error without newline', async () => { + const file = exampleTextFile('some text content without a newline'); + // eslint-disable-next-line @typescript-eslint/require-await + const spy = vi.fn(async (lines: string[]) => R.ok(lines)); + const result = await transformTextFileLines(file, /\r?\n/, spy); + + expect(spy).toHaveBeenCalledOnce(); + expect(spy).toHaveBeenCalledWith(['some text content without a newline']); + + expect(R.isOk(result)).toBe(true); + assert(R.isOk(result)); + + expect(result.right).toStrictEqual(file); + }); + it('should diagnose no error on empty file', async () => { + const file = exampleTextFile(''); + + // eslint-disable-next-line @typescript-eslint/require-await + const spy = vi.fn(async (lines: string[]) => R.ok(lines)); + const result = await transformTextFileLines(file, /\r?\n/, spy); + + expect(spy).toHaveBeenCalledOnce(); + expect(spy).toHaveBeenCalledWith([]); + + expect(R.isOk(result)).toBe(true); + assert(R.isOk(result)); + + expect(result.right).toStrictEqual(file); + }); + it('should diagnose no error on file with trailing newline', async () => { + const file = exampleTextFile(`some text content +with a +trailing newline +`); + // eslint-disable-next-line @typescript-eslint/require-await + const spy = vi.fn(async (lines: string[]) => R.ok(lines)); + const result = await transformTextFileLines(file, /\r?\n/, spy); + + expect(spy).toHaveBeenCalledOnce(); + expect(spy).toHaveBeenCalledWith([ + 'some text content', + 'with a ', + 'trailing newline', + ]); + + expect(R.isOk(result)).toBe(true); + assert(R.isOk(result)); + + expect(result.right).toStrictEqual(file); + }); + }); }); diff --git a/libs/execution/src/lib/util/file-util.ts b/libs/execution/src/lib/util/file-util.ts index bab374ac..3f4b9f38 100644 --- a/libs/execution/src/lib/util/file-util.ts +++ b/libs/execution/src/lib/util/file-util.ts @@ -4,7 +4,8 @@ import * as mime from 'mime-types'; -import { FileExtension, MimeType } from '../types'; +import * as R from '../blocks'; +import { FileExtension, MimeType, TextFile } from '../types'; export function inferMimeTypeFromFileExtensionString( fileExtension: string | undefined, @@ -50,3 +51,33 @@ export function inferFileExtensionFromContentTypeString( } return undefined; } + +export async function transformTextFileLines( + file: TextFile, + lineBreakPattern: RegExp, + transformFn: (lines: string[]) => Promise>, +): Promise> { + const lines = file.content.split(lineBreakPattern); + const lineBreak = file.content.match(lineBreakPattern)?.at(0) ?? ''; + + // There may be an additional empty line due to the previous splitting + let emptyNewline = false; + if (lines[lines.length - 1] === '') { + emptyNewline = true; + lines.pop(); + } + + const newLines = await transformFn(lines); + if (R.isErr(newLines)) { + return newLines; + } + + let newContent = newLines.right.join(lineBreak); + if (emptyNewline) { + newContent += lineBreak; + } + + return R.ok( + new TextFile(file.name, file.extension, file.mimeType, newContent), + ); +} diff --git a/libs/execution/src/lib/util/index.ts b/libs/execution/src/lib/util/index.ts index 9e67d5d9..66a9803b 100644 --- a/libs/execution/src/lib/util/index.ts +++ b/libs/execution/src/lib/util/index.ts @@ -4,4 +4,3 @@ export * from './implements-static-decorator'; export * from './file-util'; -export * from './string-util'; diff --git a/libs/execution/src/lib/util/string-util.ts b/libs/execution/src/lib/util/string-util.ts deleted file mode 100644 index 55b7d78b..00000000 --- a/libs/execution/src/lib/util/string-util.ts +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -export function splitLines(textContent: string, lineBreak: RegExp): string[] { - const lines = textContent.split(lineBreak); - - // There may be an additional empty line due to the previous splitting - if (lines[lines.length - 1] === '') { - lines.splice(lines.length - 1, 1); - } - - return lines; -} diff --git a/libs/execution/test/utils/file-util.ts b/libs/execution/test/utils/file-util.ts index 577812f5..ae0d68ec 100644 --- a/libs/execution/test/utils/file-util.ts +++ b/libs/execution/test/utils/file-util.ts @@ -12,7 +12,6 @@ import { TextFile, inferFileExtensionFromFileExtensionString, inferMimeTypeFromFileExtensionString, - splitLines, } from '../../src'; export function createBinaryFileFromLocalFile(fileName: string): BinaryFile { @@ -39,6 +38,6 @@ export function createTextFileFromLocalFile(fileName: string): TextFile { path.basename(fileName), fileExtension, mimeType, - splitLines(fileContent, /\r?\n/), + fileContent, ); } diff --git a/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts b/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts index 2dcbe314..7962f749 100644 --- a/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts +++ b/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts @@ -92,9 +92,9 @@ describe('Validation of TextFileInterpreterExecutor', () => { expect(R.isErr(result)).toEqual(false); if (R.isOk(result)) { expect(result.right.ioType).toEqual(IOType.TEXT_FILE); - expect(result.right.content).toEqual( - expect.arrayContaining(['Multiline ', 'Test File']), - ); + expect(result.right.content).toBe(`Multiline +Test File +`); } }); @@ -107,24 +107,18 @@ describe('Validation of TextFileInterpreterExecutor', () => { expect(R.isErr(result)).toEqual(false); if (R.isOk(result)) { expect(result.right.ioType).toEqual(IOType.TEXT_FILE); - expect(result.right.content).toEqual( - expect.arrayContaining(['vehicle:268435857"0']), - ); - } - }); - - it('should diagnose no error on custom lineBreak', async () => { - const text = readJvTestAsset('valid-custom-line-break.jv'); - - const testFile = readTestFile('test.txt'); - const result = await parseAndExecuteExecutor(text, testFile); - - expect(R.isErr(result)).toEqual(false); - if (R.isOk(result)) { - expect(result.right.ioType).toEqual(IOType.TEXT_FILE); - expect(result.right.content).toEqual( - expect.arrayContaining(['Multiline \nTest', 'File\n']), - ); + const expectedBytes = Buffer.from([ + 0xa, 0xd, 0xa, 0x3, 0x32, 0x2e, 0x30, 0x10, 0x0, 0x18, 0xe9, 0xa9, 0xba, + 0xef, 0xbf, 0xbd, 0x6, 0x12, 0x45, 0xa, 0x11, 0x76, 0x65, 0x68, 0x69, + 0x63, 0x6c, 0x65, 0x3a, 0x32, 0x36, 0x38, 0x34, 0x33, 0x35, 0x38, 0x35, + 0x37, 0x22, 0x30, 0xa, 0xe, 0xa, 0x8, 0x31, 0x35, 0x39, 0x32, 0x33, + 0x34, 0x37, 0x34, 0x2a, 0x2, 0x31, 0x30, 0x12, 0xf, 0xd, 0x27, 0xef, + 0xbf, 0xbd, 0x39, 0x42, 0x15, 0xef, 0xbf, 0xbd, 0xf, 0x1f, 0xef, 0xbf, + 0xbd, 0x1d, 0x0, 0x0, 0x2c, 0x43, 0x28, 0x0, 0x42, 0xb, 0xa, 0x9, 0x32, + 0x36, 0x38, 0x34, 0x33, 0x35, 0x38, 0x35, 0x37, + ]); + const actualBytes = Buffer.from(result.right.content); + expect(actualBytes).toStrictEqual(expectedBytes); } }); }); diff --git a/libs/extensions/std/exec/src/text-file-interpreter-executor.ts b/libs/extensions/std/exec/src/text-file-interpreter-executor.ts index b7417507..0f344cae 100644 --- a/libs/extensions/std/exec/src/text-file-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/text-file-interpreter-executor.ts @@ -12,7 +12,6 @@ import { type ExecutionContext, TextFile, implementsStatic, - splitLines, } from '@jvalue/jayvee-execution'; import { IOType } from '@jvalue/jayvee-language-server'; @@ -36,10 +35,6 @@ export class TextFileInterpreterExecutor extends AbstractBlockExecutor< 'encoding', context.valueTypeProvider.Primitives.Text, ); - const lineBreak = context.getPropertyValue( - 'lineBreak', - context.valueTypeProvider.Primitives.Regex, - ); const decoder = new TextDecoder(encoding); context.logger.logDebug( @@ -47,14 +42,8 @@ export class TextFileInterpreterExecutor extends AbstractBlockExecutor< ); const textContent = decoder.decode(file.content); - context.logger.logDebug( - `Splitting lines using line break /${lineBreak.source}/`, + return R.ok( + new TextFile(file.name, file.extension, file.mimeType, textContent), ); - const lines = splitLines(textContent, lineBreak); - context.logger.logDebug( - `Lines were split successfully, the resulting text file has ${lines.length} lines`, - ); - - return R.ok(new TextFile(file.name, file.extension, file.mimeType, lines)); } } diff --git a/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts b/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts index adeb9158..12bbd5f6 100644 --- a/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts +++ b/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts @@ -92,8 +92,9 @@ describe('Validation of TextLineDeleterExecutor', () => { expect(R.isErr(result)).toEqual(false); if (R.isOk(result)) { expect(result.right.ioType).toEqual(IOType.TEXT_FILE); - expect(result.right.content).toEqual( - expect.arrayContaining(['Test File']), + expect(result.right.content).toBe( + `Test File +`, ); } }); @@ -107,8 +108,10 @@ describe('Validation of TextLineDeleterExecutor', () => { expect(R.isErr(result)).toEqual(false); if (R.isOk(result)) { expect(result.right.ioType).toEqual(IOType.TEXT_FILE); - expect(result.right.content).toEqual( - expect.arrayContaining(['Multiline', 'Test File']), + expect(result.right.content).toBe( + `Multiline +Test File +`, ); } }); @@ -136,8 +139,9 @@ describe('Validation of TextLineDeleterExecutor', () => { expect(R.isErr(result)).toEqual(false); if (R.isOk(result)) { expect(result.right.ioType).toEqual(IOType.TEXT_FILE); - expect(result.right.content).toEqual( - expect.arrayContaining(['Test File']), + expect(result.right.content).toBe( + `Test File +`, ); } }); diff --git a/libs/extensions/std/exec/src/text-line-deleter-executor.ts b/libs/extensions/std/exec/src/text-line-deleter-executor.ts index c1fa2179..8ba1ba95 100644 --- a/libs/extensions/std/exec/src/text-line-deleter-executor.ts +++ b/libs/extensions/std/exec/src/text-line-deleter-executor.ts @@ -12,6 +12,40 @@ import { } from '@jvalue/jayvee-execution'; import { IOType } from '@jvalue/jayvee-language-server'; +// eslint-disable-next-line @typescript-eslint/require-await +async function deleteLines( + lines: string[], + deleteIdxs: number[], + context: ExecutionContext, +): Promise> { + let lineIdx = 0; + for (const deleteIdx of deleteIdxs) { + if (deleteIdx > lines.length) { + return R.err({ + message: `Line ${deleteIdx} does not exist in the text file, only ${lines.length} line(s) are present`, + diagnostic: { + node: context.getOrFailProperty('lines').value, + property: 'values', + index: lineIdx, + }, + }); + } + ++lineIdx; + } + + const distinctLines = new Set(deleteIdxs); + const sortedLines = [...distinctLines].sort((a, b) => a - b); + + context.logger.logDebug(`Deleting line(s) ${sortedLines.join(', ')}`); + + const reversedLines = sortedLines.reverse(); + for (const lineToDelete of reversedLines) { + lines.splice(lineToDelete - 1, 1); + } + + return R.ok(lines); +} + @implementsStatic() export class TextLineDeleterExecutor extends AbstractBlockExecutor< IOType.TEXT_FILE, @@ -23,48 +57,23 @@ export class TextLineDeleterExecutor extends AbstractBlockExecutor< super(IOType.TEXT_FILE, IOType.TEXT_FILE); } - // eslint-disable-next-line @typescript-eslint/require-await async doExecute( file: TextFile, context: ExecutionContext, ): Promise> { - const lines = context.getPropertyValue( + const deleteIdxs = context.getPropertyValue( 'lines', context.valueTypeProvider.createCollectionValueTypeOf( context.valueTypeProvider.Primitives.Integer, ), ); - const numberOfLines = file.content.length; - - let lineIndex = 0; - for (const lineNumber of lines) { - if (lineNumber > numberOfLines) { - return R.err({ - message: `Line ${lineNumber} does not exist in the text file, only ${file.content.length} line(s) are present`, - diagnostic: { - node: context.getOrFailProperty('lines').value, - property: 'values', - index: lineIndex, - }, - }); - } - - ++lineIndex; - } - - const distinctLines = new Set(lines); - const sortedLines = [...distinctLines].sort((a, b) => a - b); - - context.logger.logDebug(`Deleting line(s) ${sortedLines.join(', ')}`); - - const reversedLines = sortedLines.reverse(); - const newContent = [...file.content]; - for (const lineToDelete of reversedLines) { - newContent.splice(lineToDelete - 1, 1); - } + const lineBreakPattern = context.getPropertyValue( + 'lineBreak', + context.valueTypeProvider.Primitives.Regex, + ); - return R.ok( - new TextFile(file.name, file.extension, file.mimeType, newContent), + return R.transformTextFileLines(file, lineBreakPattern, (lines) => + deleteLines(lines, deleteIdxs, context), ); } } diff --git a/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts b/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts index 29a0eac4..f6c62f9f 100644 --- a/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts +++ b/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts @@ -93,7 +93,9 @@ describe('Validation of TextRangeSelectorExecutor', () => { if (R.isOk(result)) { expect(result.right.ioType).toEqual(IOType.TEXT_FILE); expect(result.right.content).toEqual( - expect.arrayContaining(['Multiline', 'Test ']), + `Multiline +Test +`, ); } }); diff --git a/libs/extensions/std/exec/src/text-range-selector-executor.ts b/libs/extensions/std/exec/src/text-range-selector-executor.ts index da71adaa..de5bb926 100644 --- a/libs/extensions/std/exec/src/text-range-selector-executor.ts +++ b/libs/extensions/std/exec/src/text-range-selector-executor.ts @@ -23,7 +23,6 @@ export class TextRangeSelectorExecutor extends AbstractBlockExecutor< super(IOType.TEXT_FILE, IOType.TEXT_FILE); } - // eslint-disable-next-line @typescript-eslint/require-await async doExecute( file: TextFile, context: ExecutionContext, @@ -36,20 +35,21 @@ export class TextRangeSelectorExecutor extends AbstractBlockExecutor< 'lineTo', context.valueTypeProvider.Primitives.Integer, ); - - const numberOfLines = file.content.length; - - context.logger.logDebug( - `Selecting lines from ${lineFrom} to ${ - lineTo === Number.MAX_SAFE_INTEGER || lineTo >= numberOfLines - ? 'the end' - : `${lineTo}` - }`, + const lineBreakPattern = context.getPropertyValue( + 'lineBreak', + context.valueTypeProvider.Primitives.Regex, ); - const selectedLines = file.content.slice(lineFrom - 1, lineTo); - return R.ok( - new TextFile(file.name, file.extension, file.mimeType, selectedLines), - ); + // eslint-disable-next-line @typescript-eslint/require-await + return R.transformTextFileLines(file, lineBreakPattern, async (lines) => { + context.logger.logDebug( + `Selecting lines from ${lineFrom} to ${ + lineTo === Number.MAX_SAFE_INTEGER || lineTo >= lines.length + ? 'the end' + : `${lineTo}` + }`, + ); + return R.ok(lines.slice(lineFrom - 1, lineTo)); + }); } } diff --git a/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts index 47be1e61..a3519014 100644 --- a/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts @@ -100,4 +100,27 @@ describe('Validation of CSVInterpreterExecutor', () => { ]); } }); + + it('should correctly parse newlines within csv string values', async () => { + const text = readJvTestAsset('csv-with-newline-interpreter.jv'); + + const testCsv = readTestFile('csv-with-newline.csv'); + const result = await parseAndExecuteExecutor(text, testCsv); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(2); + expect(result.right.getData()).toEqual([ + ['C1', 'C2', 'C3'], + [ + '2', + `some +text`, + 'true', + ], + ]); + } + }); }); diff --git a/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.ts b/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.ts index 6671f964..4c727046 100644 --- a/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.ts @@ -2,7 +2,10 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { parseString as parseStringAsCsv } from '@fast-csv/parse'; +// eslint-disable-next-line unicorn/prefer-node-protocol +import assert from 'assert'; + +import { Row, parseString as parseStringAsCsv } from '@fast-csv/parse'; import { type ParserOptionsArgs } from '@fast-csv/parse/build/src/ParserOptions'; import * as R from '@jvalue/jayvee-execution'; import { @@ -14,7 +17,6 @@ import { implementsStatic, } from '@jvalue/jayvee-execution'; import { IOType } from '@jvalue/jayvee-language-server'; -import { either as E } from 'fp-ts'; @implementsStatic() export class CSVInterpreterExecutor extends AbstractBlockExecutor< @@ -53,56 +55,51 @@ export class CSVInterpreterExecutor extends AbstractBlockExecutor< quote: enclosing, escape: enclosingEscape, }; - const csvData = await parseAsCsv(file.content, parseOptions); - - if (E.isLeft(csvData)) { - return Promise.resolve( - R.err({ - message: `CSV parse failed in line ${csvData.left.lineNumber}: ${csvData.left.error.message}`, - diagnostic: { node: context.getCurrentNode(), property: 'name' }, - }), - ); - } - const sheet = new Sheet(csvData.right); - - context.logger.logDebug(`Parsing raw data as CSV-sheet successful`); - return Promise.resolve(R.ok(sheet)); - } -} - -async function parseAsCsv( - lines: string[], - parseOptions: ParserOptionsArgs, -): Promise> { - let lineNumber = 1; - const rows: string[][] = []; - for await (const line of lines) { - const rowParseResult = await parseLineAsRow(line, parseOptions); - if (E.isLeft(rowParseResult)) { - return E.left({ error: rowParseResult.left, lineNumber }); - } - rows.push(rowParseResult.right); - - ++lineNumber; + return parseAsCSV(file.content, parseOptions).then( + (csvData) => { + context.logger.logDebug(`Parsing raw data as CSV-sheet successful`); + return R.ok(new Sheet(csvData)); + }, + (reason) => { + assert(typeof reason === 'string'); + return R.err({ + message: reason, + diagnostic: { + node: context.getCurrentNode(), + property: 'name', + }, + }); + }, + ); } - return E.right(rows); } -async function parseLineAsRow( - line: string, +async function parseAsCSV( + text: string, parseOptions: ParserOptionsArgs, -): Promise> { - return new Promise((resolve) => { - let row: string[]; - parseStringAsCsv(line, parseOptions) - .on('data', (data: string[]) => { - row = data; - }) - .on('error', (error) => { - resolve(E.left(error)); +): Promise { + return new Promise((resolve, reject) => { + const rows: string[][] = []; + parseStringAsCsv(text, parseOptions) + .on('data', (row: string[]) => { + rows.push(row); }) + .on('error', (error) => + reject( + `Unexpected error while parsing CSV: ${error.name}: ${error.message}`, + ), + ) + .on( + 'data-invalid', + (row: Row | null, rowCount: number, reason?: string) => + reject( + `Invalid row ${rowCount}: ${ + reason ?? 'Unknwon reason' + }: ${JSON.stringify(row ?? '')}`, + ), + ) .on('end', () => { - resolve(E.right(row)); + resolve(rows); }); }); } diff --git a/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/csv-with-newline-interpreter.jv b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/csv-with-newline-interpreter.jv new file mode 100644 index 00000000..f5d34fd1 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/csv-with-newline-interpreter.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2025 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestTextFileExtractor { } + + block TestBlock oftype CSVInterpreter { + enclosing: '"'; + } + + block TestLoader oftype TestSheetLoader { } + + TestExtractor + -> TestBlock + -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/csv-with-newline.csv b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/csv-with-newline.csv new file mode 100644 index 00000000..882eb54e --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/csv-with-newline.csv @@ -0,0 +1,3 @@ +C1,C2,C3 +2,"some +text",true diff --git a/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/csv-with-newline.csv.license b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/csv-with-newline.csv.license new file mode 100644 index 00000000..e39adc51 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/csv-with-newline.csv.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2025 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/language-server/src/stdlib/builtin-block-types/CsvInterpreter.jv b/libs/language-server/src/stdlib/builtin-block-types/CsvInterpreter.jv index 911db64a..dae6de4d 100644 --- a/libs/language-server/src/stdlib/builtin-block-types/CsvInterpreter.jv +++ b/libs/language-server/src/stdlib/builtin-block-types/CsvInterpreter.jv @@ -4,28 +4,28 @@ /** * Interprets an input file as a csv-file containing string-values delimited by `delimiter` and outputs a `Sheet`. -* +* * @example Interprets an input file as a csv-file containing string-values delimited by `;` and outputs `Sheet`. -* block AgencyCSVInterpreter oftype CSVInterpreter { +* block AgencyCSVInterpreter oftype CSVInterpreter { * delimiter: ";"; * } */ publish builtin blocktype CSVInterpreter { input default oftype TextFile; output default oftype Sheet; - + /** * The delimiter for values in the CSV file. */ property delimiter oftype text: ','; - + /** * The enclosing character that may be used for values in the CSV file. */ property enclosing oftype text: ''; - + /** * The character to escape enclosing characters in values. */ property enclosingEscape oftype text: ''; -} \ No newline at end of file +} diff --git a/libs/language-server/src/stdlib/builtin-block-types/TextFileInterpreter.jv b/libs/language-server/src/stdlib/builtin-block-types/TextFileInterpreter.jv index 2b04556a..d3b7fe06 100644 --- a/libs/language-server/src/stdlib/builtin-block-types/TextFileInterpreter.jv +++ b/libs/language-server/src/stdlib/builtin-block-types/TextFileInterpreter.jv @@ -8,14 +8,9 @@ publish builtin blocktype TextFileInterpreter { input default oftype File; output default oftype TextFile; - + /** * The encoding used for decoding the file contents. */ property encoding oftype text: 'utf-8'; - - /** - * The regex for identifying line breaks. - */ - property lineBreak oftype Regex: /\r?\n/; } diff --git a/libs/language-server/src/stdlib/builtin-block-types/TextLineDeleter.jv b/libs/language-server/src/stdlib/builtin-block-types/TextLineDeleter.jv index 30309d33..9794d91b 100644 --- a/libs/language-server/src/stdlib/builtin-block-types/TextLineDeleter.jv +++ b/libs/language-server/src/stdlib/builtin-block-types/TextLineDeleter.jv @@ -8,9 +8,14 @@ publish builtin blocktype TextLineDeleter { input default oftype TextFile; output default oftype TextFile; - + /** * The line numbers to delete. */ property lines oftype Collection; + + /** + * The regex for identifying line breaks. + */ + property lineBreak oftype Regex: /\r?\n/; } diff --git a/libs/language-server/src/stdlib/builtin-block-types/TextRangeSelector.jv b/libs/language-server/src/stdlib/builtin-block-types/TextRangeSelector.jv index 05ca216e..ecba77f8 100644 --- a/libs/language-server/src/stdlib/builtin-block-types/TextRangeSelector.jv +++ b/libs/language-server/src/stdlib/builtin-block-types/TextRangeSelector.jv @@ -19,4 +19,9 @@ publish builtin blocktype TextRangeSelector { * The default value is the biggest usable integer. */ property lineTo oftype integer: 9007199254740991; -} \ No newline at end of file + + /** + * The regex for identifying line breaks. + */ + property lineBreak oftype Regex: /\r?\n/; +}