From 2920a04b4a73cb290bcfd644dfd57d18453baa58 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 19 Jun 2022 14:18:57 +0200 Subject: [PATCH 01/11] Add YAML parser to language service --- server/package-lock.json | 23 +- server/packages/server-common/package.json | 7 +- server/packages/server-common/src/astTypes.ts | 23 ++ .../yaml-language-service/jest.config.js | 25 ++ .../yaml-language-service/package.json | 4 +- .../yaml-language-service/src/index.ts | 4 + .../src/parser/astConverter.ts | 232 ++++++++++++++++++ .../src/parser/astTypes.ts | 174 +++++++++++++ .../yaml-language-service/src/parser/index.ts | 47 ++++ .../src/parser/yamlDocument.ts | 66 +++++ .../src/services/yamlValidation.ts | 27 ++ .../src/utils/textBuffer.ts | 56 +++++ .../src/yamlLanguageService.ts | 19 +- 13 files changed, 693 insertions(+), 14 deletions(-) create mode 100644 server/packages/server-common/src/astTypes.ts create mode 100644 server/packages/yaml-language-service/jest.config.js create mode 100644 server/packages/yaml-language-service/src/index.ts create mode 100644 server/packages/yaml-language-service/src/parser/astConverter.ts create mode 100644 server/packages/yaml-language-service/src/parser/astTypes.ts create mode 100644 server/packages/yaml-language-service/src/parser/index.ts create mode 100644 server/packages/yaml-language-service/src/parser/yamlDocument.ts create mode 100644 server/packages/yaml-language-service/src/services/yamlValidation.ts create mode 100644 server/packages/yaml-language-service/src/utils/textBuffer.ts diff --git a/server/package-lock.json b/server/package-lock.json index 36cc5ff..1d8de2d 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -126,6 +126,14 @@ "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.3.tgz", "integrity": "sha512-EcswR2S8bpR7fD0YPeS7r2xXExrScVMxg4MedACaWHEtx9ftCF/qHG1xGkolzTPcEmjTavCQgbVzHUIdTMzFGA==" }, + "node_modules/yaml": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.1.tgz", + "integrity": "sha512-o96x3OPo8GjWeSLF+wOAbrPfhFOGY0W00GNaxCDv+9hkcDJEnev1yh8S7pgHF0ik6zc8sQLuL8hjHjJULZp8bw==", + "engines": { + "node": ">= 14" + } + }, "packages/common": { "name": "@gxwf-server/common", "version": "0.1.0", @@ -139,6 +147,7 @@ "license": "MIT", "dependencies": { "@types/node": "^17.0.42", + "vscode-json-languageservice": "^4.2.1", "vscode-languageserver": "^7.0.0", "vscode-languageserver-textdocument": "^1.0.4", "vscode-languageserver-types": "^3.16.0", @@ -150,9 +159,11 @@ "version": "0.1.0", "license": "MIT", "dependencies": { + "@gxwf/server-common": "*", "vscode-languageserver-textdocument": "^1.0.4", "vscode-languageserver-types": "^3.16.0", - "vscode-uri": "^3.0.3" + "vscode-uri": "^3.0.3", + "yaml": "^2.1.1" } } }, @@ -161,6 +172,7 @@ "version": "file:packages/server-common", "requires": { "@types/node": "^17.0.42", + "vscode-json-languageservice": "^4.2.1", "vscode-languageserver": "^7.0.0", "vscode-languageserver-textdocument": "^1.0.4", "vscode-languageserver-types": "^3.16.0", @@ -170,9 +182,11 @@ "@gxwf/yaml-language-service": { "version": "file:packages/yaml-language-service", "requires": { + "@gxwf/server-common": "*", "vscode-languageserver-textdocument": "^1.0.4", "vscode-languageserver-types": "^3.16.0", - "vscode-uri": "^3.0.3" + "vscode-uri": "^3.0.3", + "yaml": "^2.1.1" } }, "@types/node": { @@ -261,6 +275,11 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.3.tgz", "integrity": "sha512-EcswR2S8bpR7fD0YPeS7r2xXExrScVMxg4MedACaWHEtx9ftCF/qHG1xGkolzTPcEmjTavCQgbVzHUIdTMzFGA==" + }, + "yaml": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.1.tgz", + "integrity": "sha512-o96x3OPo8GjWeSLF+wOAbrPfhFOGY0W00GNaxCDv+9hkcDJEnev1yh8S7pgHF0ik6zc8sQLuL8hjHjJULZp8bw==" } } } diff --git a/server/packages/server-common/package.json b/server/packages/server-common/package.json index 0a3231a..56d9d1d 100644 --- a/server/packages/server-common/package.json +++ b/server/packages/server-common/package.json @@ -6,11 +6,12 @@ "license": "MIT", "type": "module", "dependencies": { - "vscode-languageserver": "^7.0.0", + "@types/node": "^17.0.42", + "vscode-json-languageservice": "^4.2.1", "vscode-languageserver-textdocument": "^1.0.4", "vscode-languageserver-types": "^3.16.0", - "vscode-uri": "^3.0.3", - "@types/node": "^17.0.42" + "vscode-languageserver": "^7.0.0", + "vscode-uri": "^3.0.3" }, "scripts": {} } diff --git a/server/packages/server-common/src/astTypes.ts b/server/packages/server-common/src/astTypes.ts new file mode 100644 index 0000000..8d0a0ed --- /dev/null +++ b/server/packages/server-common/src/astTypes.ts @@ -0,0 +1,23 @@ +import { + ArrayASTNode, + ASTNode, + BaseASTNode, + BooleanASTNode, + NullASTNode, + NumberASTNode, + ObjectASTNode, + PropertyASTNode, + StringASTNode, +} from "vscode-json-languageservice"; + +export { + ArrayASTNode, + ASTNode, + BaseASTNode, + BooleanASTNode, + NullASTNode, + NumberASTNode, + ObjectASTNode, + PropertyASTNode, + StringASTNode, +}; diff --git a/server/packages/yaml-language-service/jest.config.js b/server/packages/yaml-language-service/jest.config.js new file mode 100644 index 0000000..5f806bf --- /dev/null +++ b/server/packages/yaml-language-service/jest.config.js @@ -0,0 +1,25 @@ +// For a detailed explanation regarding each configuration property, visit: +// https://jestjs.io/docs/en/configuration.html + +module.exports = { + // A set of global variables that need to be available in all test environments + globals: { + "ts-jest": { + tsconfig: "server/tsconfig.json", + }, + }, + + // An array of directory names to be searched recursively up from the requiring module's location + moduleDirectories: ["node_modules"], + + // An array of file extensions your modules use + moduleFileExtensions: ["ts", "tsx", "js"], + + // The test environment that will be used for testing + testEnvironment: "node", + + // A map from regular expressions to paths to transformers + transform: { + "^.+\\.(ts|tsx)$": "ts-jest", + }, +}; diff --git a/server/packages/yaml-language-service/package.json b/server/packages/yaml-language-service/package.json index 82b4171..3188cd9 100644 --- a/server/packages/yaml-language-service/package.json +++ b/server/packages/yaml-language-service/package.json @@ -5,9 +5,11 @@ "author": "davelopez", "license": "MIT", "dependencies": { + "@gxwf/server-common": "*", "vscode-languageserver-textdocument": "^1.0.4", "vscode-languageserver-types": "^3.16.0", - "vscode-uri": "^3.0.3" + "vscode-uri": "^3.0.3", + "yaml": "^2.1.1" }, "scripts": {} } diff --git a/server/packages/yaml-language-service/src/index.ts b/server/packages/yaml-language-service/src/index.ts new file mode 100644 index 0000000..82908d9 --- /dev/null +++ b/server/packages/yaml-language-service/src/index.ts @@ -0,0 +1,4 @@ +import { YAMLDocument } from "./parser"; +import { LanguageService, getLanguageService } from "./yamlLanguageService"; + +export { YAMLDocument, LanguageService, getLanguageService }; diff --git a/server/packages/yaml-language-service/src/parser/astConverter.ts b/server/packages/yaml-language-service/src/parser/astConverter.ts new file mode 100644 index 0000000..3512b89 --- /dev/null +++ b/server/packages/yaml-language-service/src/parser/astConverter.ts @@ -0,0 +1,232 @@ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ + +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Red Hat, Inc. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { + Alias, + Document, + isAlias, + isMap, + isNode, + isPair, + isScalar, + isSeq, + LineCounter, + Node, + Pair, + Scalar, + YAMLMap, + YAMLSeq, +} from "yaml"; + +import { + ArrayASTNodeImpl, + ASTNode, + BooleanASTNodeImpl, + NullASTNodeImpl, + NumberASTNodeImpl, + ObjectASTNodeImpl, + PropertyASTNodeImpl, + StringASTNodeImpl, + YamlNode, +} from "./astTypes"; + +type NodeRange = [number, number, number]; + +const maxRefCount = 1000; +let refDepth = 0; + +export function convertAST( + parent: ASTNode | undefined, + node: YamlNode, + doc: Document, + lineCounter: LineCounter +): ASTNode | undefined { + if (!parent) { + // first invocation + refDepth = 0; + } + + if (!node) { + return undefined; + } + if (isMap(node)) { + return convertMap(node, parent, doc, lineCounter); + } + if (isPair(node)) { + return convertPair(node, parent, doc, lineCounter); + } + if (isSeq(node)) { + return convertSeq(node, parent, doc, lineCounter); + } + if (isScalar(node)) { + return convertScalar(node, parent); + } + if (isAlias(node)) { + if (refDepth > maxRefCount) { + // document contains excessive aliasing + return; + } + return convertAlias(node, parent, doc, lineCounter); + } +} + +function convertMap( + node: YAMLMap, + parent: ASTNode | undefined, + doc: Document, + lineCounter: LineCounter +): ASTNode { + let range: NodeRange; + if (node.flow && !node.range) { + range = collectFlowMapRange(node); + } else { + range = node.range!; + } + const result = new ObjectASTNodeImpl(parent, node, ...toFixedOffsetLength(range, lineCounter)); + for (const it of node.items) { + if (isPair(it)) { + result.properties.push(convertAST(result, it, doc, lineCounter)); + } + } + return result; +} + +function convertPair(node: Pair, parent: ASTNode | undefined, doc: Document, lineCounter: LineCounter): ASTNode { + const keyNode = node.key; + const valueNode = node.value; + const rangeStart = keyNode.range![0]; + let rangeEnd = keyNode.range![1]; + let nodeEnd = keyNode.range![2]; + if (valueNode) { + rangeEnd = valueNode.range![1]; + nodeEnd = valueNode.range![2]; + } + + const keyPlaceholder = new StringASTNodeImpl(undefined, keyNode, 0, 0); + + // Pair does not return a range using the key/value ranges to fake one. + const result = new PropertyASTNodeImpl( + parent as ObjectASTNodeImpl, + keyPlaceholder, + node, + ...toFixedOffsetLength([rangeStart, rangeEnd, nodeEnd], lineCounter) + ); + + if (isAlias(keyNode)) { + const keyAlias = new StringASTNodeImpl(parent, keyNode, ...toOffsetLength(keyNode.range!)); + keyAlias.value = keyNode.source; + result.keyNode = keyAlias; + } else { + result.keyNode = convertAST(result, keyNode, doc, lineCounter); + } + result.valueNode = convertAST(result, valueNode, doc, lineCounter); + return result; +} + +function convertSeq(node: YAMLSeq, parent: ASTNode | undefined, doc: Document, lineCounter: LineCounter): ASTNode { + const result = new ArrayASTNodeImpl(parent, node, ...toOffsetLength(node.range!)); + for (const it of node.items) { + if (isNode(it)) { + const convertedNode = convertAST(result, it, doc, lineCounter); + // due to recursion protection, convertAST may return undefined + if (convertedNode) { + result.children.push(convertedNode); + } + } + } + return result; +} + +function convertScalar(node: Scalar, parent: ASTNode | undefined): ASTNode { + if (node.value === null) { + return new NullASTNodeImpl(parent, node, ...toOffsetLength(node.range!)); + } + + switch (typeof node.value) { + case "string": { + const result = new StringASTNodeImpl(parent, node, ...toOffsetLength(node.range!)); + result.value = node.value; + return result; + } + case "boolean": + return new BooleanASTNodeImpl(parent, node, node.value, ...toOffsetLength(node.range!)); + case "number": { + const result = new NumberASTNodeImpl(parent, node, ...toOffsetLength(node.range!)); + result.value = node.value; + result.isInteger = Number.isInteger(result.value); + return result; + } + default: { + // fail safe converting, we need to return some node anyway + const result = new StringASTNodeImpl(parent, node, ...toOffsetLength(node.range!)); + result.value = node.source!; + return result; + } + } +} + +function convertAlias( + node: Alias, + parent: ASTNode | undefined, + doc: Document, + lineCounter: LineCounter +): ASTNode | undefined { + refDepth++; + const resolvedNode = node.resolve(doc); + if (resolvedNode) { + return convertAST(parent, resolvedNode, doc, lineCounter); + } else { + const resultNode = new StringASTNodeImpl(parent, node, ...toOffsetLength(node.range!)); + resultNode.value = node.source; + return resultNode; + } +} + +export function toOffsetLength(range: NodeRange): [number, number] { + return [range[0], range[1] - range[0]]; +} + +/** + * Convert offsets to offset+length with fix length to not include '\n' character in some cases + * @param range the yaml ast range + * @param lineCounter the line counter + * @returns the offset and length + */ +function toFixedOffsetLength(range: NodeRange, lineCounter: LineCounter): [number, number] { + const start = lineCounter.linePos(range[0]); + const end = lineCounter.linePos(range[1]); + + const result: [number, number] = [range[0], range[1] - range[0]]; + // -1 as range may include '\n' + if (start.line !== end.line && (lineCounter.lineStarts.length !== end.line || end.col === 1)) { + result[1]--; + } + + return result; +} + +function collectFlowMapRange(node: YAMLMap): NodeRange { + let start = Number.MAX_SAFE_INTEGER; + let end = 0; + for (const it of node.items) { + if (isPair(it)) { + if (isNode(it.key)) { + if (it.key.range && it.key.range[0] <= start) { + start = it.key.range[0]; + } + } + + if (isNode(it.value)) { + if (it.value.range && it.value.range[2] >= end) { + end = it.value.range[2]; + } + } + } + } + + return [start, end, end]; +} diff --git a/server/packages/yaml-language-service/src/parser/astTypes.ts b/server/packages/yaml-language-service/src/parser/astTypes.ts new file mode 100644 index 0000000..c043fbd --- /dev/null +++ b/server/packages/yaml-language-service/src/parser/astTypes.ts @@ -0,0 +1,174 @@ +import { + BaseASTNode, + ArrayASTNode, + ASTNode, + BooleanASTNode, + NullASTNode, + NumberASTNode, + ObjectASTNode, + PropertyASTNode, + StringASTNode, +} from "@gxwf/server-common/src/astTypes"; +import { Node, Pair } from "yaml"; + +export { + ArrayASTNode, + ASTNode, + BooleanASTNode, + NullASTNode, + NumberASTNode, + ObjectASTNode, + PropertyASTNode, + StringASTNode, +}; + +export type YamlNode = Node | Pair; + +export abstract class ASTNodeImpl { + public abstract readonly type: "object" | "property" | "array" | "number" | "boolean" | "null" | "string"; + + public offset: number; + public length: number; + public readonly parent: ASTNode | undefined; + readonly internalNode: YamlNode; + + constructor(parent: ASTNode | undefined, internalNode: YamlNode, offset: number, length: number) { + this.offset = offset; + this.length = length; + this.parent = parent; + this.internalNode = internalNode; + } + + public getNodeFromOffsetEndInclusive(offset: number): ASTNode | null { + const collector: BaseASTNode[] = []; + const findNode = (node: BaseASTNode): BaseASTNode | null => { + if (offset >= node.offset && offset <= node.offset + node.length) { + const children = node.children; + if (children && children.length) { + for (let i = 0; i < children.length && children[i].offset <= offset; i++) { + const item = findNode(children[i]); + if (item) { + collector.push(item); + } + } + return node; + } + } + return null; + }; + const foundNode = findNode(this); + let currMinDist = Number.MAX_VALUE; + let currMinNode = null; + for (const currNode of collector) { + const minDist = currNode.length + currNode.offset - offset + (offset - currNode.offset); + if (minDist < currMinDist) { + currMinNode = currNode; + currMinDist = minDist; + } + } + return (currMinNode || foundNode) as ASTNode | null; + } + + public get children(): ASTNode[] { + return []; + } + + public toString(): string { + return ( + "type: " + + this.type + + " (" + + this.offset + + "/" + + this.length + + ")" + + (this.parent ? " parent: {" + this.parent.toString() + "}" : "") + ); + } +} + +export class NullASTNodeImpl extends ASTNodeImpl implements NullASTNode { + public type: "null" = "null"; + public value = null; + constructor(parent: ASTNode | undefined, internalNode: Node, offset: number, length: number) { + super(parent, internalNode, offset, length); + } +} + +export class BooleanASTNodeImpl extends ASTNodeImpl implements BooleanASTNode { + public type: "boolean" = "boolean"; + public value: boolean; + + constructor(parent: ASTNode | undefined, internalNode: Node, boolValue: boolean, offset: number, length: number) { + super(parent, internalNode, offset, length); + this.value = boolValue; + } +} + +export class ArrayASTNodeImpl extends ASTNodeImpl implements ArrayASTNode { + public type: "array" = "array"; + public items: ASTNode[]; + + constructor(parent: ASTNode | undefined, internalNode: Node, offset: number, length: number) { + super(parent, internalNode, offset, length); + this.items = []; + } + + public get children(): ASTNode[] { + return this.items; + } +} + +export class NumberASTNodeImpl extends ASTNodeImpl implements NumberASTNode { + public type: "number" = "number"; + public isInteger: boolean; + public value: number; + + constructor(parent: ASTNode | undefined, internalNode: Node, offset: number, length: number) { + super(parent, internalNode, offset, length); + this.isInteger = true; + this.value = Number.NaN; + } +} + +export class StringASTNodeImpl extends ASTNodeImpl implements StringASTNode { + public type: "string" = "string"; + public value: string; + + constructor(parent: ASTNode | undefined, internalNode: Node, offset: number, length: number) { + super(parent, internalNode, offset, length); + this.value = ""; + } +} + +export class PropertyASTNodeImpl extends ASTNodeImpl implements PropertyASTNode { + public type: "property" = "property"; + public keyNode: StringASTNode; + public valueNode?: ASTNode; + public colonOffset: number; + + constructor(parent: ObjectASTNode, keyNode: StringASTNode, internalNode: Pair, offset: number, length: number) { + super(parent, internalNode, offset, length); + this.colonOffset = -1; + this.keyNode = keyNode; + } + + public get children(): ASTNode[] { + return this.valueNode ? [this.keyNode, this.valueNode] : [this.keyNode]; + } +} + +export class ObjectASTNodeImpl extends ASTNodeImpl implements ObjectASTNode { + public type: "object" = "object"; + public properties: PropertyASTNode[]; + + constructor(parent: ASTNode | undefined, internalNode: Node, offset: number, length: number) { + super(parent, internalNode, offset, length); + + this.properties = []; + } + + public get children(): ASTNode[] { + return this.properties; + } +} diff --git a/server/packages/yaml-language-service/src/parser/index.ts b/server/packages/yaml-language-service/src/parser/index.ts new file mode 100644 index 0000000..55749af --- /dev/null +++ b/server/packages/yaml-language-service/src/parser/index.ts @@ -0,0 +1,47 @@ +"use strict"; + +import { Parser, Composer, Document, LineCounter, ParseOptions, DocumentOptions, SchemaOptions, Node } from "yaml"; +import { TextDocument } from "vscode-languageserver-textdocument"; +import { YAMLDocument, YAMLSubDocument } from "./yamlDocument"; +import { TextBuffer } from "../utils/textBuffer"; +import { convertAST } from "./astConverter"; + +export { YAMLDocument }; + +export type YamlVersion = "1.1" | "1.2"; +export interface ParserOptions { + yamlVersion: YamlVersion; +} +export const defaultOptions: ParserOptions = { + yamlVersion: "1.2", +}; + +export function parse(textDocument: TextDocument, parserOptions: ParserOptions = defaultOptions): YAMLDocument { + const text = textDocument.getText(); + const options: ParseOptions & DocumentOptions & SchemaOptions = { + strict: false, + version: parserOptions.yamlVersion ?? defaultOptions.yamlVersion, + keepSourceTokens: true, + }; + const composer = new Composer(options); + const lineCounter = new LineCounter(); + let isLastLineEmpty = false; + if (textDocument) { + const textBuffer = new TextBuffer(textDocument); + const position = textBuffer.getPosition(text.length); + const lineContent = textBuffer.getLineContent(position.line); + isLastLineEmpty = lineContent.trim().length === 0; + } + const parser = isLastLineEmpty ? new Parser() : new Parser(lineCounter.addNewLine); + const tokens = parser.parse(text); + const tokensArr = Array.from(tokens); + const docs = composer.compose(tokensArr, true, text.length); + const parsedDocs: YAMLSubDocument[] = Array.from(docs, (doc) => getParsedSubDocument(doc, lineCounter)); + + return new YAMLDocument(parsedDocs, textDocument); +} + +function getParsedSubDocument(parsedDocument: Document, lineCounter: LineCounter): YAMLSubDocument { + const root = convertAST(undefined, parsedDocument.contents as Node, parsedDocument, lineCounter); + return new YAMLSubDocument(root, parsedDocument); +} diff --git a/server/packages/yaml-language-service/src/parser/yamlDocument.ts b/server/packages/yaml-language-service/src/parser/yamlDocument.ts new file mode 100644 index 0000000..f15bb35 --- /dev/null +++ b/server/packages/yaml-language-service/src/parser/yamlDocument.ts @@ -0,0 +1,66 @@ +import { TextDocument } from "vscode-languageserver-textdocument"; +import { Diagnostic, DiagnosticSeverity, Position } from "vscode-languageserver-types"; +import { Document, YAMLError, YAMLWarning } from "yaml"; +import { TextBuffer } from "../utils/textBuffer"; +import { ASTNode } from "./astTypes"; + +const FULL_LINE_ERROR = true; +const YAML_SOURCE = "YAML"; + +export class YAMLSubDocument { + constructor(public readonly root: ASTNode | undefined, private readonly parsedDocument: Document) {} + + get errors(): YAMLError[] { + return this.parsedDocument.errors; + } + get warnings(): YAMLWarning[] { + return this.parsedDocument.warnings; + } +} + +export class YAMLDocument { + private readonly _textBuffer: TextBuffer; + private _diagnostics: Diagnostic[] | undefined; + + constructor(public readonly subDocuments: YAMLSubDocument[], public readonly textDocument: TextDocument) { + this._textBuffer = new TextBuffer(textDocument); + this._diagnostics = undefined; + } + + public get firstDocument(): YAMLSubDocument | undefined { + return this.subDocuments.at(0); + } + + public get syntaxDiagnostics(): Diagnostic[] { + if (!this._diagnostics) { + this._diagnostics = this.getSyntaxDiagnostics(); + } + return this._diagnostics; + } + + private getSyntaxDiagnostics(): Diagnostic[] { + const syntaxErrors = this.subDocuments.flatMap((subDoc) => + subDoc.errors.map((e) => this.YAMLErrorToDiagnostics(e)) + ); + const syntaxWarnings = this.subDocuments.flatMap((subDoc) => + subDoc.warnings.map((e) => this.YAMLErrorToDiagnostics(e)) + ); + return syntaxErrors.concat(syntaxWarnings); + } + + private YAMLErrorToDiagnostics(error: YAMLError): Diagnostic { + const begin = error.pos[0]; + const end = error.pos[1]; + const severity: DiagnosticSeverity = + error instanceof YAMLWarning ? DiagnosticSeverity.Warning : DiagnosticSeverity.Error; + const start = this.textDocument.positionAt(begin); + const range = { + start, + end: FULL_LINE_ERROR + ? Position.create(start.line, this._textBuffer.getLineLength(start.line)) + : this.textDocument.positionAt(end), + }; + + return Diagnostic.create(range, error.message, severity, error.code, YAML_SOURCE); + } +} diff --git a/server/packages/yaml-language-service/src/services/yamlValidation.ts b/server/packages/yaml-language-service/src/services/yamlValidation.ts new file mode 100644 index 0000000..57cec03 --- /dev/null +++ b/server/packages/yaml-language-service/src/services/yamlValidation.ts @@ -0,0 +1,27 @@ +import { Diagnostic } from "vscode-languageserver-types"; +import { YAMLDocument } from "../parser"; +import { LanguageSettings } from "../yamlLanguageService"; + +export class YAMLValidation { + private validationEnabled?: boolean; + + constructor() { + this.validationEnabled = true; + } + + public configure(settings: LanguageSettings): void { + if (settings) { + this.validationEnabled = settings.validate; + } + } + + public async doValidation(yamlDocument: YAMLDocument): Promise { + if (!this.validationEnabled) { + return Promise.resolve([]); + } + const diagnostics: Diagnostic[] = [...yamlDocument.syntaxDiagnostics]; + // TODO: add schema validation diagnostics + + return Promise.resolve(diagnostics); + } +} diff --git a/server/packages/yaml-language-service/src/utils/textBuffer.ts b/server/packages/yaml-language-service/src/utils/textBuffer.ts new file mode 100644 index 0000000..5d68240 --- /dev/null +++ b/server/packages/yaml-language-service/src/utils/textBuffer.ts @@ -0,0 +1,56 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Red Hat. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { TextDocument } from "vscode-languageserver-textdocument"; +import { Position, Range } from "vscode-languageserver-types"; + +interface FullTextDocument { + getLineOffsets(): number[]; +} + +export class TextBuffer { + constructor(private doc: TextDocument) {} + + getLineCount(): number { + return this.doc.lineCount; + } + + getLineLength(lineNumber: number): number { + const lineOffsets = (this.doc as unknown as FullTextDocument).getLineOffsets(); + if (lineNumber >= lineOffsets.length) { + return this.doc.getText().length; + } else if (lineNumber < 0) { + return 0; + } + + const nextLineOffset = + lineNumber + 1 < lineOffsets.length ? lineOffsets[lineNumber + 1] : this.doc.getText().length; + return nextLineOffset - lineOffsets[lineNumber]; + } + + getLineContent(lineNumber: number): string { + const lineOffsets = (this.doc as unknown as FullTextDocument).getLineOffsets(); + if (lineNumber >= lineOffsets.length) { + return this.doc.getText(); + } else if (lineNumber < 0) { + return ""; + } + const nextLineOffset = + lineNumber + 1 < lineOffsets.length ? lineOffsets[lineNumber + 1] : this.doc.getText().length; + return this.doc.getText().substring(lineOffsets[lineNumber], nextLineOffset); + } + + getLineCharCode(lineNumber: number, index: number): number { + return this.doc.getText(Range.create(lineNumber - 1, index - 1, lineNumber - 1, index)).charCodeAt(0); + } + + getText(range?: Range): string { + return this.doc.getText(range); + } + + getPosition(offest: number): Position { + return this.doc.positionAt(offest); + } +} diff --git a/server/packages/yaml-language-service/src/yamlLanguageService.ts b/server/packages/yaml-language-service/src/yamlLanguageService.ts index cc1dd40..1bca5c3 100644 --- a/server/packages/yaml-language-service/src/yamlLanguageService.ts +++ b/server/packages/yaml-language-service/src/yamlLanguageService.ts @@ -1,15 +1,13 @@ -import { ASTNode } from "vscode-json-languageservice"; import { TextDocument } from "vscode-languageserver-textdocument"; -import { FormattingOptions, Hover, Position, TextEdit } from "vscode-languageserver-types"; +import { Diagnostic, FormattingOptions, Hover, Position, TextEdit } from "vscode-languageserver-types"; import { YAMLFormatter } from "./services/yamlFormatter"; - -export declare type YAMLDocument = { - root: ASTNode | undefined; - getNodeFromOffset(offset: number, includeRightBound?: boolean): ASTNode | undefined; -}; +import { parse as parseYAML, YAMLDocument } from "./parser"; +import { YAMLValidation } from "./services/yamlValidation"; export interface LanguageSettings { - format?: boolean; //Setting for whether we want to have the formatter or not + validate?: boolean; + format?: boolean; + indentation?: string; } export interface CustomFormatterOptions { @@ -21,13 +19,18 @@ export interface CustomFormatterOptions { } export interface LanguageService { + parseYAMLDocument(document: TextDocument): YAMLDocument; + doValidation(yamlDocument: YAMLDocument): Promise; doFormat(document: TextDocument, options: FormattingOptions & CustomFormatterOptions): TextEdit[]; doHover(document: TextDocument, position: Position): Hover | null; } export function getLanguageService(): LanguageService { const formatter = new YAMLFormatter(); + const validator = new YAMLValidation(); return { + parseYAMLDocument: (document: TextDocument) => parseYAML(document), + doValidation: (yamlDocument: YAMLDocument) => validator.doValidation(yamlDocument), doFormat: formatter.format.bind(formatter), doHover: (doc, pos) => { return null; From 72ae2b884ab6ac00073a98f5b2f077b87565e56b Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 19 Jun 2022 16:22:00 +0200 Subject: [PATCH 02/11] Add comment lines to parsed document --- .../src/parser/yamlDocument.ts | 65 ++++++++++++++++++- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/server/packages/yaml-language-service/src/parser/yamlDocument.ts b/server/packages/yaml-language-service/src/parser/yamlDocument.ts index f15bb35..9ec3e90 100644 --- a/server/packages/yaml-language-service/src/parser/yamlDocument.ts +++ b/server/packages/yaml-language-service/src/parser/yamlDocument.ts @@ -1,23 +1,65 @@ import { TextDocument } from "vscode-languageserver-textdocument"; import { Diagnostic, DiagnosticSeverity, Position } from "vscode-languageserver-types"; -import { Document, YAMLError, YAMLWarning } from "yaml"; +import { Document, Node, visit, YAMLError, YAMLWarning } from "yaml"; import { TextBuffer } from "../utils/textBuffer"; import { ASTNode } from "./astTypes"; const FULL_LINE_ERROR = true; const YAML_SOURCE = "YAML"; +export class LineComment { + constructor(public readonly text: string) {} +} + export class YAMLSubDocument { + private _lineComments: LineComment[] | undefined; + constructor(public readonly root: ASTNode | undefined, private readonly parsedDocument: Document) {} get errors(): YAMLError[] { return this.parsedDocument.errors; } + get warnings(): YAMLWarning[] { return this.parsedDocument.warnings; } + + get lineComments(): LineComment[] { + if (!this._lineComments) { + this._lineComments = this.collectLineComments(); + } + return this._lineComments; + } + + private collectLineComments(): LineComment[] { + const lineComments = []; + if (this.parsedDocument.commentBefore) { + const comments = this.parsedDocument.commentBefore.split("\n"); + comments.forEach((comment) => lineComments.push(new LineComment(`#${comment}`))); + } + visit(this.parsedDocument, (_key, docNode) => { + const node = docNode as Node; + if (node?.commentBefore) { + const comments = node?.commentBefore.split("\n"); + comments.forEach((comment) => lineComments.push(new LineComment(`#${comment}`))); + } + + if (node?.comment) { + lineComments.push(new LineComment(`#${node.comment}`)); + } + }); + + if (this.parsedDocument.comment) { + lineComments.push(new LineComment(`#${this.parsedDocument.comment}`)); + } + return lineComments; + } } +/** + * Represents a YAML document. + * YAML documents can contain multiple sub-documents separated by "---". + */ export class YAMLDocument { private readonly _textBuffer: TextBuffer; private _diagnostics: Diagnostic[] | undefined; @@ -27,10 +69,12 @@ export class YAMLDocument { this._diagnostics = undefined; } - public get firstDocument(): YAMLSubDocument | undefined { + /** The first or single sub-document parsed. */ + public get mainDocument(): YAMLSubDocument | undefined { return this.subDocuments.at(0); } + /** Returns basic YAML syntax errors or warnings. */ public get syntaxDiagnostics(): Diagnostic[] { if (!this._diagnostics) { this._diagnostics = this.getSyntaxDiagnostics(); @@ -38,6 +82,12 @@ export class YAMLDocument { return this._diagnostics; } + /** List of comments in this document. */ + public get lineComments(): LineComment[] { + return this.collectLineComments(); + } + + /** Collects all syntax errors and warnings found on this document. */ private getSyntaxDiagnostics(): Diagnostic[] { const syntaxErrors = this.subDocuments.flatMap((subDoc) => subDoc.errors.map((e) => this.YAMLErrorToDiagnostics(e)) @@ -48,6 +98,7 @@ export class YAMLDocument { return syntaxErrors.concat(syntaxWarnings); } + /** Converts from internal YAMLError to a document Diagnostic item. */ private YAMLErrorToDiagnostics(error: YAMLError): Diagnostic { const begin = error.pos[0]; const end = error.pos[1]; @@ -60,7 +111,15 @@ export class YAMLDocument { ? Position.create(start.line, this._textBuffer.getLineLength(start.line)) : this.textDocument.positionAt(end), }; - return Diagnostic.create(range, error.message, severity, error.code, YAML_SOURCE); } + + /** Collects all comment lines across all sub-documents contained in this document. */ + private collectLineComments(): LineComment[] { + const lineComments: LineComment[] = []; + this.subDocuments.forEach((subDocument) => { + lineComments.push(...subDocument.lineComments); + }); + return lineComments; + } } From 3fe0c4c413c3d957b8480875cb0289280d3da594 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 19 Jun 2022 16:44:59 +0200 Subject: [PATCH 03/11] Add YAML parser tests --- .../yaml-language-service/tests/testHelper.ts | 14 ++ .../tests/unit/yamlParser.test.ts | 160 ++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 server/packages/yaml-language-service/tests/testHelper.ts create mode 100644 server/packages/yaml-language-service/tests/unit/yamlParser.test.ts diff --git a/server/packages/yaml-language-service/tests/testHelper.ts b/server/packages/yaml-language-service/tests/testHelper.ts new file mode 100644 index 0000000..4aabe6f --- /dev/null +++ b/server/packages/yaml-language-service/tests/testHelper.ts @@ -0,0 +1,14 @@ +import { TextDocument } from "vscode-languageserver-textdocument"; +import { PropertyASTNode } from "../src/parser/astTypes"; + +export function toTextDocument(contents: string): TextDocument { + const textDoc = TextDocument.create("foo://bar/file.yaml", "yaml", 0, contents); + + return textDoc; +} + +export function expectPropertyToHaveKeyValue(property: PropertyASTNode, key: string, value: string): void { + expect(property.keyNode.value).toBe(key); + expect(property.valueNode).toBeDefined(); + expect(property.valueNode?.value).toBe(value); +} diff --git a/server/packages/yaml-language-service/tests/unit/yamlParser.test.ts b/server/packages/yaml-language-service/tests/unit/yamlParser.test.ts new file mode 100644 index 0000000..aa2d068 --- /dev/null +++ b/server/packages/yaml-language-service/tests/unit/yamlParser.test.ts @@ -0,0 +1,160 @@ +import { parse as parseYAML, YAMLDocument } from "../../src/parser"; +import { ArrayASTNode, ObjectASTNode, PropertyASTNode } from "../../src/parser/astTypes"; +import { expectPropertyToHaveKeyValue, toTextDocument } from "../testHelper"; + +const parse = (contents: string): YAMLDocument => { + return parseYAML(toTextDocument(contents)); +}; + +describe("YAML parser", () => { + it("creates an empty sub-document on empty text", () => { + const parsedDocument = parse(""); + expect(parsedDocument.subDocuments.length).toBe(1); + }); + + it("creates a sub-document when only comment", () => { + const parsedDocument = parse("# a comment"); + expect(parsedDocument.subDocuments.length).toBe(1); + }); + + it("creates a single sub-document with --- at the start of the file", () => { + const parsedDocument = parse("---\n# a comment\ntest: test"); + expect(parsedDocument.subDocuments.length).toBe(1); + expect(parsedDocument.subDocuments[0].lineComments.length).toBe(1); + expect(parsedDocument.subDocuments[0].lineComments[0].text).toBe("# a comment"); + }); + + it("creates a multi document with --- at the start of the file", () => { + const parsedDocument = parse("---\n# a comment\ntest: test\n...\n---\n# second document\ntest2: test2"); + expect(parsedDocument.subDocuments.length).toBe(2); + expect(parsedDocument.subDocuments[0].lineComments.length).toBe(1); + expect(parsedDocument.subDocuments[0].lineComments[0].text).toBe("# a comment"); + + expect(parsedDocument.subDocuments[1].lineComments.length).toBe(1); + expect(parsedDocument.subDocuments[1].lineComments[0].text).toBe("# second document"); + }); + + it("creates a single sub-document with directives and line comments", () => { + const parsedDocument = parse("%TAG !yaml! tag:yaml.org,2002:\n---\n# a comment\ntest"); + expect(parsedDocument.subDocuments.length).toBe(1); + expect(parsedDocument.subDocuments[0]?.root?.children?.length).toBe(0); + expect(parsedDocument.subDocuments[0].lineComments.length).toBe(1); + expect(parsedDocument.subDocuments[0].lineComments[0].text).toBe("# a comment"); + }); + + it("creates 2 sub-documents with directives and line comments", () => { + const parsedDocument = parse("%TAG !yaml! tag:yaml.org,2002:\n# a comment\ntest\n...\n---\ntest2"); + expect(parsedDocument.subDocuments.length).toBe(2); + expect(parsedDocument.subDocuments[0]?.root?.children?.length).toBe(0); + expect(parsedDocument.subDocuments[1]?.root?.children?.length).toBe(0); + expect(parsedDocument.subDocuments[1]?.root?.value).toBe("test2"); + expect(parsedDocument.subDocuments[0].lineComments.length).toBe(1); + expect(parsedDocument.subDocuments[0].lineComments[0].text).toBe("# a comment"); + }); + + it("creates a single sub-document", () => { + const parsedDocument = parse("test"); + expect(parsedDocument.subDocuments.length).toBe(1); + expect(parsedDocument.subDocuments[0]?.root?.value).toBe("test"); + expect(parsedDocument.subDocuments[0]?.root?.children?.length).toBe(0); + }); + + it("creates a single document with directives", () => { + const parsedDocument = parse("%TAG !yaml! tag:yaml.org,2002:\n---\ntest"); + expect(parsedDocument.subDocuments.length).toBe(1); + expect(parsedDocument.subDocuments[0]?.root?.value).toBe("test"); + expect(parsedDocument.subDocuments[0]?.root?.children?.length).toBe(0); + }); + + it("creates 2 sub-documents", () => { + const parsedDocument = parse("test\n---\ntest2"); + expect(parsedDocument.subDocuments.length).toBe(2); + expect(parsedDocument.subDocuments[0]?.root?.value).toBe("test"); + expect(parsedDocument.subDocuments[0]?.root?.children?.length).toBe(0); + expect(parsedDocument.subDocuments[1]?.root?.value).toBe("test2"); + expect(parsedDocument.subDocuments[1]?.root?.children?.length).toBe(0); + }); + + it("creates 3 sub-documents", () => { + const parsedDocument = parse("test\n---\ntest2\n---\ntest3"); + expect(parsedDocument.subDocuments.length).toBe(3); + expect(parsedDocument.subDocuments[0]?.root?.value).toBe("test"); + expect(parsedDocument.subDocuments[0]?.root?.children?.length).toBe(0); + expect(parsedDocument.subDocuments[1]?.root?.value).toBe("test2"); + expect(parsedDocument.subDocuments[1]?.root?.children?.length).toBe(0); + expect(parsedDocument.subDocuments[2]?.root?.value).toBe("test3"); + expect(parsedDocument.subDocuments[2]?.root?.children?.length).toBe(0); + }); + + it("creates a single document with comment", () => { + const parsedDocument = parse("# a comment\ntest"); + expect(parsedDocument.subDocuments.length).toBe(1); + expect(parsedDocument.subDocuments[0]?.root?.value).toBe("test"); + expect(parsedDocument.subDocuments[0].lineComments.length).toBe(1); + expect(parsedDocument.subDocuments[0].lineComments[0].text).toBe("# a comment"); + }); + + it("creates 2 sub-documents with comment", () => { + const parsedDocument = parse("---\n# a comment\ntest: test\n---\n# a second comment\ntest2"); + expect(parsedDocument.subDocuments.length).toBe(2); + expect(parsedDocument.subDocuments[0].root as ObjectASTNode).toBeDefined(); + const firstProperty = (parsedDocument.subDocuments[0].root as ObjectASTNode).properties[0] as PropertyASTNode; + expect(firstProperty).toBeDefined(); + expectPropertyToHaveKeyValue(firstProperty, "test", "test"); + expect(parsedDocument.subDocuments[0].lineComments.length).toBe(1); + expect(parsedDocument.subDocuments[0].lineComments[0].text).toBe("# a comment"); + + expect(parsedDocument.subDocuments[1]?.root?.value).toBe("test2"); + expect(parsedDocument.subDocuments[1]?.root?.children?.length).toBe(0); + expect(parsedDocument.subDocuments[1].lineComments.length).toBe(1); + expect(parsedDocument.subDocuments[1].lineComments[0].text).toBe("# a second comment"); + }); + + it('creates a document with "str" tag from recommended schema', () => { + const parsedDocument = parse('"yes as a string with tag": !!str yes'); + expect(parsedDocument.subDocuments.length).toBe(1); + expect(parsedDocument.subDocuments[0].errors.length).toBe(0); + }); + + it('creates a document with "int" tag from recommended schema', () => { + const parsedDocument = parse("POSTGRES_PORT: !!int 54"); + expect(parsedDocument.subDocuments.length).toBe(1); + expect(parsedDocument.subDocuments[0].errors.length).toBe(0); + }); +}); + +describe("YAML parser bugs", () => { + it('should work with "Billion Laughs" attack', () => { + const yaml = `apiVersion: v1 +data: +a: &a ["web","web","web","web","web","web","web","web","web"] +b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a] +c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b] +d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c] +e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d] +f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e] +g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f] +h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g] +i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h] +kind: ConfigMap +metadata: +name: yaml-bomb +namespace: defaul`; + const parsedDocument = parse(yaml); + expect(parsedDocument.subDocuments.length).toBe(1); + }); + + it('should not add "undefined" as array item', () => { + const yaml = `foo: +- *`; + const parsedDocument = parse(yaml); + parsedDocument.subDocuments[0].root; + expect(parsedDocument.subDocuments.length).toBe(1); + expect( + ( + ((parsedDocument.subDocuments[0].root as ObjectASTNode).properties[0] as PropertyASTNode) + .valueNode as ArrayASTNode + ).items[0] + ).toBeDefined(); + }); +}); From 10f23536fb7140978d4748585c580f181298a1a1 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 19 Jun 2022 17:22:56 +0200 Subject: [PATCH 04/11] Simplify test setup --- .github/workflows/main.yml | 4 +- client/jest.config.js | 17 +--- client/package.json | 2 +- client/tsconfig.json | 2 +- docs/CONTRIBUTING.md | 2 +- jest.config.js | 9 +++ package.json | 6 +- server/gx-workflow-ls-native/jest.config.js | 25 ------ server/jest.config.js | 16 ++++ server/package-lock.json | 78 +++++-------------- server/package.json | 3 +- .../yaml-language-service/jest.config.js | 25 ------ .../yaml-language-service/package.json | 4 +- tsconfig.json | 16 ++++ 14 files changed, 76 insertions(+), 133 deletions(-) delete mode 100644 server/gx-workflow-ls-native/jest.config.js create mode 100644 server/jest.config.js delete mode 100644 server/packages/yaml-language-service/jest.config.js create mode 100644 tsconfig.json diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 77f88cc..5d32888 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,9 +25,9 @@ jobs: - name: Lint Code run: npm run lint - name: Run server unit tests - run: npm run test-unit-server + run: npm run test-server - name: Run client unit tests - run: npm run test-unit-client + run: npm run test-client - name: Run integration tests run: xvfb-run -a npm run test:e2e if: runner.os == 'Linux' diff --git a/client/jest.config.js b/client/jest.config.js index 277728a..9d817ae 100644 --- a/client/jest.config.js +++ b/client/jest.config.js @@ -2,24 +2,15 @@ // https://jestjs.io/docs/en/configuration.html module.exports = { - // A set of global variables that need to be available in all test environments + preset: "ts-jest", globals: { "ts-jest": { - tsconfig: "tsconfig.json", + tsconfig: "../tsconfig.json", }, }, - - // An array of directory names to be searched recursively up from the requiring module's location - moduleDirectories: ["node_modules"], + // The glob patterns Jest uses to detect test files + testMatch: ["**/__tests__/*.+(ts|tsx|js)", "**/unit/*.test.ts"], // An array of file extensions your modules use moduleFileExtensions: ["ts", "tsx", "js"], - - // The test environment that will be used for testing - testEnvironment: "node", - - // A map from regular expressions to paths to transformers - transform: { - "^.+\\.(ts|tsx)$": "ts-jest", - }, }; diff --git a/client/package.json b/client/package.json index 16ee6ae..f5f7a06 100644 --- a/client/package.json +++ b/client/package.json @@ -21,6 +21,6 @@ "scripts": { "webpack": "webpack", "watch": "webpack --watch --progress", - "test-unit": "jest" + "test": "jest" } } diff --git a/client/tsconfig.json b/client/tsconfig.json index 62edefe..c7417ce 100644 --- a/client/tsconfig.json +++ b/client/tsconfig.json @@ -8,5 +8,5 @@ "skipLibCheck": true }, "include": ["tests"], - "exclude": ["node_modules", ".vscode-test-web"] + "exclude": ["node_modules"] } diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index a4e877d..7709de0 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -80,7 +80,7 @@ You can run all the unit tests with: npm test ``` -Alternatively, you can choose to run only the [server](../server/tests/unit/) or the [client](../client/tests/unit/) tests using `npm run test-unit-server` or `npm run test-unit-client` respectively. +Alternatively, you can choose to run only the [server](../server/) or the [client](../client/tests/unit/) tests using `npm run test-server` or `npm run test-client` respectively. The [integration or end to end (e2e) tests](../client/tests/e2e/suite/) will download (the first time) and launch a testing version of VSCode and then run the tests on it. You can run these tests with: diff --git a/jest.config.js b/jest.config.js index b7a04f2..fe961f8 100644 --- a/jest.config.js +++ b/jest.config.js @@ -2,6 +2,15 @@ // https://jestjs.io/docs/en/configuration.html module.exports = { + preset: "ts-jest", + globals: { + "ts-jest": { + tsconfig: "tsconfig.json", + }, + }, // The glob patterns Jest uses to detect test files testMatch: ["**/__tests__/*.+(ts|tsx|js)", "**/unit/*.test.ts"], + + // An array of file extensions your modules use + moduleFileExtensions: ["ts", "tsx", "js"], }; diff --git a/package.json b/package.json index 5f4f2c5..577e1dc 100644 --- a/package.json +++ b/package.json @@ -213,9 +213,9 @@ "watch": "concurrently --kill-others \"npm run watch-server\" \"npm run watch-client\"", "watch-server": "cd server && npm run watch", "watch-client": "cd client && npm run watch", - "test": "npm run test-unit-client && npm run test-unit-server", - "test-unit-client": "cd client && npm run test-unit && cd ..", - "test-unit-server": "cd server && npm run test-unit && cd ..", + "test": "jest", + "test-client": "cd client && npm test", + "test-server": "cd server && npm test", "test-compile": "tsc --project ./client --outDir client/out", "pretest:e2e": "npm run compile && npm run test-compile", "test:e2e": "node ./client/out/e2e/runTests.js" diff --git a/server/gx-workflow-ls-native/jest.config.js b/server/gx-workflow-ls-native/jest.config.js deleted file mode 100644 index 277728a..0000000 --- a/server/gx-workflow-ls-native/jest.config.js +++ /dev/null @@ -1,25 +0,0 @@ -// For a detailed explanation regarding each configuration property, visit: -// https://jestjs.io/docs/en/configuration.html - -module.exports = { - // A set of global variables that need to be available in all test environments - globals: { - "ts-jest": { - tsconfig: "tsconfig.json", - }, - }, - - // An array of directory names to be searched recursively up from the requiring module's location - moduleDirectories: ["node_modules"], - - // An array of file extensions your modules use - moduleFileExtensions: ["ts", "tsx", "js"], - - // The test environment that will be used for testing - testEnvironment: "node", - - // A map from regular expressions to paths to transformers - transform: { - "^.+\\.(ts|tsx)$": "ts-jest", - }, -}; diff --git a/server/jest.config.js b/server/jest.config.js new file mode 100644 index 0000000..9d817ae --- /dev/null +++ b/server/jest.config.js @@ -0,0 +1,16 @@ +// For a detailed explanation regarding each configuration property, visit: +// https://jestjs.io/docs/en/configuration.html + +module.exports = { + preset: "ts-jest", + globals: { + "ts-jest": { + tsconfig: "../tsconfig.json", + }, + }, + // The glob patterns Jest uses to detect test files + testMatch: ["**/__tests__/*.+(ts|tsx|js)", "**/unit/*.test.ts"], + + // An array of file extensions your modules use + moduleFileExtensions: ["ts", "tsx", "js"], +}; diff --git a/server/package-lock.json b/server/package-lock.json index 1d8de2d..0be7312 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -50,8 +50,7 @@ }, "node_modules/@types/node": { "version": "17.0.42", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.42.tgz", - "integrity": "sha512-Q5BPGyGKcvQgAMbsr7qEGN/kIPN6zZecYYABeTDBizOsau+2NMdSVTar9UQw21A2+JyA2KRNDYaYrPB0Rpk2oQ==" + "license": "MIT" }, "node_modules/gx-workflow-ls-format2": { "resolved": "gx-workflow-ls-format2", @@ -63,13 +62,11 @@ }, "node_modules/jsonc-parser": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz", - "integrity": "sha512-fQzRfAbIBnR0IQvftw9FJveWiHp72Fg20giDrHz6TdfB12UH/uue0D3hm57UB5KgAVuniLMCaS8P1IMj9NR7cA==" + "license": "MIT" }, "node_modules/vscode-json-languageservice": { "version": "4.2.1", - "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", - "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", + "license": "MIT", "dependencies": { "jsonc-parser": "^3.0.0", "vscode-languageserver-textdocument": "^1.0.3", @@ -80,16 +77,14 @@ }, "node_modules/vscode-jsonrpc": { "version": "6.0.0", - "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-6.0.0.tgz", - "integrity": "sha512-wnJA4BnEjOSyFMvjZdpiOwhSq9uDoK8e/kpRJDTaMYzwlkrhG1fwDIZI94CLsLzlCK5cIbMMtFlJlfR57Lavmg==", + "license": "MIT", "engines": { "node": ">=8.0.0 || >=10.0.0" } }, "node_modules/vscode-languageserver": { "version": "7.0.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-7.0.0.tgz", - "integrity": "sha512-60HTx5ID+fLRcgdHfmz0LDZAXYEV68fzwG0JWwEPBode9NuMYTIxuYXPg4ngO8i8+Ou0lM7y6GzaYWbiDL0drw==", + "license": "MIT", "dependencies": { "vscode-languageserver-protocol": "3.16.0" }, @@ -99,8 +94,7 @@ }, "node_modules/vscode-languageserver-protocol": { "version": "3.16.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.16.0.tgz", - "integrity": "sha512-sdeUoAawceQdgIfTI+sdcwkiK2KU+2cbEYA0agzM2uqaUy2UpnnGHtWTHVEtS0ES4zHU0eMFRGN+oQgDxlD66A==", + "license": "MIT", "dependencies": { "vscode-jsonrpc": "6.0.0", "vscode-languageserver-types": "3.16.0" @@ -108,39 +102,27 @@ }, "node_modules/vscode-languageserver-textdocument": { "version": "1.0.4", - "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.4.tgz", - "integrity": "sha512-/xhqXP/2A2RSs+J8JNXpiiNVvvNM0oTosNVmQnunlKvq9o4mupHOBAnnzH0lwIPKazXKvAKsVp1kr+H/K4lgoQ==" + "license": "MIT" }, "node_modules/vscode-languageserver-types": { "version": "3.16.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0.tgz", - "integrity": "sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA==" + "license": "MIT" }, "node_modules/vscode-nls": { "version": "5.0.1", - "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.0.1.tgz", - "integrity": "sha512-hHQV6iig+M21lTdItKPkJAaWrxALQb/nqpVffakO4knJOh3DrU2SXOMzUzNgo1eADPzu3qSsJY1weCzvR52q9A==" + "license": "MIT" }, "node_modules/vscode-uri": { "version": "3.0.3", - "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.3.tgz", - "integrity": "sha512-EcswR2S8bpR7fD0YPeS7r2xXExrScVMxg4MedACaWHEtx9ftCF/qHG1xGkolzTPcEmjTavCQgbVzHUIdTMzFGA==" + "license": "MIT" }, "node_modules/yaml": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.1.tgz", - "integrity": "sha512-o96x3OPo8GjWeSLF+wOAbrPfhFOGY0W00GNaxCDv+9hkcDJEnev1yh8S7pgHF0ik6zc8sQLuL8hjHjJULZp8bw==", + "license": "ISC", "engines": { "node": ">= 14" } }, - "packages/common": { - "name": "@gxwf-server/common", - "version": "0.1.0", - "extraneous": true, - "license": "MIT", - "devDependencies": {} - }, "packages/server-common": { "name": "@gxwf/server-common", "version": "0.1.0", @@ -190,9 +172,7 @@ } }, "@types/node": { - "version": "17.0.42", - "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.42.tgz", - "integrity": "sha512-Q5BPGyGKcvQgAMbsr7qEGN/kIPN6zZecYYABeTDBizOsau+2NMdSVTar9UQw21A2+JyA2KRNDYaYrPB0Rpk2oQ==" + "version": "17.0.42" }, "gx-workflow-ls-format2": { "version": "file:gx-workflow-ls-format2", @@ -218,14 +198,10 @@ } }, "jsonc-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.0.0.tgz", - "integrity": "sha512-fQzRfAbIBnR0IQvftw9FJveWiHp72Fg20giDrHz6TdfB12UH/uue0D3hm57UB5KgAVuniLMCaS8P1IMj9NR7cA==" + "version": "3.0.0" }, "vscode-json-languageservice": { "version": "4.2.1", - "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-4.2.1.tgz", - "integrity": "sha512-xGmv9QIWs2H8obGbWg+sIPI/3/pFgj/5OWBhNzs00BkYQ9UaB2F6JJaGB/2/YOZJ3BvLXQTC4Q7muqU25QgAhA==", "requires": { "jsonc-parser": "^3.0.0", "vscode-languageserver-textdocument": "^1.0.3", @@ -235,51 +211,35 @@ } }, "vscode-jsonrpc": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-6.0.0.tgz", - "integrity": "sha512-wnJA4BnEjOSyFMvjZdpiOwhSq9uDoK8e/kpRJDTaMYzwlkrhG1fwDIZI94CLsLzlCK5cIbMMtFlJlfR57Lavmg==" + "version": "6.0.0" }, "vscode-languageserver": { "version": "7.0.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-7.0.0.tgz", - "integrity": "sha512-60HTx5ID+fLRcgdHfmz0LDZAXYEV68fzwG0JWwEPBode9NuMYTIxuYXPg4ngO8i8+Ou0lM7y6GzaYWbiDL0drw==", "requires": { "vscode-languageserver-protocol": "3.16.0" } }, "vscode-languageserver-protocol": { "version": "3.16.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.16.0.tgz", - "integrity": "sha512-sdeUoAawceQdgIfTI+sdcwkiK2KU+2cbEYA0agzM2uqaUy2UpnnGHtWTHVEtS0ES4zHU0eMFRGN+oQgDxlD66A==", "requires": { "vscode-jsonrpc": "6.0.0", "vscode-languageserver-types": "3.16.0" } }, "vscode-languageserver-textdocument": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.4.tgz", - "integrity": "sha512-/xhqXP/2A2RSs+J8JNXpiiNVvvNM0oTosNVmQnunlKvq9o4mupHOBAnnzH0lwIPKazXKvAKsVp1kr+H/K4lgoQ==" + "version": "1.0.4" }, "vscode-languageserver-types": { - "version": "3.16.0", - "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0.tgz", - "integrity": "sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA==" + "version": "3.16.0" }, "vscode-nls": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-5.0.1.tgz", - "integrity": "sha512-hHQV6iig+M21lTdItKPkJAaWrxALQb/nqpVffakO4knJOh3DrU2SXOMzUzNgo1eADPzu3qSsJY1weCzvR52q9A==" + "version": "5.0.1" }, "vscode-uri": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.3.tgz", - "integrity": "sha512-EcswR2S8bpR7fD0YPeS7r2xXExrScVMxg4MedACaWHEtx9ftCF/qHG1xGkolzTPcEmjTavCQgbVzHUIdTMzFGA==" + "version": "3.0.3" }, "yaml": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.1.1.tgz", - "integrity": "sha512-o96x3OPo8GjWeSLF+wOAbrPfhFOGY0W00GNaxCDv+9hkcDJEnev1yh8S7pgHF0ik6zc8sQLuL8hjHjJULZp8bw==" + "version": "2.1.1" } } } diff --git a/server/package.json b/server/package.json index 47ed7d1..a75b4d1 100644 --- a/server/package.json +++ b/server/package.json @@ -12,8 +12,7 @@ "watch-native-server": "webpack --watch --progress --config ./gx-workflow-ls-native/webpack.config.js", "watch-format2-server": "webpack --watch --progress --config ./gx-workflow-ls-format2/webpack.config.js", "watch": "concurrently --kill-others \"npm run watch-format2-server\" \"npm run watch-native-server\"", - "test-unit-native": "cd gx-workflow-ls-native && npm run test-unit && cd ..", - "test-unit": "npm run test-unit-native" + "test": "jest" }, "workspaces": [ "gx-workflow-ls-format2", diff --git a/server/packages/yaml-language-service/jest.config.js b/server/packages/yaml-language-service/jest.config.js deleted file mode 100644 index 5f806bf..0000000 --- a/server/packages/yaml-language-service/jest.config.js +++ /dev/null @@ -1,25 +0,0 @@ -// For a detailed explanation regarding each configuration property, visit: -// https://jestjs.io/docs/en/configuration.html - -module.exports = { - // A set of global variables that need to be available in all test environments - globals: { - "ts-jest": { - tsconfig: "server/tsconfig.json", - }, - }, - - // An array of directory names to be searched recursively up from the requiring module's location - moduleDirectories: ["node_modules"], - - // An array of file extensions your modules use - moduleFileExtensions: ["ts", "tsx", "js"], - - // The test environment that will be used for testing - testEnvironment: "node", - - // A map from regular expressions to paths to transformers - transform: { - "^.+\\.(ts|tsx)$": "ts-jest", - }, -}; diff --git a/server/packages/yaml-language-service/package.json b/server/packages/yaml-language-service/package.json index 3188cd9..772a89e 100644 --- a/server/packages/yaml-language-service/package.json +++ b/server/packages/yaml-language-service/package.json @@ -11,5 +11,7 @@ "vscode-uri": "^3.0.3", "yaml": "^2.1.1" }, - "scripts": {} + "scripts": { + "test": "jest" + } } diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..209a682 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "es2019", + "lib": ["ES2019", "WebWorker"], + "module": "commonjs", + "moduleResolution": "node", + "resolveJsonModule": true, + "esModuleInterop": true, + "sourceMap": true, + "strict": true, + "composite": true + }, + "files": [], + "include": [], + "exclude": [] +} From dea9d02534c95a3d71928b5e4413043d00467b7e Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 19 Jun 2022 17:24:55 +0200 Subject: [PATCH 05/11] Add YAML syntax validation for format2 workflows --- .../src/gxFormat2WorkflowDocument.ts | 11 +++++++++-- server/gx-workflow-ls-format2/src/languageService.ts | 6 ++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/server/gx-workflow-ls-format2/src/gxFormat2WorkflowDocument.ts b/server/gx-workflow-ls-format2/src/gxFormat2WorkflowDocument.ts index 2af39ad..71e653b 100644 --- a/server/gx-workflow-ls-format2/src/gxFormat2WorkflowDocument.ts +++ b/server/gx-workflow-ls-format2/src/gxFormat2WorkflowDocument.ts @@ -1,16 +1,23 @@ import { ObjectASTNode } from "vscode-json-languageservice"; import { TextDocument, Range, Position, ASTNode, WorkflowDocument } from "@gxwf/server-common/src/languageTypes"; +import { YAMLDocument } from "@gxwf/yaml-language-service/src"; /** * This class provides information about a gxformat2 workflow document structure. */ export class GxFormat2WorkflowDocument extends WorkflowDocument { - constructor(textDocument: TextDocument) { + private _yamlDocument: YAMLDocument; + constructor(textDocument: TextDocument, yamlDocument: YAMLDocument) { super(textDocument); + this._yamlDocument = yamlDocument; } public get rootNode(): ASTNode | undefined { - return; + return this._yamlDocument.mainDocument?.root; + } + + public get yamlDocument(): YAMLDocument { + return this._yamlDocument; } public override getNodeAtPosition(position: Position): ASTNode | undefined { diff --git a/server/gx-workflow-ls-format2/src/languageService.ts b/server/gx-workflow-ls-format2/src/languageService.ts index 923c167..66fa1f0 100644 --- a/server/gx-workflow-ls-format2/src/languageService.ts +++ b/server/gx-workflow-ls-format2/src/languageService.ts @@ -25,7 +25,8 @@ export class GxFormat2WorkflowLanguageService extends WorkflowLanguageService { } public override parseWorkflowDocument(document: TextDocument): WorkflowDocument { - return new GxFormat2WorkflowDocument(document); + const yamlDocument = this._yamlLanguageService.parseYAMLDocument(document); + return new GxFormat2WorkflowDocument(document, yamlDocument); } public override format(document: TextDocument, range: Range, options: FormattingOptions): TextEdit[] { @@ -44,6 +45,7 @@ export class GxFormat2WorkflowLanguageService extends WorkflowLanguageService { } protected override async doValidation(workflowDocument: WorkflowDocument): Promise { - return []; + const format2WorkflowDocument = workflowDocument as GxFormat2WorkflowDocument; + return this._yamlLanguageService.doValidation(format2WorkflowDocument.yamlDocument); } } From 2c3de1b7ba2194dce315a07744e023369d79a744 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 19 Jun 2022 18:39:21 +0200 Subject: [PATCH 06/11] Add basic YAML formatting for gxformat2 documents --- README.md | 2 +- .../src/languageService.ts | 4 ++-- .../src/services/yamlFormatter.ts | 20 +++++++++++-------- .../src/yamlLanguageService.ts | 16 +++++++-------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 1e27b09..d463a52 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ The following table shows all the implemented features and the current support f | [Validation](#workflow-validation) | ✔️ | 🔜 | | [Documentation on Hover](#documentation-on-hover) | ✔️ | 🔜 | | [IntelliSense](#intellisense) | ✔️ | 🔜 | -| [Formatting](#formatting) | ✔️ | 🔜 | +| [Formatting](#formatting) | ✔️ | ✔️ | | [Custom Outline](#custom-outline) | ✔️ | 🔜 | | [Workflow Cleanup Command](#workflow-cleanup-command) | ✔️ | ❔ | | [Simplified Workflow Diffs](#simplified-workflow-diffs) | 🔶 | ❔ | diff --git a/server/gx-workflow-ls-format2/src/languageService.ts b/server/gx-workflow-ls-format2/src/languageService.ts index 66fa1f0..9c36b7c 100644 --- a/server/gx-workflow-ls-format2/src/languageService.ts +++ b/server/gx-workflow-ls-format2/src/languageService.ts @@ -29,8 +29,8 @@ export class GxFormat2WorkflowLanguageService extends WorkflowLanguageService { return new GxFormat2WorkflowDocument(document, yamlDocument); } - public override format(document: TextDocument, range: Range, options: FormattingOptions): TextEdit[] { - return []; + public override format(document: TextDocument, _: Range, options: FormattingOptions): TextEdit[] { + return this._yamlLanguageService.doFormat(document, options); } public override async doHover(workflowDocument: WorkflowDocument, position: Position): Promise { diff --git a/server/packages/yaml-language-service/src/services/yamlFormatter.ts b/server/packages/yaml-language-service/src/services/yamlFormatter.ts index bd7f19d..af28c09 100644 --- a/server/packages/yaml-language-service/src/services/yamlFormatter.ts +++ b/server/packages/yaml-language-service/src/services/yamlFormatter.ts @@ -1,15 +1,14 @@ -"use strict"; - import { Range, Position, TextEdit, FormattingOptions } from "vscode-languageserver-types"; import { CustomFormatterOptions, LanguageSettings } from "../yamlLanguageService"; import { TextDocument } from "vscode-languageserver-textdocument"; +import { parse, stringify, ToStringOptions } from "yaml"; export class YAMLFormatter { - private formatterEnabled = true; + private formatterEnabled? = true; - public configure(shouldFormat: LanguageSettings): void { - if (shouldFormat) { - this.formatterEnabled = shouldFormat.format || false; + public configure(settings: LanguageSettings): void { + if (settings) { + this.formatterEnabled = settings.format; } } @@ -20,8 +19,13 @@ export class YAMLFormatter { try { const text = document.getText(); - // TODO implement formatter - const formatted = text; + const ymlDoc = parse(text); + const yamlFormatOptions: ToStringOptions = { + singleQuote: options.singleQuote, + lineWidth: options.lineWidth, + indent: options.tabSize, + }; + const formatted = stringify(ymlDoc, yamlFormatOptions); return [TextEdit.replace(Range.create(Position.create(0, 0), document.positionAt(text.length)), formatted)]; } catch (error) { diff --git a/server/packages/yaml-language-service/src/yamlLanguageService.ts b/server/packages/yaml-language-service/src/yamlLanguageService.ts index 1bca5c3..1a23010 100644 --- a/server/packages/yaml-language-service/src/yamlLanguageService.ts +++ b/server/packages/yaml-language-service/src/yamlLanguageService.ts @@ -1,5 +1,5 @@ import { TextDocument } from "vscode-languageserver-textdocument"; -import { Diagnostic, FormattingOptions, Hover, Position, TextEdit } from "vscode-languageserver-types"; +import { Diagnostic, FormattingOptions, TextEdit } from "vscode-languageserver-types"; import { YAMLFormatter } from "./services/yamlFormatter"; import { parse as parseYAML, YAMLDocument } from "./parser"; import { YAMLValidation } from "./services/yamlValidation"; @@ -12,28 +12,26 @@ export interface LanguageSettings { export interface CustomFormatterOptions { singleQuote?: boolean; - bracketSpacing?: boolean; - proseWrap?: string; - printWidth?: number; - enable?: boolean; + lineWidth?: number; } export interface LanguageService { + configure(settings: LanguageSettings): void; parseYAMLDocument(document: TextDocument): YAMLDocument; doValidation(yamlDocument: YAMLDocument): Promise; doFormat(document: TextDocument, options: FormattingOptions & CustomFormatterOptions): TextEdit[]; - doHover(document: TextDocument, position: Position): Hover | null; } export function getLanguageService(): LanguageService { const formatter = new YAMLFormatter(); const validator = new YAMLValidation(); return { + configure: (settings: LanguageSettings) => { + formatter.configure(settings); + validator.configure(settings); + }, parseYAMLDocument: (document: TextDocument) => parseYAML(document), doValidation: (yamlDocument: YAMLDocument) => validator.doValidation(yamlDocument), doFormat: formatter.format.bind(formatter), - doHover: (doc, pos) => { - return null; - }, }; } From 3974e646c7822d0c0f5a7f95028caa305b50264a Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 21 Jun 2022 00:48:38 +0200 Subject: [PATCH 07/11] Refactor node manager for workflow documents Move the abstract syntax tree (AST) logic to a manager class that wraps the parsed document. --- .../src/gxFormat2WorkflowDocument.ts | 58 +--------- .../src/nativeWorkflowDocument.ts | 81 +------------- ...sonUtils.test.ts => astUtils-json.test.ts} | 31 +----- .../tests/unit/nativeWorkflowDocument.test.ts | 2 +- .../server-common/src/ast/nodeManager.ts | 100 ++++++++++++++++++ .../src/{astTypes.ts => ast/types.ts} | 4 + .../server-common/src/ast/utils.ts} | 28 ++++- .../server-common/src/languageTypes.ts | 18 ---- .../src/models/workflowDocument.ts | 63 ++--------- .../src/providers/symbolsProvider.ts | 19 ++-- .../validation/MissingPropertyValidation.ts | 4 +- .../WorkflowOutputLabelValidation.ts | 4 +- .../src/services/cleanWorkflow.ts | 11 +- .../server-common/tests/testHelpers.ts | 6 ++ .../server-common/tests/unit/astUtils.test.ts | 23 ++++ server/packages/server-common/tsconfig.json | 2 +- .../src/parser/astTypes.ts | 2 +- .../src/parser/yamlDocument.ts | 7 +- 18 files changed, 204 insertions(+), 259 deletions(-) rename server/gx-workflow-ls-native/tests/unit/{jsonUtils.test.ts => astUtils-json.test.ts} (59%) create mode 100644 server/packages/server-common/src/ast/nodeManager.ts rename server/packages/server-common/src/{astTypes.ts => ast/types.ts} (86%) rename server/{gx-workflow-ls-native/src/jsonUtils.ts => packages/server-common/src/ast/utils.ts} (60%) create mode 100644 server/packages/server-common/tests/testHelpers.ts create mode 100644 server/packages/server-common/tests/unit/astUtils.test.ts diff --git a/server/gx-workflow-ls-format2/src/gxFormat2WorkflowDocument.ts b/server/gx-workflow-ls-format2/src/gxFormat2WorkflowDocument.ts index 71e653b..633c817 100644 --- a/server/gx-workflow-ls-format2/src/gxFormat2WorkflowDocument.ts +++ b/server/gx-workflow-ls-format2/src/gxFormat2WorkflowDocument.ts @@ -1,5 +1,4 @@ -import { ObjectASTNode } from "vscode-json-languageservice"; -import { TextDocument, Range, Position, ASTNode, WorkflowDocument } from "@gxwf/server-common/src/languageTypes"; +import { TextDocument, WorkflowDocument } from "@gxwf/server-common/src/languageTypes"; import { YAMLDocument } from "@gxwf/yaml-language-service/src"; /** @@ -8,64 +7,11 @@ import { YAMLDocument } from "@gxwf/yaml-language-service/src"; export class GxFormat2WorkflowDocument extends WorkflowDocument { private _yamlDocument: YAMLDocument; constructor(textDocument: TextDocument, yamlDocument: YAMLDocument) { - super(textDocument); + super(textDocument, yamlDocument); this._yamlDocument = yamlDocument; } - public get rootNode(): ASTNode | undefined { - return this._yamlDocument.mainDocument?.root; - } - public get yamlDocument(): YAMLDocument { return this._yamlDocument; } - - public override getNodeAtPosition(position: Position): ASTNode | undefined { - return; - } - - public override getDocumentRange(): Range { - return Range.create(this.textDocument.positionAt(0), this.textDocument.positionAt(1)); - } - - public getNodeRange(node: ASTNode): Range { - return Range.create( - this.textDocument.positionAt(node.offset), - this.textDocument.positionAt(node.offset + node.length) - ); - } - - public getNodeRangeAtPosition(position: Position): Range { - const node = this.getNodeAtPosition(position); - return node ? this.getNodeRange(node) : this.getDefaultRangeAtPosition(position); - } - - public isLastNodeInParent(node: ASTNode): boolean { - const parent = node.parent; - if (!parent || !parent.children) { - return true; // Must be root - } - const lastNode = parent.children[parent.children.length - 1]; - return node === lastNode; - } - - public getPreviousSiblingNode(node: ASTNode): ASTNode | null { - const parent = node.parent; - if (!parent || !parent.children) { - return null; - } - const previousNodeIndex = parent.children.indexOf(node) - 1; - if (previousNodeIndex < 0) { - return null; - } - return parent.children[previousNodeIndex]; - } - - public override getNodeFromPath(path: string): ASTNode | null { - return null; - } - - public override getStepNodes(): ObjectASTNode[] { - return []; - } } diff --git a/server/gx-workflow-ls-native/src/nativeWorkflowDocument.ts b/server/gx-workflow-ls-native/src/nativeWorkflowDocument.ts index ff02dfc..c49ba8f 100644 --- a/server/gx-workflow-ls-native/src/nativeWorkflowDocument.ts +++ b/server/gx-workflow-ls-native/src/nativeWorkflowDocument.ts @@ -1,6 +1,5 @@ -import { JSONDocument, ObjectASTNode } from "vscode-json-languageservice"; -import { getPropertyNodeFromPath } from "./jsonUtils"; -import { TextDocument, Range, Position, ASTNode, WorkflowDocument } from "@gxwf/server-common/src/languageTypes"; +import { JSONDocument } from "vscode-json-languageservice"; +import { TextDocument, WorkflowDocument } from "@gxwf/server-common/src/languageTypes"; /** * This class provides information about a Native workflow document structure. @@ -9,85 +8,11 @@ export class NativeWorkflowDocument extends WorkflowDocument { private _jsonDocument: JSONDocument; constructor(textDocument: TextDocument, jsonDocument: JSONDocument) { - super(textDocument); + super(textDocument, jsonDocument); this._jsonDocument = jsonDocument; } public get jsonDocument(): JSONDocument { return this._jsonDocument; } - - public get rootNode(): ASTNode | undefined { - return this._jsonDocument.root; - } - - public override getNodeAtPosition(position: Position): ASTNode | undefined { - const offset = this.textDocument.offsetAt(position); - return this.jsonDocument.getNodeFromOffset(offset); - } - - public override getDocumentRange(): Range { - const root = this.jsonDocument.root; - if (root) { - return Range.create(this.textDocument.positionAt(root.offset), this.textDocument.positionAt(root.length)); - } - return Range.create(this.textDocument.positionAt(0), this.textDocument.positionAt(1)); - } - - public getNodeRange(node: ASTNode): Range { - return Range.create( - this.textDocument.positionAt(node.offset), - this.textDocument.positionAt(node.offset + node.length) - ); - } - - public getNodeRangeAtPosition(position: Position): Range { - const node = this.getNodeAtPosition(position); - return node ? this.getNodeRange(node) : this.getDefaultRangeAtPosition(position); - } - - public isLastNodeInParent(node: ASTNode): boolean { - const parent = node.parent; - if (!parent || !parent.children) { - return true; // Must be root - } - const lastNode = parent.children[parent.children.length - 1]; - return node === lastNode; - } - - public getPreviousSiblingNode(node: ASTNode): ASTNode | null { - const parent = node.parent; - if (!parent || !parent.children) { - return null; - } - const previousNodeIndex = parent.children.indexOf(node) - 1; - if (previousNodeIndex < 0) { - return null; - } - return parent.children[previousNodeIndex]; - } - - public override getNodeFromPath(path: string): ASTNode | null { - const root = this._jsonDocument.root; - if (!root) return null; - return getPropertyNodeFromPath(root, path); - } - - public override getStepNodes(): ObjectASTNode[] { - const root = this._jsonDocument.root; - if (!root) { - return []; - } - const result: ObjectASTNode[] = []; - const stepsNode = this.getNodeFromPath("steps"); - if (stepsNode && stepsNode.type === "property" && stepsNode.valueNode && stepsNode.valueNode.type === "object") { - stepsNode.valueNode.properties.forEach((stepProperty) => { - const stepNode = stepProperty.valueNode; - if (stepNode && stepNode.type === "object") { - result.push(stepNode); - } - }); - } - return result; - } } diff --git a/server/gx-workflow-ls-native/tests/unit/jsonUtils.test.ts b/server/gx-workflow-ls-native/tests/unit/astUtils-json.test.ts similarity index 59% rename from server/gx-workflow-ls-native/tests/unit/jsonUtils.test.ts rename to server/gx-workflow-ls-native/tests/unit/astUtils-json.test.ts index f9790ea..36f0441 100644 --- a/server/gx-workflow-ls-native/tests/unit/jsonUtils.test.ts +++ b/server/gx-workflow-ls-native/tests/unit/astUtils-json.test.ts @@ -1,28 +1,8 @@ -import { ASTNode, PropertyASTNode } from "vscode-json-languageservice"; -import { getPathSegments, getPropertyNodeFromPath } from "../../src/jsonUtils"; +import { getPropertyNodeFromPath } from "@gxwf/server-common/src/ast/utils"; import { getJsonDocumentRoot } from "../testHelpers"; +import { expectPropertyNodeToHaveKey } from "@gxwf/server-common/tests/testHelpers"; -describe("JSON Utility Functions", () => { - describe("getPathSegments", () => { - it.each([ - ["", []], - ["/", [""]], - ["a", ["a"]], - ["/a", ["a"]], - ["a/b", ["a", "b"]], - ["a/", ["a", ""]], - [".", [""]], - [".a", ["a"]], - ["a.b", ["a", "b"]], - ["a.", ["a", ""]], - ])("returns the expected segments", (path: string, expected: string[]) => { - const segments = getPathSegments(path); - - expect(segments).toHaveLength(expected.length); - expect(segments).toEqual(expected); - }); - }); - +describe("AST Utility Functions with JSON", () => { describe("getPropertyNodeFromPath", () => { describe("with valid path", () => { it.each([ @@ -59,8 +39,3 @@ describe("JSON Utility Functions", () => { }); }); }); - -function expectPropertyNodeToHaveKey(propertyNode: ASTNode | null, expectedPropertyKey: string): void { - expect(propertyNode?.type).toBe("property"); - expect((propertyNode as PropertyASTNode).keyNode.value).toBe(expectedPropertyKey); -} diff --git a/server/gx-workflow-ls-native/tests/unit/nativeWorkflowDocument.test.ts b/server/gx-workflow-ls-native/tests/unit/nativeWorkflowDocument.test.ts index feca4ee..3aa6217 100644 --- a/server/gx-workflow-ls-native/tests/unit/nativeWorkflowDocument.test.ts +++ b/server/gx-workflow-ls-native/tests/unit/nativeWorkflowDocument.test.ts @@ -10,7 +10,7 @@ describe("NativeWorkflowDocument", () => { [TestWorkflowProvider.workflows.validation.withThreeSteps, 3], ])("returns the expected number of steps", (wf_content: string, expectedNumSteps: number) => { const wfDocument = createNativeWorkflowDocument(wf_content); - const stepNodes = wfDocument.getStepNodes(); + const stepNodes = wfDocument.nodeManager.getStepNodes(); expect(stepNodes).toHaveLength(expectedNumSteps); }); }); diff --git a/server/packages/server-common/src/ast/nodeManager.ts b/server/packages/server-common/src/ast/nodeManager.ts new file mode 100644 index 0000000..ecd3726 --- /dev/null +++ b/server/packages/server-common/src/ast/nodeManager.ts @@ -0,0 +1,100 @@ +import { Position, Range, TextDocument } from "../languageTypes"; +import { ParsedDocument, ASTNode, ObjectASTNode } from "./types"; +import { findNodeAtOffset, getPropertyNodeFromPath } from "./utils"; + +export class ASTNodeManager { + constructor(private readonly textDocument: TextDocument, private readonly parsedDocument: ParsedDocument) {} + + public get root(): ASTNode | undefined { + return this.parsedDocument.root; + } + + public getNodeFromOffset(offset: number, includeRightBound = false): ASTNode | undefined { + if (this.root) { + return findNodeAtOffset(this.root, offset, includeRightBound); + } + return undefined; + } + + public getNodeAtPosition(position: Position): ASTNode | undefined { + const offset = this.textDocument.offsetAt(position); + return this.getNodeFromOffset(offset); + } + + public getDocumentRange(): Range { + if (this.root) { + return Range.create( + this.textDocument.positionAt(this.root.offset), + this.textDocument.positionAt(this.root.length) + ); + } + return Range.create(this.textDocument.positionAt(0), this.textDocument.positionAt(1)); + } + + /** Returns a small Range at the beginning of the document */ + public getDefaultRange(): Range { + return Range.create(this.textDocument.positionAt(0), this.textDocument.positionAt(1)); + } + + public getNodeRange(node: ASTNode): Range { + return Range.create( + this.textDocument.positionAt(node.offset), + this.textDocument.positionAt(node.offset + node.length) + ); + } + + public getNodeRangeAtPosition(position: Position): Range { + const node = this.getNodeAtPosition(position); + return node ? this.getNodeRange(node) : this.getDefaultRangeAtPosition(position); + } + + public isLastNodeInParent(node: ASTNode): boolean { + const parent = node.parent; + if (!parent || !parent.children) { + return true; // Must be root + } + const lastNode = parent.children[parent.children.length - 1]; + return node === lastNode; + } + + public getPreviousSiblingNode(node: ASTNode): ASTNode | null { + const parent = node.parent; + if (!parent || !parent.children) { + return null; + } + const previousNodeIndex = parent.children.indexOf(node) - 1; + if (previousNodeIndex < 0) { + return null; + } + return parent.children[previousNodeIndex]; + } + + public getNodeFromPath(path: string): ASTNode | null { + const root = this.root; + if (!root) return null; + return getPropertyNodeFromPath(root, path); + } + + public getStepNodes(): ObjectASTNode[] { + const root = this.root; + if (!root) { + return []; + } + const result: ObjectASTNode[] = []; + const stepsNode = this.getNodeFromPath("steps"); + if (stepsNode && stepsNode.type === "property" && stepsNode.valueNode && stepsNode.valueNode.type === "object") { + stepsNode.valueNode.properties.forEach((stepProperty) => { + const stepNode = stepProperty.valueNode; + if (stepNode && stepNode.type === "object") { + result.push(stepNode); + } + }); + } + return result; + } + + protected getDefaultRangeAtPosition(position: Position): Range { + const offset = this.textDocument.offsetAt(position); + return Range.create(this.textDocument.positionAt(offset), this.textDocument.positionAt(offset + 1)); + } +} diff --git a/server/packages/server-common/src/astTypes.ts b/server/packages/server-common/src/ast/types.ts similarity index 86% rename from server/packages/server-common/src/astTypes.ts rename to server/packages/server-common/src/ast/types.ts index 8d0a0ed..5af7b5a 100644 --- a/server/packages/server-common/src/astTypes.ts +++ b/server/packages/server-common/src/ast/types.ts @@ -21,3 +21,7 @@ export { PropertyASTNode, StringASTNode, }; + +export interface ParsedDocument { + root?: ASTNode; +} diff --git a/server/gx-workflow-ls-native/src/jsonUtils.ts b/server/packages/server-common/src/ast/utils.ts similarity index 60% rename from server/gx-workflow-ls-native/src/jsonUtils.ts rename to server/packages/server-common/src/ast/utils.ts index 14f6121..220bc0d 100644 --- a/server/gx-workflow-ls-native/src/jsonUtils.ts +++ b/server/packages/server-common/src/ast/utils.ts @@ -1,4 +1,4 @@ -import { ASTNode } from "@gxwf/server-common/src/languageTypes"; +import { ASTNode } from "./types"; export function getPathSegments(path: string): string[] | null { const segments = path.split(/[/.]/); @@ -39,3 +39,29 @@ export function getPropertyNodeFromPath(root: ASTNode, path: string): ASTNode | } return currentNode; } + +export function contains(node: ASTNode, offset: number, includeRightBound = false): boolean { + return ( + (offset >= node.offset && offset <= node.offset + node.length) || + (includeRightBound && offset === node.offset + node.length) + ); +} + +export function findNodeAtOffset(node: ASTNode, offset: number, includeRightBound: boolean): ASTNode | undefined { + if (includeRightBound === void 0) { + includeRightBound = false; + } + if (contains(node, offset, includeRightBound)) { + const children = node.children; + if (Array.isArray(children)) { + for (let i = 0; i < children.length && children[i].offset <= offset; i++) { + const item = findNodeAtOffset(children[i], offset, includeRightBound); + if (item) { + return item; + } + } + } + return node; + } + return undefined; +} diff --git a/server/packages/server-common/src/languageTypes.ts b/server/packages/server-common/src/languageTypes.ts index e4888c8..744a648 100644 --- a/server/packages/server-common/src/languageTypes.ts +++ b/server/packages/server-common/src/languageTypes.ts @@ -48,28 +48,10 @@ import { DocumentSymbolParams, } from "vscode-languageserver/browser"; import { WorkflowDocument } from "./models/workflowDocument"; -import { - ASTNode, - ArrayASTNode, - ObjectASTNode, - PropertyASTNode, - StringASTNode, - BooleanASTNode, - NumberASTNode, - NullASTNode, -} from "vscode-json-languageservice"; import { WorkflowDocuments } from "./models/workflowDocuments"; import { GalaxyWorkflowLanguageServer } from "./server"; export { - ASTNode, - ArrayASTNode, - ObjectASTNode, - PropertyASTNode, - StringASTNode, - BooleanASTNode, - NumberASTNode, - NullASTNode, TextDocument, Range, Position, diff --git a/server/packages/server-common/src/models/workflowDocument.ts b/server/packages/server-common/src/models/workflowDocument.ts index fcb010d..ab595b3 100644 --- a/server/packages/server-common/src/models/workflowDocument.ts +++ b/server/packages/server-common/src/models/workflowDocument.ts @@ -1,5 +1,7 @@ -import { TextDocument, Range, Position, ASTNode, ObjectASTNode } from "../languageTypes"; +import { TextDocument } from "../languageTypes"; import { URI } from "vscode-uri"; +import { ParsedDocument } from "../ast/types"; +import { ASTNodeManager } from "../ast/nodeManager"; /** * This class contains information about workflow semantics. @@ -7,10 +9,13 @@ import { URI } from "vscode-uri"; export abstract class WorkflowDocument { protected _textDocument: TextDocument; protected _documentUri: URI; - public abstract readonly rootNode: ASTNode | undefined; + protected _parsedDocument: ParsedDocument; + protected _nodeManager: ASTNodeManager; - constructor(textDocument: TextDocument) { + constructor(textDocument: TextDocument, parsedDocument: ParsedDocument) { this._textDocument = textDocument; + this._parsedDocument = parsedDocument; + this._nodeManager = new ASTNodeManager(textDocument, parsedDocument); this._documentUri = URI.parse(this._textDocument.uri); } @@ -22,54 +27,8 @@ export abstract class WorkflowDocument { return this._textDocument; } - public abstract getNodeAtPosition(position: Position): ASTNode | undefined; - - public abstract getDocumentRange(): Range; - - public abstract getNodeFromPath(path: string): ASTNode | null; - - public abstract getStepNodes(): ObjectASTNode[]; - - /** Returns a small Range at the beginning of the document */ - public getDefaultRange(): Range { - return Range.create(this.textDocument.positionAt(0), this.textDocument.positionAt(1)); - } - - public getNodeRange(node: ASTNode): Range { - return Range.create( - this.textDocument.positionAt(node.offset), - this.textDocument.positionAt(node.offset + node.length) - ); - } - - public getNodeRangeAtPosition(position: Position): Range { - const node = this.getNodeAtPosition(position); - return node ? this.getNodeRange(node) : this.getDefaultRangeAtPosition(position); - } - - public isLastNodeInParent(node: ASTNode): boolean { - const parent = node.parent; - if (!parent || !parent.children) { - return true; // Must be root - } - const lastNode = parent.children[parent.children.length - 1]; - return node === lastNode; - } - - public getPreviousSiblingNode(node: ASTNode): ASTNode | null { - const parent = node.parent; - if (!parent || !parent.children) { - return null; - } - const previousNodeIndex = parent.children.indexOf(node) - 1; - if (previousNodeIndex < 0) { - return null; - } - return parent.children[previousNodeIndex]; - } - - protected getDefaultRangeAtPosition(position: Position): Range { - const offset = this.textDocument.offsetAt(position); - return Range.create(this.textDocument.positionAt(offset), this.textDocument.positionAt(offset + 1)); + /** Abstract Syntax Tree Node Manager associated with this document. */ + public get nodeManager(): ASTNodeManager { + return this._nodeManager; } } diff --git a/server/packages/server-common/src/providers/symbolsProvider.ts b/server/packages/server-common/src/providers/symbolsProvider.ts index 418fc81..1d1f5a1 100644 --- a/server/packages/server-common/src/providers/symbolsProvider.ts +++ b/server/packages/server-common/src/providers/symbolsProvider.ts @@ -1,12 +1,5 @@ -import { - DocumentSymbolParams, - DocumentSymbol, - SymbolKind, - ASTNode, - PropertyASTNode, - ObjectASTNode, - WorkflowDocument, -} from "../languageTypes"; +import { ASTNode, ObjectASTNode, PropertyASTNode } from "../ast/types"; +import { DocumentSymbolParams, DocumentSymbol, SymbolKind, WorkflowDocument } from "../languageTypes"; import { GalaxyWorkflowLanguageServer } from "../server"; import { Provider } from "./provider"; @@ -32,7 +25,7 @@ export class SymbolsProvider extends Provider { } private getSymbols(workflowDocument: WorkflowDocument): DocumentSymbol[] { - const root = workflowDocument.rootNode; + const root = workflowDocument.nodeManager.root; if (!root) { return []; } @@ -48,7 +41,7 @@ export class SymbolsProvider extends Provider { if (IGNORE_SYMBOL_NAMES.has(name)) { return; } - const range = workflowDocument.getNodeRange(node); + const range = workflowDocument.nodeManager.getNodeRange(node); const selectionRange = range; const symbol = { name, kind: this.getSymbolKind(node.type), range, selectionRange, children: [] }; result.push(symbol); @@ -70,8 +63,8 @@ export class SymbolsProvider extends Provider { if (IGNORE_SYMBOL_NAMES.has(name)) { return; } - const range = workflowDocument.getNodeRange(property); - const selectionRange = workflowDocument.getNodeRange(property.keyNode); + const range = workflowDocument.nodeManager.getNodeRange(property); + const selectionRange = workflowDocument.nodeManager.getNodeRange(property.keyNode); const children: DocumentSymbol[] = []; const symbol: DocumentSymbol = { name: name, diff --git a/server/packages/server-common/src/providers/validation/MissingPropertyValidation.ts b/server/packages/server-common/src/providers/validation/MissingPropertyValidation.ts index 1765d84..232fac2 100644 --- a/server/packages/server-common/src/providers/validation/MissingPropertyValidation.ts +++ b/server/packages/server-common/src/providers/validation/MissingPropertyValidation.ts @@ -11,11 +11,11 @@ export class MissingPropertyValidationRule implements ValidationRule { validate(workflowDocument: WorkflowDocument): Promise { const result: Diagnostic[] = []; - const targetNode = workflowDocument.getNodeFromPath(this.nodePath); + const targetNode = workflowDocument.nodeManager.getNodeFromPath(this.nodePath); if (!targetNode) { result.push({ message: `Missing property "${this.nodePath}".`, - range: workflowDocument.getDefaultRange(), + range: workflowDocument.nodeManager.getDefaultRange(), severity: this.severity, }); } diff --git a/server/packages/server-common/src/providers/validation/WorkflowOutputLabelValidation.ts b/server/packages/server-common/src/providers/validation/WorkflowOutputLabelValidation.ts index 5922500..1c54742 100644 --- a/server/packages/server-common/src/providers/validation/WorkflowOutputLabelValidation.ts +++ b/server/packages/server-common/src/providers/validation/WorkflowOutputLabelValidation.ts @@ -9,7 +9,7 @@ export class WorkflowOutputLabelValidation implements ValidationRule { validate(workflowDocument: WorkflowDocument): Promise { const result: Diagnostic[] = []; - const stepNodes = workflowDocument.getStepNodes(); + const stepNodes = workflowDocument.nodeManager.getStepNodes(); stepNodes.forEach((step) => { const workflowOutputs = step.properties.find((property) => property.keyNode.value === "workflow_outputs"); if (workflowOutputs && workflowOutputs.valueNode && workflowOutputs.valueNode.type === "array") { @@ -19,7 +19,7 @@ export class WorkflowOutputLabelValidation implements ValidationRule { if (!labelNode?.valueNode?.value) { result.push({ message: `Missing label in workflow output.`, - range: workflowDocument.getNodeRange(outputNode), + range: workflowDocument.nodeManager.getNodeRange(outputNode), severity: this.severity, }); } diff --git a/server/packages/server-common/src/services/cleanWorkflow.ts b/server/packages/server-common/src/services/cleanWorkflow.ts index 2d06666..1050960 100644 --- a/server/packages/server-common/src/services/cleanWorkflow.ts +++ b/server/packages/server-common/src/services/cleanWorkflow.ts @@ -1,6 +1,6 @@ import { ApplyWorkspaceEditParams, Range, TextDocumentEdit, TextEdit } from "vscode-languageserver"; import { TextDocument } from "vscode-languageserver-textdocument"; -import { ASTNode, PropertyASTNode, WorkflowDocument } from "../languageTypes"; +import { WorkflowDocument } from "../languageTypes"; import { GalaxyWorkflowLanguageServer } from "../server"; import { ServiceBase } from "."; import { @@ -11,6 +11,7 @@ import { CleanWorkflowDocumentRequest, CleanWorkflowDocumentResult, } from "./requestsDefinitions"; +import { ASTNode, PropertyASTNode } from "../ast/types"; /** * Service for handling workflow `cleaning` requests. @@ -101,11 +102,11 @@ export class CleanWorkflowService extends ServiceBase { changes.push(TextEdit.replace(range, "")); // Remove trailing comma in previous property node - const isLastNode = workflowDocument.isLastNodeInParent(node); + const isLastNode = workflowDocument.nodeManager.isLastNodeInParent(node); if (isLastNode) { - let previousNode = workflowDocument.getPreviousSiblingNode(node); + let previousNode = workflowDocument.nodeManager.getPreviousSiblingNode(node); while (previousNode && nodesToRemove.includes(previousNode)) { - previousNode = workflowDocument.getPreviousSiblingNode(previousNode); + previousNode = workflowDocument.nodeManager.getPreviousSiblingNode(previousNode); } if (previousNode) { const range = this.getFullNodeRange(workflowDocument.textDocument, previousNode); @@ -135,7 +136,7 @@ export class CleanWorkflowService extends ServiceBase { } private getNonEssentialNodes(workflowDocument: WorkflowDocument, cleanablePropertyNames: string[]): ASTNode[] { - const root = workflowDocument.rootNode; + const root = workflowDocument.nodeManager.root; if (!root) { return []; } diff --git a/server/packages/server-common/tests/testHelpers.ts b/server/packages/server-common/tests/testHelpers.ts new file mode 100644 index 0000000..0e56c61 --- /dev/null +++ b/server/packages/server-common/tests/testHelpers.ts @@ -0,0 +1,6 @@ +import { ASTNode, PropertyASTNode } from "../src/ast/types"; + +export function expectPropertyNodeToHaveKey(propertyNode: ASTNode | null, expectedPropertyKey: string): void { + expect(propertyNode?.type).toBe("property"); + expect((propertyNode as PropertyASTNode).keyNode.value).toBe(expectedPropertyKey); +} diff --git a/server/packages/server-common/tests/unit/astUtils.test.ts b/server/packages/server-common/tests/unit/astUtils.test.ts new file mode 100644 index 0000000..cbc996f --- /dev/null +++ b/server/packages/server-common/tests/unit/astUtils.test.ts @@ -0,0 +1,23 @@ +import { getPathSegments } from "@gxwf/server-common/src/ast/utils"; + +describe("AST Utility Functions", () => { + describe("getPathSegments", () => { + it.each([ + ["", []], + ["/", [""]], + ["a", ["a"]], + ["/a", ["a"]], + ["a/b", ["a", "b"]], + ["a/", ["a", ""]], + [".", [""]], + [".a", ["a"]], + ["a.b", ["a", "b"]], + ["a.", ["a", ""]], + ])("returns the expected segments", (path: string, expected: string[]) => { + const segments = getPathSegments(path); + + expect(segments).toHaveLength(expected.length); + expect(segments).toEqual(expected); + }); + }); +}); diff --git a/server/packages/server-common/tsconfig.json b/server/packages/server-common/tsconfig.json index 1e1120b..eae02d0 100644 --- a/server/packages/server-common/tsconfig.json +++ b/server/packages/server-common/tsconfig.json @@ -10,7 +10,7 @@ "strict": true, "composite": true, "declaration": true, - "rootDir": "./src", + "rootDirs": ["src", "tests"], "baseUrl": "." }, "include": ["src/**/*"] diff --git a/server/packages/yaml-language-service/src/parser/astTypes.ts b/server/packages/yaml-language-service/src/parser/astTypes.ts index c043fbd..7d07f0f 100644 --- a/server/packages/yaml-language-service/src/parser/astTypes.ts +++ b/server/packages/yaml-language-service/src/parser/astTypes.ts @@ -8,7 +8,7 @@ import { ObjectASTNode, PropertyASTNode, StringASTNode, -} from "@gxwf/server-common/src/astTypes"; +} from "@gxwf/server-common/src/ast/types"; import { Node, Pair } from "yaml"; export { diff --git a/server/packages/yaml-language-service/src/parser/yamlDocument.ts b/server/packages/yaml-language-service/src/parser/yamlDocument.ts index 9ec3e90..faa8310 100644 --- a/server/packages/yaml-language-service/src/parser/yamlDocument.ts +++ b/server/packages/yaml-language-service/src/parser/yamlDocument.ts @@ -1,3 +1,4 @@ +import { ParsedDocument } from "@gxwf/server-common/src/ast/types"; import { TextDocument } from "vscode-languageserver-textdocument"; import { Diagnostic, DiagnosticSeverity, Position } from "vscode-languageserver-types"; import { Document, Node, visit, YAMLError, YAMLWarning } from "yaml"; @@ -60,7 +61,7 @@ export class YAMLSubDocument { * Represents a YAML document. * YAML documents can contain multiple sub-documents separated by "---". */ -export class YAMLDocument { +export class YAMLDocument implements ParsedDocument { private readonly _textBuffer: TextBuffer; private _diagnostics: Diagnostic[] | undefined; @@ -69,6 +70,10 @@ export class YAMLDocument { this._diagnostics = undefined; } + public get root(): ASTNode | undefined { + return this.mainDocument?.root; + } + /** The first or single sub-document parsed. */ public get mainDocument(): YAMLSubDocument | undefined { return this.subDocuments.at(0); From d415c2191035a86ab285a8b161ed528bf5452303 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 21 Jun 2022 23:46:47 +0200 Subject: [PATCH 08/11] Mark custom outline as supported in gxformat2 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d463a52..74f19f0 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ The following table shows all the implemented features and the current support f | [Documentation on Hover](#documentation-on-hover) | ✔️ | 🔜 | | [IntelliSense](#intellisense) | ✔️ | 🔜 | | [Formatting](#formatting) | ✔️ | ✔️ | -| [Custom Outline](#custom-outline) | ✔️ | 🔜 | +| [Custom Outline](#custom-outline) | ✔️ | ✔️ | | [Workflow Cleanup Command](#workflow-cleanup-command) | ✔️ | ❔ | | [Simplified Workflow Diffs](#simplified-workflow-diffs) | 🔶 | ❔ | From 281ced7677aefeaff1452978de5da2837fa25ed2 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 26 Jun 2022 13:09:07 +0200 Subject: [PATCH 09/11] Fix debug hover imports --- .../src/providers/hover/debugHoverContentContributor.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/packages/server-common/src/providers/hover/debugHoverContentContributor.ts b/server/packages/server-common/src/providers/hover/debugHoverContentContributor.ts index d2dc5e1..d8bde24 100644 --- a/server/packages/server-common/src/providers/hover/debugHoverContentContributor.ts +++ b/server/packages/server-common/src/providers/hover/debugHoverContentContributor.ts @@ -1,4 +1,5 @@ -import { ASTNode, PropertyASTNode, WorkflowDocument, Position, HoverContentContributor } from "../../languageTypes"; +import { WorkflowDocument, Position, HoverContentContributor } from "../../languageTypes"; +import { ASTNode, PropertyASTNode } from "../../ast/types"; import { ArrayASTNode, BooleanASTNode, NullASTNode, NumberASTNode, StringASTNode } from "vscode-json-languageservice"; /** @@ -6,7 +7,7 @@ import { ArrayASTNode, BooleanASTNode, NullASTNode, NumberASTNode, StringASTNode */ export class DebugHoverContentContributor implements HoverContentContributor { public onHoverContent(workflowDocument: WorkflowDocument, position: Position): string { - const node = workflowDocument.getNodeAtPosition(position); + const node = workflowDocument.nodeManager.getNodeAtPosition(position); return node ? this.printNode(node) : ""; } From 66e43ab0acfca688c48f56aa32e9e11c87234835 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 26 Jun 2022 13:10:34 +0200 Subject: [PATCH 10/11] Add basic tests for AST utilities with YAML --- .../tests/testHelpers.ts | 23 +++++++++++ .../tests/unit/astUtils-yaml.test.ts | 41 +++++++++++++++++++ .../tests/testHelpers.ts | 3 +- 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 server/gx-workflow-ls-format2/tests/testHelpers.ts create mode 100644 server/gx-workflow-ls-format2/tests/unit/astUtils-yaml.test.ts diff --git a/server/gx-workflow-ls-format2/tests/testHelpers.ts b/server/gx-workflow-ls-format2/tests/testHelpers.ts new file mode 100644 index 0000000..7b0f11d --- /dev/null +++ b/server/gx-workflow-ls-format2/tests/testHelpers.ts @@ -0,0 +1,23 @@ +import { ASTNode } from "@gxwf/server-common/src/ast/types"; +import { TextDocument } from "@gxwf/server-common/src/languageTypes"; +import { getLanguageService, YAMLDocument } from "@gxwf/yaml-language-service/src"; +import { GxFormat2WorkflowDocument } from "../src/gxFormat2WorkflowDocument"; + +export function toYamlDocument(contents: string): { textDoc: TextDocument; yamlDoc: YAMLDocument } { + const textDoc = TextDocument.create("foo://bar/file.gxwf.yaml", "gxformat2", 0, contents); + + const ls = getLanguageService(); + const yamlDoc = ls.parseYAMLDocument(textDoc) as YAMLDocument; + return { textDoc, yamlDoc }; +} + +export function getYamlDocumentRoot(contents: string): ASTNode { + const { yamlDoc } = toYamlDocument(contents); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return yamlDoc.root!; +} + +export function createFormat2WorkflowDocument(contents: string): GxFormat2WorkflowDocument { + const { textDoc, yamlDoc } = toYamlDocument(contents); + return new GxFormat2WorkflowDocument(textDoc, yamlDoc); +} diff --git a/server/gx-workflow-ls-format2/tests/unit/astUtils-yaml.test.ts b/server/gx-workflow-ls-format2/tests/unit/astUtils-yaml.test.ts new file mode 100644 index 0000000..4c51005 --- /dev/null +++ b/server/gx-workflow-ls-format2/tests/unit/astUtils-yaml.test.ts @@ -0,0 +1,41 @@ +import { getPropertyNodeFromPath } from "@gxwf/server-common/src/ast/utils"; +import { getYamlDocumentRoot } from "../testHelpers"; +import { expectPropertyNodeToHaveKey } from "@gxwf/server-common/tests/testHelpers"; + +describe("AST Utility Functions with YAML", () => { + describe("getPropertyNodeFromPath", () => { + describe("with valid path", () => { + it.each([ + ["key:\n key2: 0\n", "key"], + ["key:\n key2: 0\n", "key/key2"], + ["key:\n key2: 0\nkey3: val", "key3"], + ["key:\n key2: 0}\n key3: val", "key/key3"], + ["key:\n key2:\n key3: val", "key/key2"], + ["key:\n key2:\n key3: val", "key/key2/key3"], + ])("returns the expected property node at the given path", (contents: string, path: string) => { + const root = getYamlDocumentRoot(contents); + const pathItems = path.split("/"); + const expectedPropertyKey = pathItems[pathItems.length - 1] as string; + + const propertyNode = getPropertyNodeFromPath(root, path); + + expectPropertyNodeToHaveKey(propertyNode, expectedPropertyKey); + }); + }); + + describe("with invalid path", () => { + it.each([ + ["key:\n key2: 0\n", "key2"], + ["key:\n key2: 0\n", "key3"], + ["key:\n key2: 0\n", "key/5"], + ["key:\n key2:\n key3: val", "key/key3"], + ])("returns null", (contents: string, path: string) => { + const root = getYamlDocumentRoot(contents); + + const propertyNode = getPropertyNodeFromPath(root, path); + + expect(propertyNode).toBeNull(); + }); + }); + }); +}); diff --git a/server/gx-workflow-ls-native/tests/testHelpers.ts b/server/gx-workflow-ls-native/tests/testHelpers.ts index 92cf987..768be97 100644 --- a/server/gx-workflow-ls-native/tests/testHelpers.ts +++ b/server/gx-workflow-ls-native/tests/testHelpers.ts @@ -1,4 +1,5 @@ -import { ASTNode, getLanguageService, JSONDocument } from "vscode-json-languageservice"; +import { ASTNode } from "@gxwf/server-common/src/ast/types"; +import { getLanguageService, JSONDocument } from "vscode-json-languageservice"; import { TextDocument } from "@gxwf/server-common/src/languageTypes"; import { NativeWorkflowDocument } from "../src/nativeWorkflowDocument"; From 8cfab298541bc8d87933c92f1ee9a4853b19e607 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Sun, 26 Jun 2022 13:18:35 +0200 Subject: [PATCH 11/11] Update test-data --- test-data/json/validation/test_wf_01.ga | 1 + test-data/yaml/validation/test_wf_00.gxwf.yml | 5 +++++ test-data/yaml/{ => validation}/test_wf_01.gxwf.yml | 0 3 files changed, 6 insertions(+) create mode 100644 test-data/yaml/validation/test_wf_00.gxwf.yml rename test-data/yaml/{ => validation}/test_wf_01.gxwf.yml (100%) diff --git a/test-data/json/validation/test_wf_01.ga b/test-data/json/validation/test_wf_01.ga index a0a51e1..7ad56ff 100644 --- a/test-data/json/validation/test_wf_01.ga +++ b/test-data/json/validation/test_wf_01.ga @@ -5,6 +5,7 @@ "steps": { "0": { "id": 0, + "label": "Test Step", "name": "Test Step", "type": "data_input", "annotation": "Step description", diff --git a/test-data/yaml/validation/test_wf_00.gxwf.yml b/test-data/yaml/validation/test_wf_00.gxwf.yml new file mode 100644 index 0000000..b5e534f --- /dev/null +++ b/test-data/yaml/validation/test_wf_00.gxwf.yml @@ -0,0 +1,5 @@ +class: GalaxyWorkflow +label: Test Workflow Without Steps +inputs: {} +outputs: {} +steps: {} diff --git a/test-data/yaml/test_wf_01.gxwf.yml b/test-data/yaml/validation/test_wf_01.gxwf.yml similarity index 100% rename from test-data/yaml/test_wf_01.gxwf.yml rename to test-data/yaml/validation/test_wf_01.gxwf.yml