Xml ast refactor (#818)

* Refactor XML AST

* Add findLastIndex util function.

* Refactor names of certain nodes

* fix some names

* Rename other elements

* Remove benchmarking log

benchmarks/index.ts

@@ -215,7 +215,6 @@ const runner = new Runner(options as any);
 runner.run();
 
 function execSync(command: string, options: ExecSyncOptions = {}) {
-    console.log(command);
     return childProcess.execSync(command, { stdio: 'inherit', cwd: cwd, ...options });
 }
 

scripts/scrape-roku-docs.ts

@@ -298,7 +298,7 @@ class Runner {
                                 const arg = call.args[i];
                                 let paramName = `param${i}`;
                                 if (isVariableExpression(arg)) {
-                                    paramName = arg.getName(ParseMode.BrightScript)
+                                    paramName = arg.getName(ParseMode.BrightScript);
                                 }
                                 signature.params.push({
                                     name: paramName,
@@ -311,7 +311,7 @@ class Runner {
                             component.constructors.push(signature);
                         }
                     } else if (match[1]) {
-                        const signature = this.getConstructorSignature(name, match[1])
+                        const signature = this.getConstructorSignature(name, match[1]);
 
                         if (signature) {
                             component.constructors.push(signature);
@@ -332,7 +332,7 @@ class Runner {
     }
 
     private getConstructorSignature(componentName: string, sourceCode: string) {
-        const foundParamTexts = this.findParamTexts(sourceCode)
+        const foundParamTexts = this.findParamTexts(sourceCode);
 
         if (foundParamTexts && foundParamTexts[0].toLowerCase() === componentName.toLowerCase()) {
             const signature = {
@@ -797,6 +797,7 @@ function deepSearch<T = any>(object, key, predicate): T {
         return object;
     }
 
+    // eslint-disable-next-line @typescript-eslint/prefer-for-of
     for (let i = 0; i < Object.keys(object).length; i++) {
         let value = object[Object.keys(object)[i]];
         if (typeof value === 'object' && value) {
@@ -1078,16 +1079,16 @@ class TokenManager {
     }
 
     /**
-    * Find any `is deprecated` text between the specified items
-    */
+     * Find any `is deprecated` text between the specified items
+     */
     public getDeprecatedDescription(startToken: Token, endToken: Token) {
         const deprecatedDescription = this.find<marked.Tokens.Text>(x => !!/is\s*deprecated/i.exec(x?.text), startToken, endToken)?.text;
         return deprecatedDescription;
     }
 
     /**
-    * Sets `deprecatedDescription` and `isDeprecated` on passed in entity if `deprecated` is mentioned between the two tokens
-    */
+     * Sets `deprecatedDescription` and `isDeprecated` on passed in entity if `deprecated` is mentioned between the two tokens
+     */
     public setDeprecatedData(entity: PossiblyDeprecated, startToken: Token, endToken: Token) {
         entity.deprecatedDescription = this.getDeprecatedDescription(startToken, endToken);
         if (entity.deprecatedDescription) {

src/Program.ts

@@ -886,7 +886,7 @@ export class Program {
         let funcNames = new Set<string>();
         let currentScope = scope;
         while (isXmlScope(currentScope)) {
-            for (let name of currentScope.xmlFile.ast.component.api?.functions.map((f) => f.name) ?? []) {
+            for (let name of currentScope.xmlFile.ast.componentElement.interfaceElement?.functions.map((f) => f.name) ?? []) {
                 if (!filterName || name === filterName) {
                     funcNames.add(name);
                 }

src/XmlScope.ts

@@ -7,7 +7,7 @@ import type { Program } from './Program';
 import util from './util';
 import { isXmlFile } from './astUtils/reflection';
 import { SGFieldTypes } from './parser/SGTypes';
-import type { SGTag } from './parser/SGTypes';
+import type { SGElement } from './parser/SGTypes';
 
 export class XmlScope extends Scope {
     constructor(
@@ -52,25 +52,26 @@ export class XmlScope extends Scope {
     }
 
     private diagnosticValidateInterface(callableContainerMap: CallableContainerMap) {
-        if (!this.xmlFile.parser.ast?.component?.api) {
+        if (!this.xmlFile.parser.ast?.componentElement?.interfaceElement) {
             return;
         }
-        const { api } = this.xmlFile.parser.ast.component;
+        const iface = this.xmlFile.parser.ast.componentElement.interfaceElement;
+
         //validate functions
-        for (const fun of api.functions) {
-            const name = fun.name;
+        for (const func of iface.functions) {
+            const name = func.name;
             if (!name) {
-                this.diagnosticMissingAttribute(fun, 'name');
+                this.diagnosticMissingAttribute(func, 'name');
             } else if (!callableContainerMap.has(name.toLowerCase())) {
                 this.diagnostics.push({
                     ...DiagnosticMessages.xmlFunctionNotFound(name),
-                    range: fun.getAttribute('name').value.range,
+                    range: func.getAttribute('name').tokens.value.range,
                     file: this.xmlFile
                 });
             }
         }
         //validate fields
-        for (const field of api.fields) {
+        for (const field of iface.fields) {
             const { id, type } = field;
             if (!id) {
                 this.diagnosticMissingAttribute(field, 'id');
@@ -82,18 +83,17 @@ export class XmlScope extends Scope {
             } else if (!SGFieldTypes.includes(type.toLowerCase())) {
                 this.diagnostics.push({
                     ...DiagnosticMessages.xmlInvalidFieldType(type),
-                    range: field.getAttribute('type').value.range,
+                    range: field.getAttribute('type').tokens.value.range,
                     file: this.xmlFile
                 });
             }
         }
     }
 
-    private diagnosticMissingAttribute(tag: SGTag, name: string) {
-        const { text, range } = tag.tag;
+    private diagnosticMissingAttribute(tag: SGElement, name: string) {
         this.diagnostics.push({
-            ...DiagnosticMessages.xmlTagMissingAttribute(text, name),
-            range: range,
+            ...DiagnosticMessages.xmlTagMissingAttribute(tag.tokens.startTagName.text, name),
+            range: tag.tokens.startTagName.range,
             file: this.xmlFile
         });
     }

src/astUtils/creators.ts

@@ -1,9 +1,9 @@
 import type { Range } from 'vscode-languageserver';
 import type { Identifier, Token } from '../lexer/Token';
+import { SGAttribute, SGComponent, SGInterface, SGInterfaceField, SGInterfaceFunction, SGScript } from '../parser/SGTypes';
 import { TokenKind } from '../lexer/TokenKind';
 import type { Expression } from '../parser/AstNode';
-import { LiteralExpression, CallExpression, DottedGetExpression, VariableExpression, FunctionExpression } from '../parser/Expression';
-import type { SGAttribute } from '../parser/SGTypes';
+import { CallExpression, DottedGetExpression, FunctionExpression, LiteralExpression, VariableExpression } from '../parser/Expression';
 import { Block, MethodStatement } from '../parser/Statement';
 
 /**
@@ -177,12 +177,86 @@ export function createCall(callee: Expression, args?: Expression[]) {
  * Create an SGAttribute without any ranges
  */
 export function createSGAttribute(keyName: string, value: string) {
-    return {
-        key: {
-            text: keyName
-        },
-        value: {
-            text: value
-        }
-    } as SGAttribute;
+    return new SGAttribute(
+        { text: keyName },
+        { text: '=' },
+        { text: '"' },
+        { text: value },
+        { text: '"' }
+    );
+}
+
+export function createSGInterfaceField(id: string, attributes: { type?: string; alias?: string; value?: string; onChange?: string; alwaysNotify?: string } = {}) {
+    const attrs = [
+        createSGAttribute('id', id)
+    ];
+    for (let key in attributes) {
+        attrs.push(
+            createSGAttribute(key, attributes[key])
+        );
+    }
+    return new SGInterfaceField(
+        { text: '<' },
+        { text: 'field' },
+        attrs,
+        { text: '/>' }
+    );
+}
+
+export function createSGComponent(name: string, parentName?: string) {
+    const attributes = [
+        createSGAttribute('name', name)
+    ];
+    if (parentName) {
+        attributes.push(
+            createSGAttribute('extends', parentName)
+        );
+    }
+    return new SGComponent(
+        { text: '<' },
+        { text: 'component' },
+        attributes,
+        { text: '>' },
+        [],
+        { text: '</' },
+        { text: 'component' },
+        { text: '>' }
+    );
+}
+
+export function createSGInterfaceFunction(functionName: string) {
+    return new SGInterfaceFunction(
+        { text: '<' },
+        { text: 'function' },
+        [createSGAttribute('name', functionName)],
+        { text: '/>' }
+    );
+}
+
+export function createSGInterface() {
+    return new SGInterface(
+        { text: '<' },
+        { text: 'interface' },
+        [],
+        { text: '>' },
+        [],
+        { text: '</' },
+        { text: 'interface' },
+        { text: '>' }
+    );
+}
+
+export function createSGScript(attributes: { type?: string; uri?: string }) {
+    const attrs = [] as SGAttribute[];
+    for (let key in attributes) {
+        attrs.push(
+            createSGAttribute(key, attributes[key])
+        );
+    }
+    return new SGScript(
+        { text: '<' },
+        { text: 'script' },
+        attrs,
+        { text: '/>' }
+    );
 }

src/astUtils/xml.ts

@@ -1,26 +1,26 @@
-import type { SGChildren, SGComponent, SGField, SGFunction, SGInterface, SGNode, SGScript, SGTag } from '../parser/SGTypes';
+import type { SGChildren, SGComponent, SGInterfaceField, SGInterfaceFunction, SGInterface, SGNode, SGScript, SGElement } from '../parser/SGTypes';
 
-export function isSGComponent(tag: SGTag): tag is SGComponent {
+export function isSGComponent(tag: SGElement): tag is SGComponent {
     return tag?.constructor.name === 'SGComponent';
 }
-export function isSGInterface(tag: SGTag): tag is SGInterface {
+export function isSGInterface(tag: SGElement): tag is SGInterface {
     return tag?.constructor.name === 'SGInterface';
 }
-export function isSGScript(tag: SGTag): tag is SGScript {
+export function isSGScript(tag: SGElement): tag is SGScript {
     return tag?.constructor.name === 'SGScript';
 }
-export function isSGChildren(tag: SGTag): tag is SGChildren {
+export function isSGChildren(tag: SGElement): tag is SGChildren {
     return tag?.constructor.name === 'SGChildren';
 }
-export function isSGField(tag: SGTag): tag is SGField {
+export function isSGInterfaceField(tag: SGElement): tag is SGInterfaceField {
     return tag?.constructor.name === 'SGField';
 }
-export function isSGFunction(tag: SGTag): tag is SGFunction {
+export function isSGInterfaceFunction(tag: SGElement): tag is SGInterfaceFunction {
     return tag?.constructor.name === 'SGFunction';
 }
-export function isSGNode(tag: SGTag): tag is SGNode {
+export function isSGNode(tag: SGElement): tag is SGNode {
     return tag?.constructor.name === 'SGNode';
 }
-export function isSGCustomization(tag: SGTag): tag is SGNode {
-    return isSGNode(tag) && tag.tag?.text?.toLowerCase() === 'customization';
+export function isSGCustomization(tag: SGElement): tag is SGNode {
+    return isSGNode(tag) && tag.tokens.startTagName?.text?.toLowerCase() === 'customization';
 }

src/bscPlugin/BscPlugin.ts

@@ -6,6 +6,7 @@ import { CompletionsProcessor } from './completions/CompletionsProcessor';
 import { HoverProcessor } from './hover/HoverProcessor';
 import { BrsFileSemanticTokensProcessor } from './semanticTokens/BrsFileSemanticTokensProcessor';
 import { BrsFilePreTranspileProcessor } from './transpile/BrsFilePreTranspileProcessor';
+import { XmlFilePreTranspileProcessor } from './transpile/XmlFilePreTranspileProcessor';
 import { BrsFileValidator } from './validation/BrsFileValidator';
 import { ProgramValidator } from './validation/ProgramValidator';
 import { ScopeValidator } from './validation/ScopeValidator';
@@ -55,6 +56,8 @@ export class BscPlugin implements CompilerPlugin {
     public beforeFileTranspile(event: BeforeFileTranspileEvent) {
         if (isBrsFile(event.file)) {
             return new BrsFilePreTranspileProcessor(event as any).process();
+        } else if (isXmlFile(event.file)) {
+            return new XmlFilePreTranspileProcessor(event as any).process();
         }
     }
 }

src/bscPlugin/codeActions/CodeActionsProcessor.ts

@@ -98,9 +98,9 @@ export class CodeActionsProcessor {
 
     private addMissingExtends(diagnostic: DiagnosticMessageType<'xmlComponentMissingExtendsAttribute'>) {
         const srcPath = this.event.file.srcPath;
-        const { component } = (this.event.file as XmlFile).parser.ast;
+        const { componentElement } = (this.event.file as XmlFile).parser.ast;
         //inject new attribute after the final attribute, or after the `<component` if there are no attributes
-        const pos = (component.attributes[component.attributes.length - 1] ?? component.tag).range.end;
+        const pos = (componentElement.attributes[componentElement.attributes.length - 1] ?? componentElement.tokens.startTagName).range.end;
         this.event.codeActions.push(
             codeActionUtil.createCodeAction({
                 title: `Extend "Group"`,

src/bscPlugin/transpile/XmlFilePreTranspileProcessor.ts

@@ -0,0 +1,66 @@
+import { createSGScript } from '../../astUtils/creators';
+import { isXmlFile } from '../../astUtils/reflection';
+import type { XmlFile } from '../../files/XmlFile';
+import type { BeforeFileTranspileEvent } from '../../interfaces';
+import util from '../../util';
+
+export class XmlFilePreTranspileProcessor {
+    public constructor(
+        private event: BeforeFileTranspileEvent<XmlFile>
+    ) {
+    }
+
+    public process() {
+        if (!isXmlFile(this.event.file)) {
+            return;
+        }
+
+        //insert any missing script imports
+        this.injectScriptImports();
+
+        //transform any brighterscript `type` attributes to `brightscript`
+        for (const script of this.event.file.ast?.componentElement?.scriptElements ?? []) {
+            const type = script.getAttribute('type');
+            if (/text\/brighterscript/i.test(type?.value)) {
+                this.event.editor.setProperty(
+                    type,
+                    'value',
+                    type.value.replace(/text\/brighterscript/i, 'text/brightscript')
+                );
+            }
+
+            //replace `.bs` extensions with `.brs`
+            const uri = script.getAttribute('uri');
+            if (/\.bs/i.test(uri?.value)) {
+                this.event.editor.setProperty(
+                    uri,
+                    'value',
+                    uri.value.replace(/\.bs/i, '.brs')
+                );
+            }
+        }
+    }
+
+    /**
+     * Inject any missing scripts into the xml file
+     */
+    private injectScriptImports() {
+        // eslint-disable-next-line @typescript-eslint/dot-notation
+        const extraImportScripts = this.event.file['getMissingImportsForTranspile']().map(uri => {
+            return createSGScript({
+                type: 'text/brightscript',
+                uri: util.sanitizePkgPath(uri.replace(/\.bs$/, '.brs'))
+            });
+        });
+
+        if (extraImportScripts) {
+            //add new scripts after the LAST `<script>` tag that was created explicitly by the user, or at the top of the component if it has no scripts
+            let lastScriptIndex = util.findLastIndex(this.event.file.ast.componentElement.elements, x => x.tokens.startTagName.text.toLowerCase() === 'script');
+            lastScriptIndex = lastScriptIndex >= 0
+                ? lastScriptIndex + 1
+                : 0;
+
+            this.event.editor.arraySplice(this.event.file.ast.componentElement.elements, lastScriptIndex, 0, ...extraImportScripts);
+        }
+    }
+}