Support when tokens have null ranges

src/astUtils/creators.spec.ts

@@ -1,6 +1,4 @@
 import { expect } from '../chai-config.spec';
-import { interpolatedRange } from '..';
-import { TranspileState } from '../parser/TranspileState';
 import { createStringLiteral } from './creators';
 
 describe('creators', () => {
@@ -20,13 +18,4 @@ describe('creators', () => {
             expect(createStringLiteral('hello world"').token.text).to.equal('hello world"');
         });
     });
-
-    describe('interpolatedRange', () => {
-        it('can be used in sourcemaps', () => {
-            const state = new TranspileState('source/main.brs', {});
-            const node = state.sourceNode({ range: interpolatedRange }, 'code');
-            //should not crash
-            node.toStringWithSourceMap();
-        });
-    });
 });

src/astUtils/creators.ts

@@ -9,6 +9,7 @@ import { Block, MethodStatement } from '../parser/Statement';
 /**
  * A range that points to the beginning of the file. Used to give non-null ranges to programmatically-added source code.
  * (Hardcoded range to prevent circular dependency issue in `../util.ts`)
+ * @deprecated don't use this, it screws up sourcemaps. Just set range to null
  */
 export const interpolatedRange = {
     start: {
@@ -83,7 +84,7 @@ const tokenDefaults = {
     [TokenKind.Whitespace]: ' '
 };
 
-export function createToken<T extends TokenKind>(kind: T, text?: string, range = interpolatedRange): Token & { kind: T } {
+export function createToken<T extends TokenKind>(kind: T, text?: string, range?: Range): Token & { kind: T } {
     return {
         kind: kind,
         text: text ?? tokenDefaults[kind as string] ?? kind.toString().toLowerCase(),
@@ -140,7 +141,7 @@ export function createBooleanLiteral(value: 'true' | 'false', range?: Range) {
 export function createFunctionExpression(kind: TokenKind.Sub | TokenKind.Function) {
     return new FunctionExpression(
         [],
-        new Block([], interpolatedRange),
+        new Block([]),
         createToken(kind),
         kind === TokenKind.Sub ? createToken(TokenKind.EndSub) : createToken(TokenKind.EndFunction),
         createToken(TokenKind.LeftParen),

src/astUtils/reflection.spec.ts

@@ -5,7 +5,7 @@ import { FunctionExpression, NamespacedVariableNameExpression, BinaryExpression,
 import type { Token } from '../lexer/Token';
 import { TokenKind } from '../lexer/TokenKind';
 import { isPrintStatement, isIfStatement, isBody, isAssignmentStatement, isBlock, isExpressionStatement, isCommentStatement, isExitForStatement, isExitWhileStatement, isFunctionStatement, isIncrementStatement, isGotoStatement, isLabelStatement, isReturnStatement, isEndStatement, isStopStatement, isForStatement, isForEachStatement, isWhileStatement, isDottedSetStatement, isIndexedSetStatement, isLibraryStatement, isNamespaceStatement, isImportStatement, isExpression, isBinaryExpression, isCallExpression, isFunctionExpression, isNamespacedVariableNameExpression, isDottedGetExpression, isXmlAttributeGetExpression, isIndexedGetExpression, isGroupingExpression, isLiteralExpression, isEscapedCharCodeLiteralExpression, isArrayLiteralExpression, isAALiteralExpression, isUnaryExpression, isVariableExpression, isSourceLiteralExpression, isNewExpression, isCallfuncExpression, isTemplateStringQuasiExpression, isTemplateStringExpression, isTaggedTemplateStringExpression, isBrsFile, isXmlFile, isClassStatement, isStatement, isAnnotationExpression, isTryCatchStatement, isCatchStatement, isThrowStatement } from './reflection';
-import { createToken, createStringLiteral, interpolatedRange as range } from './creators';
+import { createToken, createStringLiteral } from './creators';
 import { Program } from '../Program';
 import { BrsFile } from '../files/BrsFile';
 import { XmlFile } from '../files/XmlFile';
@@ -25,12 +25,12 @@ describe('reflection', () => {
     });
 
     describe('Statements', () => {
-        const ident = createToken(TokenKind.Identifier, 'a', range);
-        const expr = createStringLiteral('', range);
-        const token = createToken(TokenKind.StringLiteral, '', range);
+        const ident = createToken(TokenKind.Identifier, 'a');
+        const expr = createStringLiteral('');
+        const token = createToken(TokenKind.StringLiteral, '');
         const body = new Body([]);
         const assignment = new AssignmentStatement(undefined, ident, expr);
-        const block = new Block([], range);
+        const block = new Block([]);
         const expression = new ExpressionStatement(expr);
         const comment = new CommentStatement([token]);
         const exitFor = new ExitForStatement({ exitFor: token });
@@ -50,7 +50,7 @@ describe('reflection', () => {
         const dottedSet = new DottedSetStatement(expr, ident, expr);
         const indexedSet = new IndexedSetStatement(expr, expr, expr, token, token);
         const library = new LibraryStatement({ library: token, filePath: token });
-        const namespace = new NamespaceStatement(token, new NamespacedVariableNameExpression(createVariableExpression('a', range)), body, token);
+        const namespace = new NamespaceStatement(token, new NamespacedVariableNameExpression(createVariableExpression('a')), body, token);
         const cls = new ClassStatement(token, ident, [], token);
         const imports = new ImportStatement(token, token);
         const catchStmt = new CatchStatement({ catch: token }, ident, block);
@@ -183,19 +183,19 @@ describe('reflection', () => {
     });
 
     describe('Expressions', () => {
-        const ident = createToken(TokenKind.Identifier, 'a', range);
-        const expr = createStringLiteral('', range);
-        const token = createToken(TokenKind.StringLiteral, '', range);
-        const block = new Block([], range);
+        const ident = createToken(TokenKind.Identifier, 'a');
+        const expr = createStringLiteral('');
+        const token = createToken(TokenKind.StringLiteral, '');
+        const block = new Block([]);
         const charCode: Token & { charCode: number } = {
             kind: TokenKind.EscapedCharCodeLiteral,
             text: '0',
-            range: range,
+            range: undefined,
             isReserved: false,
             charCode: 0,
             leadingWhitespace: ''
         };
-        const nsVar = new NamespacedVariableNameExpression(createVariableExpression('a', range));
+        const nsVar = new NamespacedVariableNameExpression(createVariableExpression('a'));
         const binary = new BinaryExpression(expr, token, expr);
         const call = new CallExpression(expr, token, token, []);
         const fun = new FunctionExpression([], block, token, token, token, token);

src/files/BrsFile.spec.ts

@@ -16,15 +16,16 @@ import { DiagnosticMessages } from '../DiagnosticMessages';
 import type { StandardizedFileEntry } from 'roku-deploy';
 import util, { standardizePath as s } from '../util';
 import PluginInterface from '../PluginInterface';
-import { expectCompletionsIncludes, expectDiagnostics, expectHasDiagnostics, expectZeroDiagnostics, getTestGetTypedef, getTestTranspile, trim } from '../testHelpers.spec';
-import { ParseMode } from '../parser/Parser';
+import { expectCompletionsIncludes, expectDiagnostics, expectHasDiagnostics, expectZeroDiagnostics, getTestGetTypedef, getTestTranspile, trim, trimMap } from '../testHelpers.spec';
+import { ParseMode, Parser } from '../parser/Parser';
 import { Logger } from '../Logger';
 import { ImportStatement } from '../parser/Statement';
 import { createToken } from '../astUtils/creators';
 import * as fsExtra from 'fs-extra';
 import { URI } from 'vscode-uri';
 import undent from 'undent';
 import { tempDir, rootDir } from '../testHelpers.spec';
+import * as fileUrl from 'file-url';
 
 let sinon = sinonImport.createSandbox();
 
@@ -2194,6 +2195,237 @@ describe('BrsFile', () => {
     });
 
     describe('transpile', () => {
+        describe('null tokens', () => {
+            it('succeeds when token locations are omitted', () => {
+                doTest(`
+                    library "something" 'comment before func
+                    sub main(arg0, arg1 as string, arg2 = invalid)
+                        'comment
+                        aa = {
+                            'comment
+                            one: 1
+                            "two": 2
+                        }
+                        arr = [
+                            'comment
+                            1
+                            'comment
+                            2
+                        ]
+                        val = +3
+                        print "hello"
+                        'comment after print
+                        num = 1
+                        num++
+                        num += 2
+                        num = +num
+                        test(num)
+                        for i = 0 to 10 step 1
+                            exit for
+                        end for
+                        while true
+                            exit while
+                        end while
+                        if true then
+                            print 1
+                        else if true
+                            print 1
+                        else
+                            print 1
+                        end if
+                        dim thing[1, 2]
+                        label1:
+                        goto label1
+                        end
+                        stop
+                        stuff = [
+                            1
+                            2
+                            3
+                        ]
+                        for each item in stuff
+                            print item
+                        end for
+                        m.thing = 1
+                        m.thing += 1
+                        m[1] = 1
+                        m[1] += 1
+                        m[1, 2] = 2
+                        try
+                            print m.b.c
+                        catch e
+                            print e
+                        end try
+                        throw "crash"
+                        for i = 0 to 10
+                            continue
+                        end for
+                        print m@name
+                        print (1 + 2)
+                    end sub
+
+                    sub test(p1)
+                        return p1
+                    end sub
+                `);
+            });
+
+            it('works for bs content', () => {
+                program.setFile('source/lib.bs', ``);
+                doTest(`
+                    import "pkg:/source/lib.bs"
+                    @annotation()
+                    sub test()
+                        two = 2
+                        print \`1\${two}\${3}\n\`
+                        print (1 as integer)
+                        print SOURCE_LINE_NUM
+                        print FUNCTION_NAME
+                        print SOURCE_FUNCTION_NAME
+                        print PKG_LOCATION
+                        print PKG_PATH
+                        print LINE_NUM
+                        print new Person()
+                        [email protected]()
+                        [email protected](1, 2)
+                        print tag\`stuff\${LINE_NUM}\${LINE_NUM}\`
+                        print 1 = 1 ? 1 : 2
+                        print 1 = 1 ? m.one : m.two
+                        print 1 ?? 2
+                        print m.one ?? m.two
+                        print /123/gi
+                    end sub
+                    function tag(param1, param2)
+                    end function
+                    const a = 1
+                    namespace alpha
+                        function beta()
+                            throw "An error has occurred"
+                        end function
+                        function charlie()
+                        end function
+                    end namespace
+                    sub test()
+                        ' alpha.charlie()
+                    end sub
+
+                    enum Direction
+                        up = "up"
+                    end enum
+
+                    class Person
+                        name as string
+                        sub new()
+                            print m.name
+                        end sub
+
+                        sub test()
+                        end sub
+                    end class
+
+                    interface Beta
+                        name as string
+                    end interface
+                `, `
+                    'import "pkg:/source/lib.bs"
+
+                    sub test()
+                        two = 2
+                        print ("1" + bslib_toString(two) + bslib_toString(3) + chr(10))
+                        print 1
+                        print -1
+                        print "test"
+                        print "test"
+                        print "pkg:/source/main.brs:" + str(LINE_NUM)
+                        print "pkg:/source/main.brs"
+                        print LINE_NUM
+                        print Person()
+                        m.callfunc("someCallfunc", invalid)
+                        m.callfunc("someCallfunc", 1, 2)
+                        print tag(["stuff", "", ""], [LINE_NUM, LINE_NUM])
+                        print bslib_ternary(1 = 1, 1, 2)
+                        print (function(__bsCondition, m)
+                                if __bsCondition then
+                                    return m.one
+                                else
+                                    return m.two
+                                end if
+                            end function)(1 = 1, m)
+                        print bslib_coalesce(1, 2)
+                        print (function(m)
+                                __bsConsequent = m.one
+                                if __bsConsequent <> invalid then
+                                    return __bsConsequent
+                                else
+                                    return m.two
+                                end if
+                            end function)(m)
+                        print CreateObject("roRegex", "123", "gi")
+                    end sub
+
+                    function tag(param1, param2)
+                    end function
+
+                    function alpha_beta()
+                        throw "An error has occurred"
+                    end function
+
+                    function alpha_charlie()
+                    end function
+
+                    sub test()
+                        ' alpha.charlie()
+                    end sub
+
+                    function __Person_builder()
+                        instance = {}
+                        instance.new = sub()
+                            m.name = invalid
+                            print m.name
+                        end sub
+                        instance.test = sub()
+                        end sub
+                        return instance
+                    end function
+                    function Person()
+                        instance = __Person_builder()
+                        instance.new()
+                        return instance
+                    end function
+                `);
+            });
+
+            it('handles source literals properly', () => {
+                const pathUrl = fileUrl(rootDir);
+                let text = `"${pathUrl.substring(0, 4)}" + "${pathUrl.substring(4)}`;
+                doTest(`
+                    sub test()
+                        print SOURCE_FILE_PATH
+                        print SOURCE_LOCATION
+                    end sub
+                `, `
+                    sub test()
+                        print ${text}/source/main.bs"
+                        print ${text}/source/main.bs:-1"
+                    end sub
+                `);
+            });
+            function doTest(source: string, expected = source) {
+                const file = program.setFile<BrsFile>('source/main.bs', '');
+                //override the parser with our locationless parser
+                file['_parser'] = Parser.parse(source, { mode: ParseMode.BrighterScript, trackLocations: false });
+                program.getScopesForFile(file).forEach(x => x['cache'].clear());
+                program.validate();
+                expectZeroDiagnostics(program);
+                const result = file.transpile();
+                expect(
+                    trimMap(undent(result.code))
+                ).to.eql(
+                    undent(expected)
+                );
+            }
+        });
+
         it('transpilies libpkg:/ paths when encountered', () => {
             program.setFile('source/lib.bs', `
                 import "libpkg:/source/numbers.bs"