From f7fbdbb4c68e1ed1b78987d4cbcb64bc672b18b6 Mon Sep 17 00:00:00 2001
From: Bronley <bronley@gmail.com>
Date: Sat, 13 Mar 2021 13:00:07 -0500
Subject: [PATCH 1/4] Move logic to CommentFlagProcessor

---
 package-lock.json                |   8 ++
 package.json                     |   1 +
 src/CommentFlagProcessor.spec.ts | 128 +++++++++++++++++++
 src/CommentFlagProcessor.ts      | 209 +++++++++++++++++++++++++++++++
 src/files/BrsFile.ts             |  81 ++----------
 src/interfaces.ts                |   6 +-
 src/util.spec.ts                 | 110 ----------------
 src/util.ts                      |  97 --------------
 8 files changed, 361 insertions(+), 279 deletions(-)
 create mode 100644 src/CommentFlagProcessor.spec.ts
 create mode 100644 src/CommentFlagProcessor.ts

diff --git a/package-lock.json b/package-lock.json
index 66dde2bba..eed637b01 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1148,6 +1148,14 @@
             "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=",
             "dev": true
         },
+        "chevrotain": {
+            "version": "7.0.1",
+            "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-7.0.1.tgz",
+            "integrity": "sha512-B/44jrdw5GAzy483LEeVSgXSX0qOYM8lUd3l5+yf6Vl6OQjEUCm2BUiYbHRCIK6xCEvCLAFe1kj8uyV6+zdaVw==",
+            "requires": {
+                "regexp-to-ast": "0.5.0"
+            }
+        },
         "chokidar": {
             "version": "3.4.3",
             "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.3.tgz",
diff --git a/package.json b/package.json
index 3d77273ee..2d3c0c30c 100644
--- a/package.json
+++ b/package.json
@@ -112,6 +112,7 @@
         "@xml-tools/parser": "^1.0.7",
         "array-flat-polyfill": "^1.0.1",
         "chalk": "^2.4.2",
+        "chevrotain": "^7.0.1",
         "chokidar": "^3.0.2",
         "clear": "^0.1.0",
         "cross-platform-clear-console": "^2.3.0",
diff --git a/src/CommentFlagProcessor.spec.ts b/src/CommentFlagProcessor.spec.ts
new file mode 100644
index 000000000..187a41cd4
--- /dev/null
+++ b/src/CommentFlagProcessor.spec.ts
@@ -0,0 +1,128 @@
+import { expect } from 'chai';
+import { Range } from 'vscode-languageserver';
+import { CommentFlagProcessor } from './CommentFlagProcessor';
+import { Lexer } from './lexer/Lexer';
+
+describe('CommentFlagProcessor', () => {
+    let processor: CommentFlagProcessor;
+
+    describe('tokenizeByWhitespace', () => {
+        beforeEach(() => {
+            processor = new CommentFlagProcessor(null);
+        });
+
+        it('works with single chars', () => {
+            expect(processor['tokenizeByWhitespace']('a b c')).to.deep.equal([{
+                startIndex: 0,
+                text: 'a'
+            }, {
+                startIndex: 2,
+                text: 'b'
+            },
+            {
+                startIndex: 4,
+                text: 'c'
+            }]);
+        });
+
+        it('works with tabs', () => {
+            expect(processor['tokenizeByWhitespace']('a\tb\t c')).to.deep.equal([{
+                startIndex: 0,
+                text: 'a'
+            }, {
+                startIndex: 2,
+                text: 'b'
+            },
+            {
+                startIndex: 5,
+                text: 'c'
+            }]);
+
+            it('works with leading whitespace', () => {
+                expect(processor['tokenizeByWhitespace']('  \ta\tb\t c')).to.deep.equal([{
+                    startIndex: 4,
+                    text: 'a'
+                }, {
+                    startIndex: 6,
+                    text: 'b'
+                },
+                {
+                    startIndex: 9,
+                    text: 'c'
+                }]);
+            });
+
+            it('works with multiple characters in a word', () => {
+                expect(processor['tokenizeByWhitespace']('abc 123')).to.deep.equal([{
+                    startIndex: 0,
+                    text: 'abc'
+                }, {
+                    startIndex: 4,
+                    text: '123'
+                }]);
+            });
+        });
+    });
+
+    describe('tokenize', () => {
+        beforeEach(() => {
+            processor = new CommentFlagProcessor(null, [`'`]);
+        });
+
+        it('skips non disable comments', () => {
+            expect(
+                processor['tokenize'](`'not disable comment`, null)
+            ).not.to.exist;
+        });
+
+        it('tokenizes bs:disable-line comment', () => {
+            expect(
+                processor['tokenize'](`'bs:disable-line`, null)
+            ).to.eql({
+                commentTokenText: `'`,
+                disableType: 'line',
+                codes: []
+            });
+        });
+
+        it('works for special case', () => {
+            const token = Lexer.scan(`print "hi" 'bs:disable-line: 123456 999999   aaaab`).tokens[2];
+            expect(
+                processor['tokenize'](token.text, token.range)
+            ).to.eql({
+                commentTokenText: `'`,
+                disableType: 'line',
+                codes: [{
+                    code: '123456',
+                    range: Range.create(0, 29, 0, 35)
+                }, {
+                    code: '999999',
+                    range: Range.create(0, 36, 0, 42)
+                }, {
+                    code: 'aaaab',
+                    range: Range.create(0, 45, 0, 50)
+                }]
+            });
+        });
+
+        it('tokenizes bs:disable-line comment with codes', () => {
+            const token = Lexer.scan(`'bs:disable-line:1 2 3`).tokens[0];
+            expect(
+                processor['tokenize'](token.text, token.range)
+            ).to.eql({
+                commentTokenText: `'`,
+                disableType: 'line',
+                codes: [{
+                    code: '1',
+                    range: Range.create(0, 17, 0, 18)
+                }, {
+                    code: '2',
+                    range: Range.create(0, 19, 0, 20)
+                }, {
+                    code: '3',
+                    range: Range.create(0, 21, 0, 22)
+                }]
+            });
+        });
+    });
+});
diff --git a/src/CommentFlagProcessor.ts b/src/CommentFlagProcessor.ts
new file mode 100644
index 000000000..46bd2191b
--- /dev/null
+++ b/src/CommentFlagProcessor.ts
@@ -0,0 +1,209 @@
+import type { Range } from 'vscode-languageserver';
+import { DiagnosticMessages } from './DiagnosticMessages';
+import type { BscFile, BsDiagnostic, CommentFlag, DiagnosticCode } from './interfaces';
+import { util } from './util';
+
+export class CommentFlagProcessor {
+    public constructor(
+        /**
+         * The file this processor applies to
+         */
+        public file: BscFile,
+        /**
+         * An array of strings containing the types of text that a comment starts with. (i.e. `REM`, `'`, `<!--`)
+         */
+        public commentStarters = [] as string[],
+        /**
+         * An array of strings containing the types of text that a comment ends with. (i.e. `-->`)
+         */
+        public commentFinishers = [] as string[],
+        /**
+         * Valid diagnostic codes. Codes NOT in this list will be flagged
+         */
+        public diagnosticCodes = [] as DiagnosticCode[],
+        /**
+         * Diagnostic codes to never filter (these codes will always be flagged)
+         */
+        public ignoreDiagnosticCodes = [] as DiagnosticCode[]
+    ) {
+
+        this.allCodesExceptIgnores = this.diagnosticCodes.filter(x => !this.ignoreDiagnosticCodes.includes(x));
+
+    }
+
+    /**
+     * List of comment flags generated during processing
+     */
+    public commentFlags = [] as CommentFlag[];
+
+    /**
+     * List of diagnostics generated during processing
+     */
+    public diagnostics = [] as BsDiagnostic[];
+
+    /**
+     * A list of all codes EXCEPT the ones in `ignoreDiagnosticCodes`
+     */
+    public allCodesExceptIgnores: DiagnosticCode[];
+
+    public tryAdd(text: string, range: Range) {
+        const tokenized = this.tokenize(text, range);
+        if (!tokenized) {
+            return;
+        }
+
+        let affectedRange: Range;
+        if (tokenized.disableType === 'line') {
+            affectedRange = util.createRange(range.start.line, 0, range.start.line, range.start.character);
+        } else if (tokenized.disableType === 'next-line') {
+            affectedRange = util.createRange(range.start.line + 1, 0, range.start.line + 1, Number.MAX_SAFE_INTEGER);
+        }
+
+        let commentFlag: CommentFlag;
+
+        //statement to disable EVERYTHING
+        if (tokenized.codes.length === 0) {
+            commentFlag = {
+                file: this.file,
+                //null means all codes
+                codes: null,
+                range: range,
+                affectedRange: affectedRange
+            };
+
+            //disable specific diagnostic codes
+        } else {
+            let codes = [] as number[];
+            for (let codeToken of tokenized.codes) {
+                let codeInt = parseInt(codeToken.code);
+                if (isNaN(codeInt)) {
+                    //don't validate non-numeric codes
+                    continue;
+                    //add a warning for unknown codes
+                } else if (this.diagnosticCodes.includes(codeInt)) {
+                    codes.push(codeInt);
+                } else {
+                    this.diagnostics.push({
+                        ...DiagnosticMessages.unknownDiagnosticCode(codeInt),
+                        file: this.file,
+                        range: codeToken.range
+                    });
+                }
+            }
+            if (codes.length > 0) {
+                commentFlag = {
+                    file: this.file,
+                    codes: codes,
+                    range: range,
+                    affectedRange: affectedRange
+                };
+            }
+        }
+
+        if (commentFlag) {
+            this.commentFlags.push(commentFlag);
+
+            //add an ignore for everything in this comment except for Unknown_diagnostic_code_1014
+            this.commentFlags.push({
+                affectedRange: commentFlag.range,
+                range: commentFlag.range,
+                codes: this.allCodesExceptIgnores,
+                file: this.file
+            });
+        }
+    }
+
+    /**
+     * Small tokenizer for bs:disable comments
+     */
+    private tokenize(text: string, range: Range) {
+        let lowerText = text.toLowerCase();
+        let offset = 0;
+        let commentTokenText: string;
+
+        for (const starter of this.commentStarters) {
+            if (text.startsWith(starter)) {
+                commentTokenText = starter;
+                offset = starter.length;
+                lowerText = lowerText.substring(commentTokenText.length);
+                break;
+            }
+        }
+
+        let disableType: 'line' | 'next-line';
+        //trim leading/trailing whitespace
+        let len = lowerText.length;
+        lowerText = lowerText.trimLeft();
+        offset += len - lowerText.length;
+        if (lowerText.startsWith('bs:disable-line')) {
+            lowerText = lowerText.substring('bs:disable-line'.length);
+            offset += 'bs:disable-line'.length;
+            disableType = 'line';
+        } else if (lowerText.startsWith('bs:disable-next-line')) {
+            lowerText = lowerText.substring('bs:disable-next-line'.length);
+            offset += 'bs:disable-next-line'.length;
+            disableType = 'next-line';
+        } else {
+            return null;
+        }
+
+        //discard the colon
+        if (lowerText.startsWith(':')) {
+            lowerText = lowerText.substring(1);
+            offset += 1;
+        }
+
+        let items = this.tokenizeByWhitespace(lowerText);
+        let codes = [] as Array<{ code: string; range: Range }>;
+        for (let item of items) {
+            codes.push({
+                code: item.text,
+                range: util.createRange(
+                    range.start.line,
+                    range.start.character + offset + item.startIndex,
+                    range.start.line,
+                    range.start.character + offset + item.startIndex + item.text.length
+                )
+            });
+        }
+
+        return {
+            commentTokenText: commentTokenText,
+            disableType: disableType,
+            codes: codes
+        };
+    }
+
+    /**
+     * Given a string, extract each item split by whitespace
+     * @param text
+     */
+    private tokenizeByWhitespace(text: string) {
+        let tokens = [] as Array<{ startIndex: number; text: string }>;
+        let currentToken = null;
+        for (let i = 0; i < text.length; i++) {
+            let char = text[i];
+            //if we hit whitespace
+            if (char === ' ' || char === '\t') {
+                if (currentToken) {
+                    tokens.push(currentToken);
+                    currentToken = null;
+                }
+
+                //we hit non-whitespace
+            } else {
+                if (!currentToken) {
+                    currentToken = {
+                        startIndex: i,
+                        text: ''
+                    };
+                }
+                currentToken.text += char;
+            }
+        }
+        if (currentToken) {
+            tokens.push(currentToken);
+        }
+        return tokens;
+    }
+}
diff --git a/src/files/BrsFile.ts b/src/files/BrsFile.ts
index fb9f2e245..d8f6621f6 100644
--- a/src/files/BrsFile.ts
+++ b/src/files/BrsFile.ts
@@ -1,11 +1,11 @@
 import type { CodeWithSourceMap } from 'source-map';
 import { SourceNode } from 'source-map';
-import type { CompletionItem, Hover, Range, Position } from 'vscode-languageserver';
+import type { CompletionItem, Hover, Position } from 'vscode-languageserver';
 import { CompletionItemKind, SymbolKind, Location, SignatureInformation, ParameterInformation, DocumentSymbol, SymbolInformation, TextEdit } from 'vscode-languageserver';
 import chalk from 'chalk';
 import * as path from 'path';
 import type { Scope } from '../Scope';
-import { diagnosticCodes, DiagnosticMessages } from '../DiagnosticMessages';
+import { DiagnosticCodeMap, diagnosticCodes, DiagnosticMessages } from '../DiagnosticMessages';
 import { FunctionScope } from '../FunctionScope';
 import type { Callable, CallableArg, CallableParam, CommentFlag, FunctionCall, BsDiagnostic, FileReference } from '../interfaces';
 import type { Token } from '../lexer';
@@ -26,6 +26,7 @@ import { isCallExpression, isClassMethodStatement, isClassStatement, isCommentSt
 import type { BscType } from '../types/BscType';
 import { createVisitor, WalkMode } from '../astUtils/visitors';
 import type { DependencyGraph } from '../DependencyGraph';
+import { CommentFlagProcessor } from '../CommentFlagProcessor';
 
 /**
  * Holds all details about this file within the scope of the whole program
@@ -231,7 +232,7 @@ export class BrsFile {
                 });
             });
 
-            this.getIgnores(lexer.tokens);
+            this.getCommentFlags(lexer.tokens);
 
             let preprocessor = new Preprocessor();
 
@@ -390,77 +391,17 @@ export class BrsFile {
      * Find all comment flags in the source code. These enable or disable diagnostic messages.
      * @param lines - the lines of the program
      */
-    public getIgnores(tokens: Token[]) {
-        //TODO use the comment statements found in the AST for this instead of text search
-        let allCodesExcept1014 = diagnosticCodes.filter((x) => x !== DiagnosticMessages.unknownDiagnosticCode(0).code);
+    public getCommentFlags(tokens: Token[]) {
+        const processor = new CommentFlagProcessor(this, ['rem', `'`], [], diagnosticCodes, [DiagnosticCodeMap.unknownDiagnosticCode]);
+
         this.commentFlags = [];
         for (let token of tokens) {
-            let tokenized = util.tokenizeBsDisableComment(token);
-            if (!tokenized) {
-                continue;
-            }
-
-            let affectedRange: Range;
-            if (tokenized.disableType === 'line') {
-                affectedRange = util.createRange(token.range.start.line, 0, token.range.start.line, token.range.start.character);
-            } else if (tokenized.disableType === 'next-line') {
-                affectedRange = util.createRange(token.range.start.line + 1, 0, token.range.start.line + 1, Number.MAX_SAFE_INTEGER);
-            }
-
-            let commentFlag: CommentFlag;
-
-            //statement to disable EVERYTHING
-            if (tokenized.codes.length === 0) {
-                commentFlag = {
-                    file: this,
-                    //null means all codes
-                    codes: null,
-                    range: token.range,
-                    affectedRange: affectedRange
-                };
-
-                //disable specific diagnostic codes
-            } else {
-                let codes = [] as number[];
-                for (let codeToken of tokenized.codes) {
-                    let codeInt = parseInt(codeToken.code);
-                    if (isNaN(codeInt)) {
-                        //don't validate non-numeric codes
-                        continue;
-                    }
-                    //add a warning for unknown codes
-                    if (diagnosticCodes.includes(codeInt)) {
-                        codes.push(codeInt);
-                    } else {
-                        this.diagnostics.push({
-                            ...DiagnosticMessages.unknownDiagnosticCode(codeInt),
-                            file: this,
-                            range: codeToken.range
-                        });
-                    }
-                }
-                if (codes.length > 0) {
-                    commentFlag = {
-                        file: this,
-                        codes: codes,
-                        range: token.range,
-                        affectedRange: affectedRange
-                    };
-                }
-            }
-
-            if (commentFlag) {
-                this.commentFlags.push(commentFlag);
-
-                //add an ignore for everything in this comment except for Unknown_diagnostic_code_1014
-                this.commentFlags.push({
-                    affectedRange: commentFlag.range,
-                    range: commentFlag.range,
-                    codes: allCodesExcept1014,
-                    file: this
-                });
+            if (token.kind === TokenKind.Comment) {
+                processor.tryAdd(token.text, token.range);
             }
         }
+        this.commentFlags.push(...processor.commentFlags);
+        this.diagnostics.push(...processor.diagnostics);
     }
 
     public scopesByFunc = new Map<FunctionExpression, FunctionScope>();
diff --git a/src/interfaces.ts b/src/interfaces.ts
index 7eb66f079..23d95d209 100644
--- a/src/interfaces.ts
+++ b/src/interfaces.ts
@@ -163,7 +163,7 @@ export interface CallableContainer {
 export type CallableContainerMap = Map<string, CallableContainer[]>;
 
 export interface CommentFlag {
-    file: BrsFile;
+    file: BscFile;
     /**
      * The location of the ignore comment.
      */
@@ -172,7 +172,7 @@ export interface CommentFlag {
      * The range that this flag applies to (i.e. the lines that should be suppressed/re-enabled)
      */
     affectedRange: Range;
-    codes: number[] | null;
+    codes: DiagnosticCode[] | null;
 }
 
 type ValidateHandler = (scope: Scope, files: BscFile[], callables: CallableContainerMap) => void;
@@ -232,3 +232,5 @@ export interface ExpressionInfo {
     varExpressions: Expression[];
     uniqueVarNames: string[];
 }
+
+export type DiagnosticCode = number | string;
diff --git a/src/util.spec.ts b/src/util.spec.ts
index 6563c46ea..809847157 100644
--- a/src/util.spec.ts
+++ b/src/util.spec.ts
@@ -396,116 +396,6 @@ describe('util', () => {
         });
     });
 
-    describe('tokenizeByWhitespace', () => {
-        it('works with single chars', () => {
-            expect(util.tokenizeByWhitespace('a b c')).to.deep.equal([{
-                startIndex: 0,
-                text: 'a'
-            }, {
-                startIndex: 2,
-                text: 'b'
-            },
-            {
-                startIndex: 4,
-                text: 'c'
-            }]);
-        });
-
-        it('works with tabs', () => {
-            expect(util.tokenizeByWhitespace('a\tb\t c')).to.deep.equal([{
-                startIndex: 0,
-                text: 'a'
-            }, {
-                startIndex: 2,
-                text: 'b'
-            },
-            {
-                startIndex: 5,
-                text: 'c'
-            }]);
-
-            it('works with leading whitespace', () => {
-                expect(util.tokenizeByWhitespace('  \ta\tb\t c')).to.deep.equal([{
-                    startIndex: 4,
-                    text: 'a'
-                }, {
-                    startIndex: 6,
-                    text: 'b'
-                },
-                {
-                    startIndex: 9,
-                    text: 'c'
-                }]);
-            });
-
-            it('works with multiple characters in a word', () => {
-                expect(util.tokenizeByWhitespace('abc 123')).to.deep.equal([{
-                    startIndex: 0,
-                    text: 'abc'
-                }, {
-                    startIndex: 4,
-                    text: '123'
-                }]);
-            });
-        });
-    });
-
-    describe('tokenizeBsDisableComment', () => {
-        it('skips non disable comments', () => {
-            expect(util.tokenizeBsDisableComment(
-                Lexer.scan(`'not disable comment`).tokens[0]
-            )).not.to.exist;
-        });
-
-        it('tokenizes bs:disable-line comment', () => {
-            expect(util.tokenizeBsDisableComment(
-                Lexer.scan(`'bs:disable-line`).tokens[0])
-            ).to.eql({
-                commentTokenText: `'`,
-                disableType: 'line',
-                codes: []
-            });
-        });
-
-        it('works for special case', () => {
-            expect(util.tokenizeBsDisableComment(
-                Lexer.scan(`print "hi" 'bs:disable-line: 123456 999999   aaaab`).tokens[2])
-            ).to.eql({
-                commentTokenText: `'`,
-                disableType: 'line',
-                codes: [{
-                    code: '123456',
-                    range: Range.create(0, 29, 0, 35)
-                }, {
-                    code: '999999',
-                    range: Range.create(0, 36, 0, 42)
-                }, {
-                    code: 'aaaab',
-                    range: Range.create(0, 45, 0, 50)
-                }]
-            });
-        });
-
-        it('tokenizes bs:disable-line comment with codes', () => {
-            expect(util.tokenizeBsDisableComment(
-                Lexer.scan(`'bs:disable-line:1 2 3`).tokens[0])
-            ).to.eql({
-                commentTokenText: `'`,
-                disableType: 'line',
-                codes: [{
-                    code: '1',
-                    range: Range.create(0, 17, 0, 18)
-                }, {
-                    code: '2',
-                    range: Range.create(0, 19, 0, 20)
-                }, {
-                    code: '3',
-                    range: Range.create(0, 21, 0, 22)
-                }]
-            });
-        });
-    });
-
     describe('getTextForRange', () => {
         const testArray = ['The quick', 'brown fox', 'jumps over', 'the lazy dog'];
         const testString = testArray.join('\n');
diff --git a/src/util.ts b/src/util.ts
index 05d605c4f..36607bf42 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -656,103 +656,6 @@ export class Util {
         }
     }
 
-    /**
-     * Small tokenizer for bs:disable comments
-     */
-    public tokenizeBsDisableComment(token: Token) {
-        if (token.kind !== TokenKind.Comment) {
-            return null;
-        }
-        let lowerText = token.text.toLowerCase();
-        let offset = 0;
-        let commentTokenText: string;
-
-        if (token.text.startsWith(`'`)) {
-            commentTokenText = `'`;
-            offset = 1;
-            lowerText = lowerText.substring(1);
-        } else if (lowerText.startsWith('rem')) {
-            commentTokenText = lowerText.substring(0, 3);
-            offset = 3;
-            lowerText = lowerText.substring(3);
-        }
-
-        let disableType: 'line' | 'next-line';
-        //trim leading/trailing whitespace
-        let len = lowerText.length;
-        lowerText = lowerText.trimLeft();
-        offset += len - lowerText.length;
-        if (lowerText.startsWith('bs:disable-line')) {
-            lowerText = lowerText.substring('bs:disable-line'.length);
-            offset += 'bs:disable-line'.length;
-            disableType = 'line';
-        } else if (lowerText.startsWith('bs:disable-next-line')) {
-            lowerText = lowerText.substring('bs:disable-next-line'.length);
-            offset += 'bs:disable-next-line'.length;
-            disableType = 'next-line';
-        } else {
-            return null;
-        }
-        //do something with the colon
-        if (lowerText.startsWith(':')) {
-            lowerText = lowerText.substring(1);
-            offset += 1;
-        }
-
-        let items = this.tokenizeByWhitespace(lowerText);
-        let codes = [] as Array<{ code: string; range: Range }>;
-        for (let item of items) {
-            codes.push({
-                code: item.text,
-                range: util.createRange(
-                    token.range.start.line,
-                    token.range.start.character + offset + item.startIndex,
-                    token.range.start.line,
-                    token.range.start.character + offset + item.startIndex + item.text.length
-                )
-            });
-        }
-
-        return {
-            commentTokenText: commentTokenText,
-            disableType: disableType,
-            codes: codes
-        };
-    }
-
-    /**
-     * Given a string, extract each item split by whitespace
-     * @param text
-     */
-    public tokenizeByWhitespace(text: string) {
-        let tokens = [] as Array<{ startIndex: number; text: string }>;
-        let currentToken = null;
-        for (let i = 0; i < text.length; i++) {
-            let char = text[i];
-            //if we hit whitespace
-            if (char === ' ' || char === '\t') {
-                if (currentToken) {
-                    tokens.push(currentToken);
-                    currentToken = null;
-                }
-
-                //we hit non-whitespace
-            } else {
-                if (!currentToken) {
-                    currentToken = {
-                        startIndex: i,
-                        text: ''
-                    };
-                }
-                currentToken.text += char;
-            }
-        }
-        if (currentToken) {
-            tokens.push(currentToken);
-        }
-        return tokens;
-    }
-
     /**
      * Walks up the chain
      * @param currentPath

From 4542b74aff19672864e32a7d7f50da57812c0d47 Mon Sep 17 00:00:00 2001
From: Bronley <bronley@gmail.com>
Date: Sat, 13 Mar 2021 14:05:36 -0500
Subject: [PATCH 2/4] Add support for xml comment flags

---
 src/CommentFlagProcessor.ts |  4 ---
 src/files/BrsFile.ts        |  2 +-
 src/files/XmlFile.spec.ts   | 52 +++++++++++++++++++++++++++++++++++++
 src/files/XmlFile.ts        | 32 ++++++++++++++++++++---
 src/interfaces.ts           |  2 +-
 src/parser/SGParser.ts      |  5 +++-
 src/util.ts                 | 17 +++++-------
 7 files changed, 94 insertions(+), 20 deletions(-)

diff --git a/src/CommentFlagProcessor.ts b/src/CommentFlagProcessor.ts
index 46bd2191b..67b4e658c 100644
--- a/src/CommentFlagProcessor.ts
+++ b/src/CommentFlagProcessor.ts
@@ -13,10 +13,6 @@ export class CommentFlagProcessor {
          * An array of strings containing the types of text that a comment starts with. (i.e. `REM`, `'`, `<!--`)
          */
         public commentStarters = [] as string[],
-        /**
-         * An array of strings containing the types of text that a comment ends with. (i.e. `-->`)
-         */
-        public commentFinishers = [] as string[],
         /**
          * Valid diagnostic codes. Codes NOT in this list will be flagged
          */
diff --git a/src/files/BrsFile.ts b/src/files/BrsFile.ts
index d8f6621f6..e1e8a645e 100644
--- a/src/files/BrsFile.ts
+++ b/src/files/BrsFile.ts
@@ -392,7 +392,7 @@ export class BrsFile {
      * @param lines - the lines of the program
      */
     public getCommentFlags(tokens: Token[]) {
-        const processor = new CommentFlagProcessor(this, ['rem', `'`], [], diagnosticCodes, [DiagnosticCodeMap.unknownDiagnosticCode]);
+        const processor = new CommentFlagProcessor(this, ['rem', `'`], diagnosticCodes, [DiagnosticCodeMap.unknownDiagnosticCode]);
 
         this.commentFlags = [];
         for (let token of tokens) {
diff --git a/src/files/XmlFile.spec.ts b/src/files/XmlFile.spec.ts
index 9bbac5b44..f549980c3 100644
--- a/src/files/XmlFile.spec.ts
+++ b/src/files/XmlFile.spec.ts
@@ -1093,4 +1093,56 @@ describe('XmlFile', () => {
         `);
         expect(file.scriptTagImports[0]?.text).to.eql('SingleQuotedFile.brs');
     });
+
+    describe('commentFlags', () => {
+        it('ignores warning from previous line comment', () => {
+            //component without a name attribute
+            program.addOrReplaceFile<XmlFile>('components/file.xml', trim`
+                <?xml version="1.0" encoding="utf-8" ?>
+                <!--bs:disable-next-line-->
+                <component>
+                </component>
+            `);
+            program.validate();
+            expectZeroDiagnostics(program);
+        });
+
+        it('ignores warning from previous line just for the specified code', () => {
+            //component without a name attribute
+            program.addOrReplaceFile<XmlFile>('components/file.xml', trim`
+                <?xml version="1.0" encoding="utf-8" ?>
+                <!--bs:disable-next-line 1006-->
+                <component>
+                </component>
+            `);
+            program.validate();
+            expect(program.getDiagnostics().map(x => x.message)).to.eql([
+                DiagnosticMessages.xmlComponentMissingExtendsAttribute().message
+            ]);
+        });
+
+        it('ignores warning from previous line comment', () => {
+            //component without a name attribute
+            program.addOrReplaceFile<XmlFile>('components/file.xml', trim`
+                <?xml version="1.0" encoding="utf-8" ?>
+                <component> <!--bs:disable-line-->
+                </component>
+            `);
+            program.validate();
+            expectZeroDiagnostics(program);
+        });
+
+        it('ignores warning from previous line just for the specified code', () => {
+            //component without a name attribute
+            program.addOrReplaceFile<XmlFile>('components/file.xml', trim`
+                <?xml version="1.0" encoding="utf-8" ?>
+                <component> <!--bs:disable-line 1006-->
+                </component>
+            `);
+            program.validate();
+            expect(program.getDiagnostics().map(x => x.message)).to.eql([
+                DiagnosticMessages.xmlComponentMissingExtendsAttribute().message
+            ]);
+        });
+    });
 });
diff --git a/src/files/XmlFile.ts b/src/files/XmlFile.ts
index ee927ea9f..51288fd99 100644
--- a/src/files/XmlFile.ts
+++ b/src/files/XmlFile.ts
@@ -2,18 +2,20 @@ import * as path from 'path';
 import type { CodeWithSourceMap } from 'source-map';
 import { SourceNode } from 'source-map';
 import type { CompletionItem, Hover, Location, Position, Range } from 'vscode-languageserver';
-import { DiagnosticMessages } from '../DiagnosticMessages';
+import { DiagnosticCodeMap, diagnosticCodes, DiagnosticMessages } from '../DiagnosticMessages';
 import type { FunctionScope } from '../FunctionScope';
-import type { Callable, BsDiagnostic, File, FileReference, FunctionCall } from '../interfaces';
+import type { Callable, BsDiagnostic, File, FileReference, FunctionCall, CommentFlag } from '../interfaces';
 import type { Program } from '../Program';
 import util from '../util';
-import SGParser from '../parser/SGParser';
+import SGParser, { rangeFromTokenValue } from '../parser/SGParser';
 import chalk from 'chalk';
 import { Cache } from '../Cache';
 import type { DependencyGraph } from '../DependencyGraph';
 import type { SGAst, SGToken } from '../parser/SGTypes';
 import { SGScript } from '../parser/SGTypes';
 import { SGTranspileState } from '../parser/SGTranspileState';
+import { CommentFlagProcessor } from '../CommentFlagProcessor';
+import type { IToken, TokenType } from 'chevrotain';
 
 export class XmlFile {
     constructor(
@@ -49,6 +51,8 @@ export class XmlFile {
      */
     public extension: string;
 
+    public commentFlags = [] as CommentFlag[];
+
     /**
      * The list of script imports delcared in the XML of this file.
      * This excludes parent imports and auto codebehind imports
@@ -187,6 +191,8 @@ export class XmlFile {
             file: this
         }));
 
+        this.getCommentFlags(this.parser.tokens as any[]);
+
         if (!this.parser.ast.root) {
             //skip empty XML
             return;
@@ -199,6 +205,26 @@ export class XmlFile {
         this.validateComponent(this.parser.ast);
     }
 
+    /**
+     * Collect all bs: comment flags
+     */
+    public getCommentFlags(tokens: Array<IToken & { tokenType: TokenType }>) {
+        const processor = new CommentFlagProcessor(this, ['<!--'], diagnosticCodes, [DiagnosticCodeMap.unknownDiagnosticCode]);
+
+        this.commentFlags = [];
+        for (let token of tokens) {
+            if (token.tokenType.name === 'Comment') {
+                processor.tryAdd(
+                    //remove the close comment symbol
+                    token.image.replace(/\-\-\>$/, ''),
+                    rangeFromTokenValue(token)
+                );
+            }
+        }
+        this.commentFlags.push(...processor.commentFlags);
+        this.diagnostics.push(...processor.diagnostics);
+    }
+
     private validateComponent(ast: SGAst) {
         const { root, component } = ast;
         if (!component) {
diff --git a/src/interfaces.ts b/src/interfaces.ts
index 23d95d209..90d50f781 100644
--- a/src/interfaces.ts
+++ b/src/interfaces.ts
@@ -13,7 +13,7 @@ import type { SourceNode } from 'source-map';
 import type { BscType } from './types/BscType';
 
 export interface BsDiagnostic extends Diagnostic {
-    file: File;
+    file: BscFile;
     /**
      * A generic data container where additional details of the diagnostic can be stored. These are stripped out before being sent to a languageclient, and not printed to the console.
      */
diff --git a/src/parser/SGParser.ts b/src/parser/SGParser.ts
index bf76cae1a..6d6ebe112 100644
--- a/src/parser/SGParser.ts
+++ b/src/parser/SGParser.ts
@@ -14,6 +14,8 @@ export default class SGParser {
      */
     public ast: SGAst = new SGAst();
 
+    public tokens: IToken[];
+
     /**
      * The list of diagnostics found during the parse process
      */
@@ -81,6 +83,7 @@ export default class SGParser {
         this.diagnostics = [];
 
         const { cst, tokenVector, lexErrors, parseErrors } = parser.parse(fileContents);
+        this.tokens = tokenVector;
 
         if (lexErrors.length) {
             for (const err of lexErrors) {
@@ -324,7 +327,7 @@ function rangeFromTokens(start: IToken, end?: IToken) {
 }
 
 //make range not including quotes
-function rangeFromTokenValue(token: IToken) {
+export function rangeFromTokenValue(token: IToken) {
     if (!token) {
         return undefined;
     }
diff --git a/src/util.ts b/src/util.ts
index 36607bf42..66f034dc8 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -26,7 +26,7 @@ import type { DottedGetExpression, Expression, VariableExpression } from './pars
 import { Logger, LogLevel } from './Logger';
 import type { Token } from './lexer';
 import { TokenKind } from './lexer';
-import { isBrsFile, isDottedGetExpression, isExpression, isVariableExpression, WalkMode } from './astUtils';
+import { isDottedGetExpression, isExpression, isVariableExpression, WalkMode } from './astUtils';
 import { CustomType } from './types/CustomType';
 import { SourceNode } from 'source-map';
 import type { SGAttribute } from './parser/SGTypes';
@@ -642,15 +642,12 @@ export class Util {
      * @param diagnostic
      */
     public diagnosticIsSuppressed(diagnostic: BsDiagnostic) {
-        //for now, we only support suppressing brs file diagnostics
-        if (isBrsFile(diagnostic.file)) {
-            for (let flag of diagnostic.file.commentFlags) {
-                //this diagnostic is affected by this flag
-                if (this.rangeContains(flag.affectedRange, diagnostic.range.start)) {
-                    //if the flag acts upon this diagnostic's code
-                    if (flag.codes === null || flag.codes.includes(diagnostic.code as number)) {
-                        return true;
-                    }
+        for (let flag of diagnostic.file?.commentFlags ?? []) {
+            //this diagnostic is affected by this flag
+            if (this.rangeContains(flag.affectedRange, diagnostic.range.start)) {
+                //if the flag acts upon this diagnostic's code
+                if (flag.codes === null || flag.codes.includes(diagnostic.code as number)) {
+                    return true;
                 }
             }
         }

From 72fd570702a5096fe6634392e00afc147c24a83e Mon Sep 17 00:00:00 2001
From: Bronley <bronley@gmail.com>
Date: Sat, 13 Mar 2021 14:10:43 -0500
Subject: [PATCH 3/4] fix lint and tsc issues

---
 src/DiagnosticCollection.spec.ts | 4 ++--
 src/ProgramBuilder.ts            | 6 +++---
 src/util.spec.ts                 | 1 -
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/DiagnosticCollection.spec.ts b/src/DiagnosticCollection.spec.ts
index 8bee6621a..2404fe2a6 100644
--- a/src/DiagnosticCollection.spec.ts
+++ b/src/DiagnosticCollection.spec.ts
@@ -2,7 +2,7 @@ import type { BsDiagnostic } from '.';
 import { DiagnosticCollection } from './DiagnosticCollection';
 import type { Workspace } from './LanguageServer';
 import type { ProgramBuilder } from './ProgramBuilder';
-import type { File } from './interfaces';
+import type { BscFile } from './interfaces';
 import util from './util';
 import { expect } from 'chai';
 
@@ -100,7 +100,7 @@ describe('DiagnosticCollection', () => {
             diagnostics.push({
                 file: {
                     pathAbsolute: filePath
-                } as File,
+                } as BscFile,
                 range: util.createRange(0, 0, 0, 0),
                 //the code doesn't matter as long as the messages are different, so just enforce unique messages for this test files
                 code: 123,
diff --git a/src/ProgramBuilder.ts b/src/ProgramBuilder.ts
index bd830254d..c06b9ec89 100644
--- a/src/ProgramBuilder.ts
+++ b/src/ProgramBuilder.ts
@@ -2,7 +2,7 @@ import * as debounce from 'debounce-promise';
 import * as path from 'path';
 import * as rokuDeploy from 'roku-deploy';
 import type { BsConfig } from './BsConfig';
-import type { BsDiagnostic, File, FileObj, FileResolver } from './interfaces';
+import type { BscFile, BsDiagnostic, FileObj, FileResolver } from './interfaces';
 import { Program } from './Program';
 import { standardizePath as s, util } from './util';
 import { Watcher } from './Watcher';
@@ -65,7 +65,7 @@ export class ProgramBuilder {
     private staticDiagnostics = [] as BsDiagnostic[];
 
     public addDiagnostic(filePathAbsolute: string, diagnostic: Partial<BsDiagnostic>) {
-        let file: File = this.program.getFileByPathAbsolute(filePathAbsolute);
+        let file: BscFile = this.program.getFileByPathAbsolute(filePathAbsolute);
         if (!file) {
             file = {
                 pkgPath: this.program.getPkgPath(filePathAbsolute),
@@ -73,7 +73,7 @@ export class ProgramBuilder {
                 getDiagnostics: () => {
                     return [<any>diagnostic];
                 }
-            };
+            } as BscFile;
         }
         diagnostic.file = file;
         this.staticDiagnostics.push(<any>diagnostic);
diff --git a/src/util.spec.ts b/src/util.spec.ts
index 809847157..a8d877f64 100644
--- a/src/util.spec.ts
+++ b/src/util.spec.ts
@@ -2,7 +2,6 @@ import { expect } from 'chai';
 import * as path from 'path';
 import util, { standardizePath as s } from './util';
 import { Position, Range } from 'vscode-languageserver';
-import { Lexer } from './lexer';
 import type { BsConfig } from './BsConfig';
 import * as fsExtra from 'fs-extra';
 import { createSandbox } from 'sinon';

From 7d189d6b52c57047c539de8e879c3bded40fb7c3 Mon Sep 17 00:00:00 2001
From: Bronley <bronley@gmail.com>
Date: Thu, 18 Mar 2021 17:32:35 -0400
Subject: [PATCH 4/4] verify bs:disable comment with leading space works

---
 src/CommentFlagProcessor.spec.ts | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/src/CommentFlagProcessor.spec.ts b/src/CommentFlagProcessor.spec.ts
index 187a41cd4..b3d539fc4 100644
--- a/src/CommentFlagProcessor.spec.ts
+++ b/src/CommentFlagProcessor.spec.ts
@@ -124,5 +124,33 @@ describe('CommentFlagProcessor', () => {
                 }]
             });
         });
+
+        it('tokenizes bs:disable-line comment with leading space', () => {
+            const token = Lexer.scan(`' bs:disable-line:1`).tokens[0];
+            expect(
+                processor['tokenize'](token.text, token.range)
+            ).to.eql({
+                commentTokenText: `'`,
+                disableType: 'line',
+                codes: [{
+                    code: '1',
+                    range: Range.create(0, 18, 0, 19)
+                }]
+            });
+        });
+
+        it('tokenizes bs:disable-line comment with leading tab', () => {
+            const token = Lexer.scan(`'\tbs:disable-line:1`).tokens[0];
+            expect(
+                processor['tokenize'](token.text, token.range)
+            ).to.eql({
+                commentTokenText: `'`,
+                disableType: 'line',
+                codes: [{
+                    code: '1',
+                    range: Range.create(0, 18, 0, 19)
+                }]
+            });
+        });
     });
 });