Support for type definition files

.vscode/launch.json

@@ -13,7 +13,7 @@
             "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha",
             "sourceMaps": true,
             "args": [
-                "${file}"
+                "${relativeFile}"
             ],
             "skipFiles": [
                 "${workspaceFolder}/node_modules/**/*.js",

src/BsConfig.ts

@@ -98,6 +98,12 @@ export interface BsConfig {
      */
     emitFullPaths?: boolean;
 
+    /**
+     * Emit type definition files (`d.bs`)
+     * @default true
+     */
+    emitDefinitions?: boolean;
+
     /**
      * A list of filters used to exclude diagnostics from the output
      */

src/Program.spec.ts

@@ -60,7 +60,10 @@ describe('Program', () => {
 
                 //resolve lib.brs from memory instead of going to disk
                 program.fileResolvers.push((pathAbsolute) => {
-                    if (pathAbsolute === s`${rootDir}/source/lib.brs`) {
+                    if (
+                        pathAbsolute === s`${rootDir}/source/lib.brs` ||
+                        pathAbsolute === s`${rootDir}/source/lib.d.bs`
+                    ) {
                         return `'comment`;
                     }
                 });

src/Program.ts

@@ -18,12 +18,13 @@ import { globalFile } from './globalCallables';
 import { parseManifest, ManifestValue } from './preprocessor/Manifest';
 import { URI } from 'vscode-uri';
 import PluginInterface from './PluginInterface';
-import { isXmlFile } from './astUtils/reflection';
+import { isBrsFile, isXmlFile } from './astUtils/reflection';
 const startOfSourcePkgPath = `source${path.sep}`;
 
 export interface SourceObj {
     pathAbsolute: string;
     source: string;
+    definitions?: string;
 }
 
 export interface TranspileObj {
@@ -278,6 +279,43 @@ export class Program {
         return this.getComponent(componentName)?.scope;
     }
 
+    /**
+     * Get the type definitions for the specified file.
+     * The results are cached for future performance boosts.
+     */
+    private async getTypedefsForFile(pathAbsolute: string) {
+        //you can only get type definitions for .brs files
+        if (!pathAbsolute.toLowerCase().endsWith('.brs')) {
+            return;
+        }
+
+        let typedefPath = pathAbsolute.replace(/.brs$/i, '.d.bs');
+        let lowerTypedefPath = typedefPath.toLowerCase();
+        //specifically check for `undefined`, because `null` means "does not exist"
+        if (this.typedefCache[lowerTypedefPath] === undefined) {
+            let contents: string | null;
+            try {
+                contents = await this.getFileContents(typedefPath);
+            } catch (e) {
+                //something went wrong trying to load the d file...just ignore the error
+                this.logger.info(`Exception throw trying to load '${typedefPath}'`, e);
+                contents = null;
+            }
+            this.typedefCache[lowerTypedefPath] = contents ?? null;
+        }
+        return this.typedefCache[lowerTypedefPath];
+    }
+
+    /**
+     * A cache for type defintions.
+     * `undefined` means never checked, so we should check the FS.
+     * `null` means file does not exist on FS (so don't check the FS again).
+     * `string` contains the cached type definition file.
+     *
+     * The key is the all-lowercase src path of the typedef file
+     */
+    public typedefCache = {} as { [lowerTypedefSrcPath: string]: string };
+
     /**
      * Load a file into the program. If that file already exists, it is replaced.
      * If file contents are provided, those are used, Otherwise, the file is loaded from the file system
@@ -294,55 +332,61 @@ export class Program {
     public async addOrReplaceFile(fileEntry: FileObj, fileContents?: string): Promise<XmlFile | BrsFile>;
     public async addOrReplaceFile(fileParam: FileObj | string, fileContents?: string): Promise<XmlFile | BrsFile> {
         assert.ok(fileParam, 'fileEntry is required');
-        let pathAbsolute: string;
+        let srcPath: string;
         let pkgPath: string;
         if (typeof fileParam === 'string') {
-            pathAbsolute = s`${this.options.rootDir}/${fileParam}`;
+            srcPath = s`${this.options.rootDir}/${fileParam}`;
             pkgPath = s`${fileParam}`;
         } else {
-            pathAbsolute = s`${fileParam.src}`;
+            srcPath = s`${fileParam.src}`;
             pkgPath = s`${fileParam.dest}`;
         }
-        let file = await this.logger.time(LogLevel.debug, ['Program.addOrReplaceFile()', chalk.green(pathAbsolute)], async () => {
+        let file = await this.logger.time(LogLevel.debug, ['Program.addOrReplaceFile()', chalk.green(srcPath)], async () => {
 
-            assert.ok(pathAbsolute, 'fileEntry.src is required');
+            assert.ok(srcPath, 'fileEntry.src is required');
             assert.ok(pkgPath, 'fileEntry.dest is required');
 
             //if the file is already loaded, remove it
-            if (this.hasFile(pathAbsolute)) {
-                this.removeFile(pathAbsolute);
+            if (this.hasFile(srcPath)) {
+                this.removeFile(srcPath);
             }
-            let fileExtension = path.extname(pathAbsolute).toLowerCase();
+            let fileExtension = path.extname(srcPath).toLowerCase();
             let file: BrsFile | XmlFile | undefined;
 
             //load the file contents by file path if not provided
             let getFileContents = async () => {
                 if (fileContents === undefined) {
-                    return this.getFileContents(pathAbsolute);
+                    return this.getFileContents(srcPath);
                 } else {
                     return fileContents;
                 }
             };
-            //get the extension of the file
-            if (fileExtension === '.brs' || fileExtension === '.bs') {
-                let brsFile = new BrsFile(pathAbsolute, pkgPath, this);
+
+            //if this is a type definition file, store its contents in the cache
+            if (srcPath.toLowerCase().endsWith('.d.bs')) {
+                this.typedefCache[srcPath.toLowerCase()] = await getFileContents();
+
+            } else if (fileExtension === '.brs' || fileExtension === '.bs') {
+                let brsFile = new BrsFile(srcPath, pkgPath, this);
 
                 //add file to the `source` dependency list
                 if (brsFile.pkgPath.startsWith(startOfSourcePkgPath)) {
                     this.createSourceScope();
                     this.dependencyGraph.addDependency('scope:source', brsFile.dependencyGraphKey);
                 }
 
+
                 //add the file to the program
-                this.files[pathAbsolute] = brsFile;
+                this.files[srcPath] = brsFile;
                 let fileContents: SourceObj = {
-                    pathAbsolute: pathAbsolute,
-                    source: await getFileContents()
+                    pathAbsolute: srcPath,
+                    source: await getFileContents(),
+                    definitions: await this.getTypedefsForFile(srcPath)
                 };
                 this.plugins.emit('beforeFileParse', fileContents);
 
-                this.logger.time(LogLevel.info, ['parse', chalk.green(pathAbsolute)], () => {
-                    brsFile.parse(fileContents.source);
+                this.logger.time(LogLevel.info, ['parse', chalk.green(srcPath)], () => {
+                    brsFile.parse(fileContents.source, fileContents.definitions);
                 });
                 file = brsFile;
 
@@ -355,11 +399,11 @@ export class Program {
                 //resides in the components folder (Roku will only parse xml files in the components folder)
                 pkgPath.toLowerCase().startsWith(util.pathSepNormalize(`components/`))
             ) {
-                let xmlFile = new XmlFile(pathAbsolute, pkgPath, this);
+                let xmlFile = new XmlFile(srcPath, pkgPath, this);
                 //add the file to the program
-                this.files[pathAbsolute] = xmlFile;
+                this.files[srcPath] = xmlFile;
                 let fileContents: SourceObj = {
-                    pathAbsolute: pathAbsolute,
+                    pathAbsolute: srcPath,
                     source: await getFileContents()
                 };
                 this.plugins.emit('beforeFileParse', fileContents);
@@ -461,6 +505,12 @@ export class Program {
      */
     public removeFile(pathAbsolute: string) {
         pathAbsolute = s`${pathAbsolute}`;
+
+        if (pathAbsolute.endsWith('.d.bs')) {
+            delete this.typedefCache[pathAbsolute.toLowerCase()];
+            return;
+        }
+
         let file = this.getFile(pathAbsolute);
         if (file) {
             this.plugins.emit('beforeFileDispose', file);
@@ -786,6 +836,13 @@ export class Program {
                 fsExtra.writeFile(outputPath, result.code),
                 writeMapPromise
             ]);
+
+            if (isBrsFile(file) && this.options.emitDefinitions !== false) {
+                const typedef = file.getTypedef();
+                const typedefPath = outputPath.replace(/\.brs$/i, '.d.bs');
+                await fsExtra.writeFile(typedefPath, typedef);
+            }
+
             this.plugins.emit('afterFileTranspile', entry);
         });
 
@@ -837,4 +894,4 @@ export class Program {
     }
 }
 
-export type FileResolver = (pathAbsolute: string) => string | undefined | Thenable<string | undefined>;
+export type FileResolver = (pathAbsolute: string) => string | undefined | Thenable<string | undefined> | void;

src/ProgramBuilder.spec.ts

@@ -1,19 +1,20 @@
 import { expect } from 'chai';
 import * as fsExtra from 'fs-extra';
-import * as sinonImport from 'sinon';
+import { createSandbox } from 'sinon';
+const sinon = createSandbox();
 import { Program } from './Program';
 import { ProgramBuilder } from './ProgramBuilder';
 import { standardizePath as s, util } from './util';
 import { Logger, LogLevel } from './Logger';
 
-let sinon = sinonImport.createSandbox();
-let tmpPath = s`${process.cwd()}/.tmp`;
-let rootDir = s`${tmpPath}/rootDir`;
-let stagingFolderPath = s`${tmpPath}/staging`;
-
 describe('ProgramBuilder', () => {
+
+    let tmpPath = s`${process.cwd()}/.tmp`;
+    let rootDir = s`${tmpPath}/rootDir`;
+    let stagingFolderPath = s`${tmpPath}/staging`;
+
     beforeEach(() => {
-        fsExtra.ensureDirSync(tmpPath);
+        fsExtra.ensureDirSync(rootDir);
         fsExtra.emptyDirSync(tmpPath);
     });
     afterEach(() => {
@@ -23,25 +24,13 @@ describe('ProgramBuilder', () => {
     });
 
     let builder: ProgramBuilder;
-    let b: any;
-    let setVfsFile: (filePath: string, contents: string) => void;
     beforeEach(async () => {
         builder = new ProgramBuilder();
-        b = builder;
-        b.options = await util.normalizeAndResolveConfig(undefined);
-        b.program = new Program(b.options);
-        b.logger = new Logger();
-        let vfs = {};
-        setVfsFile = (filePath, contents) => {
-            vfs[filePath] = contents;
-        };
-        sinon.stub(b.program.util, 'getFileContents').callsFake((filePath) => {
-            if (vfs[filePath]) {
-                return vfs[filePath];
-            } else {
-                throw new Error(`Cannot find file "${filePath}"`);
-            }
+        builder.options = await util.normalizeAndResolveConfig({
+            rootDir: rootDir
         });
+        builder.program = new Program(builder.options);
+        builder.logger = new Logger();
     });
 
 
@@ -62,21 +51,53 @@ describe('ProgramBuilder', () => {
                 dest: 'file.xml'
             }]));
 
-            b.program = {
-                addOrReplaceFile: () => { }
-            };
-            let stub = sinon.stub(b.program, 'addOrReplaceFile');
-            await b.loadAllFilesAST();
+            let stub = sinon.stub(builder.program, 'addOrReplaceFile');
+            await builder['loadAllFilesAST']();
             expect(stub.getCalls()).to.be.lengthOf(3);
         });
+
+        it('loads all type definitions first', async () => {
+            const requestedFiles = [] as string[];
+            builder.program.fileResolvers.push((filePath) => {
+                requestedFiles.push(s(filePath));
+            });
+            fsExtra.outputFileSync(s`${rootDir}/source/main.brs`, '');
+            fsExtra.outputFileSync(s`${rootDir}/source/main.d.bs`, '');
+            fsExtra.outputFileSync(s`${rootDir}/source/lib.d.bs`, '');
+            fsExtra.outputFileSync(s`${rootDir}/source/lib.brs`, '');
+            const stub = sinon.stub(builder.program, 'addOrReplaceFile');
+            await builder['loadAllFilesAST']();
+            const srcPaths = stub.getCalls().map(x => x.args[0].src);
+            //the d files should be first
+            expect(srcPaths.indexOf(s`${rootDir}/source/main.d.bs`)).within(0, 1);
+            expect(srcPaths.indexOf(s`${rootDir}/source/lib.d.bs`)).within(0, 1);
+            //the non-d files should be last
+            expect(srcPaths.indexOf(s`${rootDir}/source/main.brs`)).within(2, 3);
+            expect(srcPaths.indexOf(s`${rootDir}/source/lib.brs`)).within(2, 3);
+
+            //the d files should NOT be requested from the FS
+            expect(requestedFiles).not.to.include(s`${rootDir}/source/lib.d.bs`);
+            expect(requestedFiles).not.to.include(s`${rootDir}/source/main.d.bs`);
+        });
+
+        it('does not load non-existent type definition file', async () => {
+            const requestedFiles = [] as string[];
+            builder.program.fileResolvers.push((filePath) => {
+                requestedFiles.push(s(filePath));
+            });
+            fsExtra.outputFileSync(s`${rootDir}/source/main.brs`, '');
+            await builder['loadAllFilesAST']();
+            //the d file should not be requested because `loadAllFilesAST` knows it doesn't exist
+            expect(requestedFiles).not.to.include(s`${rootDir}/source/main.d.bs`);
+        });
     });
 
     describe('run', () => {
         it('uses default options when the config file fails to parse', async () => {
             //supress the console log statements for the bsconfig parse errors
             sinon.stub(console, 'log').returns(undefined);
             //totally bogus config file
-            setVfsFile(s`${rootDir}/bsconfig.json`, '{');
+            fsExtra.outputFileSync(s`${rootDir}/bsconfig.json`, '{');
             await builder.run({
                 project: s`${rootDir}/bsconfig.json`,
                 username: 'john'

src/ProgramBuilder.ts

@@ -388,9 +388,46 @@ export class ProgramBuilder {
             return util.getFilePaths(this.options);
         });
         this.logger.debug('ProgramBuilder.loadAllFilesAST() files:', files);
-        //parse every file
+
+        const allTypedefFiles = [] as string[];
+        const actualTypedefMap = {};
+        const actualTypedefFiles = [] as FileObj[];
+        const nonTypeFiles = [] as FileObj[];
+        for (const file of files) {
+            const srcLower = file.src.toLowerCase();
+            if (srcLower.endsWith('.d.bs')) {
+                actualTypedefMap[srcLower] = true;
+                actualTypedefFiles.push(file);
+            } else {
+                nonTypeFiles.push(file);
+            }
+            if (srcLower.endsWith('.brs')) {
+                allTypedefFiles.push(srcLower.slice(0, -4) + '.d.bs');
+            }
+        }
+
+        //mark the missing type files as `null` so Program doesn't ask the FS for them
+        for (const filePath of allTypedefFiles) {
+            if (!actualTypedefMap[filePath]) {
+                this.program.typedefCache[s(filePath)] = null;
+            }
+        }
+
+        //preload every type definition file first, which eliminates duplicate file loading
+        await Promise.all(
+            actualTypedefFiles.map(async (file) => {
+                try {
+                    await this.program.addOrReplaceFile(file);
+                } catch (e) {
+                    //log the error, but don't fail this process because the file might be fixable later
+                    this.logger.log(e);
+                }
+            })
+        );
+
+        //parse every file other than the type definitions
         await Promise.all(
-            files.map(async (file) => {
+            nonTypeFiles.map(async (file) => {
                 try {
                     let fileExtension = path.extname(file.src).toLowerCase();