Support .d.ts files (#2746)

Fixes #1432

cli/BUILD.gn

@@ -111,6 +111,7 @@ ts_sources = [
   "../js/mock_builtin.js",
   "../js/net.ts",
   "../js/os.ts",
+  "../js/performance.ts",
   "../js/permissions.ts",
   "../js/plugins.d.ts",
   "../js/process.ts",
@@ -127,6 +128,7 @@ ts_sources = [
   "../js/text_encoding.ts",
   "../js/timers.ts",
   "../js/truncate.ts",
+  "../js/type_directives.ts",
   "../js/types.ts",
   "../js/url.ts",
   "../js/url_search_params.ts",
@@ -135,7 +137,6 @@ ts_sources = [
   "../js/window.ts",
   "../js/workers.ts",
   "../js/write_file.ts",
-  "../js/performance.ts",
   "../js/version.ts",
   "../js/xeval.ts",
   "../tsconfig.json",

js/compiler.ts

@@ -10,6 +10,7 @@ import { cwd } from "./dir";
 import { sendSync, msg, flatbuffers } from "./dispatch_flatbuffers";
 import * as os from "./os";
 import { TextDecoder, TextEncoder } from "./text_encoding";
+import { getMappedModuleName, parseTypeDirectives } from "./type_directives";
 import { assert, notImplemented } from "./util";
 import * as util from "./util";
 import { window } from "./window";
@@ -110,6 +111,7 @@ interface SourceFile {
   filename: string | undefined;
   mediaType: msg.MediaType;
   sourceCode: string | undefined;
+  typeDirectives?: Record<string, string>;
 }
 
 interface EmitResult {
@@ -119,7 +121,7 @@ interface EmitResult {
 
 /** Ops to Rust to resolve and fetch a modules meta data. */
 function fetchSourceFile(specifier: string, referrer: string): SourceFile {
-  util.log("compiler.fetchSourceFile", { specifier, referrer });
+  util.log("fetchSourceFile", { specifier, referrer });
   // Send FetchSourceFile message
   const builder = flatbuffers.createBuilder();
   const specifier_ = builder.createString(specifier);
@@ -146,7 +148,8 @@ function fetchSourceFile(specifier: string, referrer: string): SourceFile {
     moduleName: fetchSourceFileRes.moduleName() || undefined,
     filename: fetchSourceFileRes.filename() || undefined,
     mediaType: fetchSourceFileRes.mediaType(),
-    sourceCode
+    sourceCode,
+    typeDirectives: parseTypeDirectives(sourceCode)
   };
 }
 
@@ -168,7 +171,7 @@ function humanFileSize(bytes: number): string {
 
 /** Ops to rest for caching source map and compiled js */
 function cache(extension: string, moduleId: string, contents: string): void {
-  util.log("compiler.cache", moduleId);
+  util.log("cache", extension, moduleId);
   const builder = flatbuffers.createBuilder();
   const extension_ = builder.createString(extension);
   const moduleId_ = builder.createString(moduleId);
@@ -189,7 +192,7 @@ const encoder = new TextEncoder();
 function emitBundle(fileName: string, data: string): void {
   // For internal purposes, when trying to emit to `$deno$` just no-op
   if (fileName.startsWith("$deno$")) {
-    console.warn("skipping compiler.emitBundle", fileName);
+    console.warn("skipping emitBundle", fileName);
     return;
   }
   const encodedData = encoder.encode(data);
@@ -217,7 +220,7 @@ function getExtension(
 }
 
 class Host implements ts.CompilerHost {
-  extensionCache: Record<string, ts.Extension> = {};
+  private _extensionCache: Record<string, ts.Extension> = {};
 
   private readonly _options: ts.CompilerOptions = {
     allowJs: true,
@@ -232,23 +235,37 @@ class Host implements ts.CompilerHost {
     target: ts.ScriptTarget.ESNext
   };
 
+  private _sourceFileCache: Record<string, SourceFile> = {};
+
   private _resolveModule(specifier: string, referrer: string): SourceFile {
+    util.log("host._resolveModule", { specifier, referrer });
     // Handle built-in assets specially.
     if (specifier.startsWith(ASSETS)) {
       const moduleName = specifier.split("/").pop()!;
+      if (moduleName in this._sourceFileCache) {
+        return this._sourceFileCache[moduleName];
+      }
       const assetName = moduleName.includes(".")
         ? moduleName
         : `${moduleName}.d.ts`;
       assert(assetName in assetSourceCode, `No such asset "${assetName}"`);
       const sourceCode = assetSourceCode[assetName];
-      return {
+      const sourceFile = {
         moduleName,
         filename: specifier,
         mediaType: msg.MediaType.TypeScript,
         sourceCode
       };
+      this._sourceFileCache[moduleName] = sourceFile;
+      return sourceFile;
+    }
+    const sourceFile = fetchSourceFile(specifier, referrer);
+    assert(sourceFile.moduleName != null);
+    const { moduleName } = sourceFile;
+    if (!(moduleName! in this._sourceFileCache)) {
+      this._sourceFileCache[moduleName!] = sourceFile;
     }
-    return fetchSourceFile(specifier, referrer);
+    return sourceFile;
   }
 
   /* Deno specific APIs */
@@ -277,7 +294,7 @@ class Host implements ts.CompilerHost {
    * options which were ignored, or `undefined`.
    */
   configure(path: string, configurationText: string): ConfigureResponse {
-    util.log("compile.configure", path);
+    util.log("host.configure", path);
     const { config, error } = ts.parseConfigFileTextToJson(
       path,
       configurationText
@@ -308,7 +325,10 @@ class Host implements ts.CompilerHost {
 
   /* TypeScript CompilerHost APIs */
 
-  fileExists(_fileName: string): boolean {
+  fileExists(fileName: string): boolean {
+    if (fileName.endsWith("package.json")) {
+      throw new TypeError("Automatic type resolution not supported");
+    }
     return notImplemented();
   }
 
@@ -342,13 +362,17 @@ class Host implements ts.CompilerHost {
   ): ts.SourceFile | undefined {
     assert(!shouldCreateNewSourceFile);
     util.log("getSourceFile", fileName);
-    const SourceFile = this._resolveModule(fileName, ".");
-    if (!SourceFile || !SourceFile.sourceCode) {
+    const sourceFile =
+      fileName in this._sourceFileCache
+        ? this._sourceFileCache[fileName]
+        : this._resolveModule(fileName, ".");
+    assert(sourceFile != null);
+    if (!sourceFile.sourceCode) {
       return undefined;
     }
     return ts.createSourceFile(
       fileName,
-      SourceFile.sourceCode,
+      sourceFile.sourceCode,
       languageVersion
     );
   }
@@ -362,26 +386,37 @@ class Host implements ts.CompilerHost {
     containingFile: string
   ): Array<ts.ResolvedModuleFull | undefined> {
     util.log("resolveModuleNames()", { moduleNames, containingFile });
+    const typeDirectives: Record<string, string> | undefined =
+      containingFile in this._sourceFileCache
+        ? this._sourceFileCache[containingFile].typeDirectives
+        : undefined;
     return moduleNames.map(
       (moduleName): ts.ResolvedModuleFull | undefined => {
-        const SourceFile = this._resolveModule(moduleName, containingFile);
-        if (SourceFile.moduleName) {
-          const resolvedFileName = SourceFile.moduleName;
+        const mappedModuleName = getMappedModuleName(
+          moduleName,
+          containingFile,
+          typeDirectives
+        );
+        const sourceFile = this._resolveModule(
+          mappedModuleName,
+          containingFile
+        );
+        if (sourceFile.moduleName) {
+          const resolvedFileName = sourceFile.moduleName;
           // This flags to the compiler to not go looking to transpile functional
           // code, anything that is in `/$asset$/` is just library code
           const isExternalLibraryImport = moduleName.startsWith(ASSETS);
           const extension = getExtension(
             resolvedFileName,
-            SourceFile.mediaType
+            sourceFile.mediaType
           );
-          this.extensionCache[resolvedFileName] = extension;
+          this._extensionCache[resolvedFileName] = extension;
 
-          const r = {
+          return {
             resolvedFileName,
             isExternalLibraryImport,
             extension
           };
-          return r;
         } else {
           return undefined;
         }
@@ -407,7 +442,7 @@ class Host implements ts.CompilerHost {
       } else {
         assert(sourceFiles != null && sourceFiles.length == 1);
         const sourceFileName = sourceFiles![0].fileName;
-        const maybeExtension = this.extensionCache[sourceFileName];
+        const maybeExtension = this._extensionCache[sourceFileName];
 
         if (maybeExtension) {
           // NOTE: If it's a `.json` file we don't want to write it to disk.

js/type_directives.ts

@@ -0,0 +1,87 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+
+interface DirectiveInfo {
+  path: string;
+  start: number;
+  end: number;
+}
+
+/** Remap the module name based on any supplied type directives passed. */
+export function getMappedModuleName(
+  moduleName: string,
+  containingFile: string,
+  typeDirectives?: Record<string, string>
+): string {
+  if (containingFile.endsWith(".d.ts") && !moduleName.endsWith(".d.ts")) {
+    moduleName = `${moduleName}.d.ts`;
+  }
+  if (!typeDirectives) {
+    return moduleName;
+  }
+  if (moduleName in typeDirectives) {
+    return typeDirectives[moduleName];
+  }
+  return moduleName;
+}
+
+/** Matches directives that look something like this and parses out the value
+ * of the directive:
+ *
+ *      // @deno-types="./foo.d.ts"
+ *
+ * [See Diagram](http://bit.ly/31nZPCF)
+ */
+const typeDirectiveRegEx = /@deno-types\s*=\s*(["'])((?:(?=(\\?))\3.)*?)\1/gi;
+
+/** Matches `import` or `export from` statements and parses out the value of the
+ * module specifier in the second capture group:
+ *
+ *      import * as foo from "./foo.js"
+ *      export { a, b, c } from "./bar.js"
+ *
+ * [See Diagram](http://bit.ly/2GSkJlF)
+ */
+const importExportRegEx = /(?:import|export)\s+[\s\S]*?from\s+(["'])((?:(?=(\\?))\3.)*?)\1/;
+
+/** Parses out any Deno type directives that are part of the source code, or
+ * returns `undefined` if there are not any.
+ */
+export function parseTypeDirectives(
+  sourceCode: string | undefined
+): Record<string, string> | undefined {
+  if (!sourceCode) {
+    return;
+  }
+
+  // collect all the directives in the file and their start and end positions
+  const directives: DirectiveInfo[] = [];
+  let maybeMatch: RegExpExecArray | null = null;
+  while ((maybeMatch = typeDirectiveRegEx.exec(sourceCode))) {
+    const [matchString, , path] = maybeMatch;
+    const { index: start } = maybeMatch;
+    directives.push({
+      path,
+      start,
+      end: start + matchString.length
+    });
+  }
+  if (!directives.length) {
+    return;
+  }
+
+  // work from the last directive backwards for the next `import`/`export`
+  // statement
+  directives.reverse();
+  const directiveRecords: Record<string, string> = {};
+  for (const { path, start, end } of directives) {
+    const searchString = sourceCode.substring(end);
+    const maybeMatch = importExportRegEx.exec(searchString);
+    if (maybeMatch) {
+      const [, , fromPath] = maybeMatch;
+      directiveRecords[fromPath] = path;
+    }
+    sourceCode = sourceCode.substring(0, start);
+  }
+
+  return directiveRecords;
+}

tests/error_type_definitions.test

@@ -0,0 +1,4 @@
+args: run --reload tests/error_type_definitions.ts
+check_stderr: true
+exit_code: 1
+output: tests/error_type_definitions.ts.out

tests/error_type_definitions.ts

@@ -0,0 +1,5 @@
+// @deno-types="./type_definitions/bar.d.ts"
+import { Bar } from "./type_definitions/bar.js";
+
+const bar = new Bar();
+console.log(bar);

tests/error_type_definitions.ts.out

@@ -0,0 +1,4 @@
+[WILDCARD]error: Uncaught TypeError: Automatic type resolution not supported
+[WILDCARD]js/compiler.ts:[WILDCARD]
+    at fileExists (js/compiler.ts:[WILDCARD])
+[WILDCARD]

tests/type_definitions.test

@@ -0,0 +1,2 @@
+args: run --reload tests/type_definitions.ts
+output: tests/type_definitions.ts.out

tests/type_definitions.ts

@@ -0,0 +1,4 @@
+// @deno-types="./type_definitions/foo.d.ts"
+import { foo } from "./type_definitions/foo.js";
+
+console.log(foo);

tests/type_definitions.ts.out

@@ -0,0 +1 @@
+[WILDCARD]foo

tests/type_definitions/bar.d.ts

@@ -0,0 +1,7 @@
+/// <reference types="baz" />
+
+declare namespace bar {
+  export class Bar {
+    baz: string;
+  }
+}

tests/type_definitions/foo.d.ts

@@ -0,0 +1,2 @@
+/** An exported value. */
+export const foo: string;

tests/type_definitions/foo.js

@@ -0,0 +1 @@
+export const foo = "foo";

website/manual.md

@@ -580,6 +580,48 @@ import { test, assertEquals } from "./deps.ts";
 This design circumvents a plethora of complexity spawned by package management
 software, centralized code repositories, and superfluous file formats.
 
+### Using external type definitions
+
+Deno supports both JavaScript and TypeScript as first class languages at
+runtime. This means it requires fully qualified module names, including the
+extension (or a server providing the correct media type). In addition, Deno has
+no "magical" module resolution.
+
+The out of the box TypeScript compiler though relies on both extension-less
+modules and the Node.js module resolution logic to apply types to JavaScript
+modules.
+
+In order to bridge this gap, Deno supports compiler hints that inform Deno the
+location of `.d.ts` files and the JavaScript code they relate to. A compiler
+hint looks like this:
+
+```ts
+// @deno-types="./foo.d.ts"
+import * as foo from "./foo.js";
+```
+
+Where the hint effects the next `import` statement (or `export ... from`
+statement) where the value of the `@deno-types` will be substituted at compile
+time instead of the specified module. Like in the above example, the Deno
+compiler will load `./foo.d.ts` instead of `./foo.js`. Deno will still load
+`./foo.js` when it runs the program.
+
+**Not all type definitions are supported.**
+
+Deno will use the compiler hint to load the indicated `.d.ts` files, but some
+`.d.ts` files contain unsupported features. Specifically, some `.d.ts` files
+expect to be able to load or reference type definitions from other packages
+using the module resolution logic. For example a type reference directive to
+include `node`, expecting to resolve to some path like
+`./node_modules/@types/node/index.d.ts`. Since this depends on non-relative
+"magical" resolution, Deno cannot resolve this.
+
+**Why not use the triple-slash type reference?**
+
+The TypeScript compiler supports triple-slash directives, including a type
+reference directive. If Deno used this, it would interfere with the behavior of
+the TypeScript compiler.
+
 ### Testing if current file is the main program
 
 To test if the current script has been executed as the main input to the program