Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support pulling type information directly from .wasm modules #39784

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/compiler/commandLineParser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,13 @@ namespace ts {
category: Diagnostics.Experimental_Options,
description: Diagnostics.Enables_experimental_support_for_emitting_type_metadata_for_decorators
},
{
name: "experimentalWasmModules",
type: "boolean",
affectsSemanticDiagnostics: true,
category: Diagnostics.Experimental_Options,
description: Diagnostics.Enables_experimental_loading_of_type_information_from_experimental_wasm_modules
},

// Advanced
{
Expand Down
8 changes: 8 additions & 0 deletions src/compiler/diagnosticMessages.json
Original file line number Diff line number Diff line change
Expand Up @@ -4473,6 +4473,10 @@
"category": "Message",
"code": 6235
},
"Enables experimental loading of type information from experimental wasm modules.": {
"category": "Message",
"code": 6236
},

"Projects to reference": {
"category": "Message",
Expand Down Expand Up @@ -4872,6 +4876,10 @@
"category": "Error",
"code": 7055
},
"Module '{0}' was resolved to '{1}', but '--experimentalWasmModules' is not used.": {
"category": "Error",
"code": 7056
},

"You cannot rename this element.": {
"category": "Error",
Expand Down
8 changes: 6 additions & 2 deletions src/compiler/emitter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1226,10 +1226,14 @@ namespace ts {
return getPipelinePhase(currentPhase + 1, emitHint, node);
}

function getTextPos() {
return writer.getTextPos();
}

function pipelineEmitWithNotification(hint: EmitHint, node: Node) {
Debug.assert(lastNode === node);
const pipelinePhase = getNextPipelinePhase(PipelinePhase.Notification, hint, node);
onEmitNode(hint, node, pipelinePhase);
onEmitNode(hint, node, pipelinePhase, getTextPos);
Debug.assert(lastNode === node);
}

Expand Down Expand Up @@ -2903,7 +2907,7 @@ namespace ts {

emitSignatureHead(node);
if (onEmitNode) {
onEmitNode(EmitHint.Unspecified, body, emitBlockCallback);
onEmitNode(EmitHint.Unspecified, body, emitBlockCallback, getTextPos);
}
else {
emitBlockFunctionBody(body);
Expand Down
15 changes: 10 additions & 5 deletions src/compiler/moduleNameResolver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ namespace ts {
TypeScript, /** '.ts', '.tsx', or '.d.ts' */
JavaScript, /** '.js' or '.jsx' */
Json, /** '.json' */
Wasm, /** '.wasm' */
TSConfig, /** '.json' with `tsconfig` used instead of `index` */
DtsOnly /** Only '.d.ts' */
}
Expand Down Expand Up @@ -912,7 +913,6 @@ namespace ts {

const jsOnlyExtensions = [Extensions.JavaScript];
const tsExtensions = [Extensions.TypeScript, Extensions.JavaScript];
const tsPlusJsonExtensions = [...tsExtensions, Extensions.Json];
const tsconfigExtensions = [Extensions.TSConfig];
function tryResolveJSModuleWorker(moduleName: string, initialDir: string, host: ModuleResolutionHost): ResolvedModuleWithFailedLookupLocations {
return nodeModuleNameResolverWorker(moduleName, initialDir, { moduleResolution: ModuleResolutionKind.NodeJs, allowJs: true }, host, /*cache*/ undefined, jsOnlyExtensions, /*redirectedReferences*/ undefined);
Expand All @@ -921,7 +921,7 @@ namespace ts {
export function nodeModuleNameResolver(moduleName: string, containingFile: string, compilerOptions: CompilerOptions, host: ModuleResolutionHost, cache?: ModuleResolutionCache, redirectedReference?: ResolvedProjectReference): ResolvedModuleWithFailedLookupLocations;
/* @internal */ export function nodeModuleNameResolver(moduleName: string, containingFile: string, compilerOptions: CompilerOptions, host: ModuleResolutionHost, cache?: ModuleResolutionCache, redirectedReference?: ResolvedProjectReference, lookupConfig?: boolean): ResolvedModuleWithFailedLookupLocations; // eslint-disable-line @typescript-eslint/unified-signatures
export function nodeModuleNameResolver(moduleName: string, containingFile: string, compilerOptions: CompilerOptions, host: ModuleResolutionHost, cache?: ModuleResolutionCache, redirectedReference?: ResolvedProjectReference, lookupConfig?: boolean): ResolvedModuleWithFailedLookupLocations {
return nodeModuleNameResolverWorker(moduleName, getDirectoryPath(containingFile), compilerOptions, host, cache, lookupConfig ? tsconfigExtensions : (compilerOptions.resolveJsonModule ? tsPlusJsonExtensions : tsExtensions), redirectedReference);
return nodeModuleNameResolverWorker(moduleName, getDirectoryPath(containingFile), compilerOptions, host, cache, lookupConfig ? tsconfigExtensions : [...tsExtensions, ...(compilerOptions.resolveJsonModule ? [Extensions.Json] : []), ...(compilerOptions.experimentalWasmModules ? [Extensions.Wasm] : [])], redirectedReference);
}

function nodeModuleNameResolverWorker(moduleName: string, containingDirectory: string, compilerOptions: CompilerOptions, host: ModuleResolutionHost, cache: ModuleResolutionCache | undefined, extensions: Extensions[], redirectedReference: ResolvedProjectReference | undefined): ResolvedModuleWithFailedLookupLocations {
Expand Down Expand Up @@ -1057,9 +1057,9 @@ namespace ts {
* in cases when we know upfront that all load attempts will fail (because containing folder does not exists) however we still need to record all failed lookup locations.
*/
function loadModuleFromFile(extensions: Extensions, candidate: string, onlyRecordFailures: boolean, state: ModuleResolutionState): PathAndExtension | undefined {
if (extensions === Extensions.Json || extensions === Extensions.TSConfig) {
const extensionLess = tryRemoveExtension(candidate, Extension.Json);
return (extensionLess === undefined && extensions === Extensions.Json) ? undefined : tryAddingExtensions(extensionLess || candidate, extensions, onlyRecordFailures, state);
if (extensions === Extensions.Json || extensions === Extensions.TSConfig || extensions === Extensions.Wasm) {
const extensionLess = tryRemoveExtension(candidate, Extension.Json) || tryRemoveExtension(candidate, Extension.Wasm);
return (extensionLess === undefined && (extensions === Extensions.Json || extensions === Extensions.Wasm)) ? undefined : tryAddingExtensions(extensionLess || candidate, extensions, onlyRecordFailures, state);
}

// First, try adding an extension. An import of "foo" could be matched by a file "foo.ts", or "foo.js" by "foo.js.ts"
Expand Down Expand Up @@ -1100,6 +1100,8 @@ namespace ts {
case Extensions.TSConfig:
case Extensions.Json:
return tryExtension(Extension.Json);
case Extensions.Wasm:
return tryExtension(Extension.Wasm);
}

function tryExtension(ext: Extension): PathAndExtension | undefined {
Expand Down Expand Up @@ -1168,6 +1170,7 @@ namespace ts {
switch (extensions) {
case Extensions.JavaScript:
case Extensions.Json:
case Extensions.Wasm:
packageFile = readPackageJsonMainField(jsonContent, candidate, state);
break;
case Extensions.TypeScript:
Expand Down Expand Up @@ -1243,6 +1246,8 @@ namespace ts {
return extension === Extension.Ts || extension === Extension.Tsx || extension === Extension.Dts;
case Extensions.DtsOnly:
return extension === Extension.Dts;
case Extensions.Wasm:
return extension === Extension.Wasm;
}
}

Expand Down
1 change: 1 addition & 0 deletions src/compiler/moduleSpecifiers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,7 @@ namespace ts.moduleSpecifiers {
case Extension.Js:
case Extension.Jsx:
case Extension.Json:
case Extension.Wasm:
return ext;
case Extension.TsBuildInfo:
return Debug.fail(`Extension ${Extension.TsBuildInfo} is unsupported:: FileName:: ${fileName}`);
Expand Down
14 changes: 12 additions & 2 deletions src/compiler/parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -603,7 +603,7 @@ namespace ts {
}
}

export function createSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, setParentNodes = false, scriptKind?: ScriptKind): SourceFile {
export function createSourceFile(fileName: string, sourceText: string | Uint8Array, languageVersion: ScriptTarget, setParentNodes = false, scriptKind?: ScriptKind): SourceFile {
performance.mark("beforeParse");
let result: SourceFile;

Expand Down Expand Up @@ -810,9 +810,10 @@ namespace ts {
// attached to the EOF token.
let parseErrorBeforeNextFinishedNode = false;

export function parseSourceFile(fileName: string, sourceText: string, languageVersion: ScriptTarget, syntaxCursor: IncrementalParser.SyntaxCursor | undefined, setParentNodes = false, scriptKind?: ScriptKind): SourceFile {
export function parseSourceFile(fileName: string, sourceText: string | Uint8Array, languageVersion: ScriptTarget, syntaxCursor: IncrementalParser.SyntaxCursor | undefined, setParentNodes = false, scriptKind?: ScriptKind): SourceFile {
scriptKind = ensureScriptKind(fileName, scriptKind);
if (scriptKind === ScriptKind.JSON) {
Debug.assert(isString(sourceText));
const result = parseJsonText(fileName, sourceText, languageVersion, syntaxCursor, setParentNodes);
convertToObjectWorker(result, result.parseDiagnostics, /*returnValue*/ false, /*knownRootOptions*/ undefined, /*jsonConversionNotifier*/ undefined);
result.referencedFiles = emptyArray;
Expand All @@ -823,7 +824,16 @@ namespace ts {
result.pragmas = emptyMap as ReadonlyPragmaMap;
return result;
}
else if (scriptKind === ScriptKind.Wasm) {
Debug.assert(!isString(sourceText), `Got string source for wasm module ${fileName}`);
const result = wasm.declarationsFor(wasm.parse(fileName, sourceText));
if (setParentNodes) {
fixupParentReferences(result);
}
return result;
}

Debug.assert(isString(sourceText), `Got nonstring source for file ${fileName}`);
initializeState(fileName, sourceText, languageVersion, syntaxCursor, scriptKind);

const result = parseSourceFileWorker(languageVersion, setParentNodes, scriptKind);
Expand Down
14 changes: 11 additions & 3 deletions src/compiler/program.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,10 @@ namespace ts {
const existingDirectories = new Map<string, boolean>();
const getCanonicalFileName = createGetCanonicalFileName(system.useCaseSensitiveFileNames);
function getSourceFile(fileName: string, languageVersion: ScriptTarget, onError?: (message: string) => void): SourceFile | undefined {
let text: string | undefined;
let text: string | Uint8Array | undefined;
try {
performance.mark("beforeIORead");
text = compilerHost.readFile(fileName);
text = endsWith(fileName, ".wasm") ? compilerHost.readFileBuffer(fileName) : compilerHost.readFile(fileName);
performance.mark("afterIORead");
performance.measure("I/O Read", "beforeIORead", "afterIORead");
}
Expand Down Expand Up @@ -179,6 +179,7 @@ namespace ts {
getNewLine: () => newLine,
fileExists: fileName => system.fileExists(fileName),
readFile: fileName => system.readFile(fileName),
readFileBuffer: fileName => system.readFileBuffer(fileName),
trace: (s: string) => system.write(s + newLine),
directoryExists: directoryName => system.directoryExists(directoryName),
getEnvironmentVariable: name => system.getEnvironmentVariable ? system.getEnvironmentVariable(name) : "",
Expand Down Expand Up @@ -2843,7 +2844,7 @@ namespace ts {
}

const isFromNodeModulesSearch = resolution.isExternalLibraryImport;
const isJsFile = !resolutionExtensionIsTSOrJson(resolution.extension);
const isJsFile = !resolutionExtensionIsTSOrJson(resolution.extension) && resolution.extension !== Extension.Wasm;
const isJsFileFromNodeModules = isFromNodeModulesSearch && isJsFile;
const resolvedFileName = resolution.resolvedFileName;

Expand Down Expand Up @@ -3709,6 +3710,7 @@ namespace ts {
getCurrentDirectory(): string;
fileExists(fileName: string): boolean;
readFile(fileName: string): string | undefined;
readFileBuffer(path: string): Uint8Array | undefined;
readDirectory?(rootDir: string, extensions: readonly string[], excludes: readonly string[] | undefined, includes: readonly string[], depth?: number): string[];
trace?(s: string): void;
onUnRecoverableConfigFileDiagnostic?: DiagnosticReporter;
Expand All @@ -3723,6 +3725,7 @@ namespace ts {
return directoryStructureHost.readDirectory(root, extensions, excludes, includes, depth);
},
readFile: f => directoryStructureHost.readFile(f),
readFileBuffer: f => directoryStructureHost.readFileBuffer(f),
useCaseSensitiveFileNames: host.useCaseSensitiveFileNames(),
getCurrentDirectory: () => host.getCurrentDirectory(),
onUnRecoverableConfigFileDiagnostic: host.onUnRecoverableConfigFileDiagnostic || returnUndefined,
Expand Down Expand Up @@ -3785,6 +3788,8 @@ namespace ts {
return needAllowJs();
case Extension.Json:
return needResolveJsonModule();
case Extension.Wasm:
return needExperimentalWasmModules();
}

function needJsx() {
Expand All @@ -3796,6 +3801,9 @@ namespace ts {
function needResolveJsonModule() {
return options.resolveJsonModule ? undefined : Diagnostics.Module_0_was_resolved_to_1_but_resolveJsonModule_is_not_used;
}
function needExperimentalWasmModules() {
return options.experimentalWasmModules ? undefined : Diagnostics.Module_0_was_resolved_to_1_but_experimentalWasmModules_is_not_used;
}
}

function getModuleNames({ imports, moduleAugmentations }: SourceFile): string[] {
Expand Down
31 changes: 23 additions & 8 deletions src/compiler/sys.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1050,6 +1050,7 @@ namespace ts {
write(s: string): void;
writeOutputIsTTY?(): boolean;
readFile(path: string, encoding?: string): string | undefined;
readFileBuffer(path: string): Uint8Array | undefined;
getFileSize?(path: string): number;
writeFile(path: string, data: string, writeByteOrderMark?: boolean): void;

Expand Down Expand Up @@ -1190,6 +1191,7 @@ namespace ts {
return process.stdout.isTTY;
},
readFile,
readFileBuffer,
writeFile,
watchFile,
watchDirectory,
Expand Down Expand Up @@ -1551,14 +1553,7 @@ namespace ts {
}
}

function readFileWorker(fileName: string, _encoding?: string): string | undefined {
let buffer: Buffer;
try {
buffer = _fs.readFileSync(fileName);
}
catch (e) {
return undefined;
}
function decodeBuffer(buffer: Buffer) {
let len = buffer.length;
if (len >= 2 && buffer[0] === 0xFE && buffer[1] === 0xFF) {
// Big endian UTF-16 byte order mark detected. Since big endian is not supported by node.js,
Expand All @@ -1583,13 +1578,33 @@ namespace ts {
return buffer.toString("utf8");
}

function readFileWorker(fileName: string, _encoding: undefined, bufferOutput: true): Buffer | undefined;
function readFileWorker(fileName: string, _encoding?: string): string | undefined;
function readFileWorker(fileName: string, _encoding?: string, bufferOutput?: true): string | Buffer | undefined {
let buffer: Buffer;
try {
buffer = _fs.readFileSync(fileName);
}
catch (e) {
return undefined;
}
return bufferOutput ? buffer : decodeBuffer(buffer);
}

function readFile(fileName: string, _encoding?: string): string | undefined {
perfLogger.logStartReadFile(fileName);
const file = readFileWorker(fileName, _encoding);
perfLogger.logStopReadFile();
return file;
}

function readFileBuffer(fileName: string): Uint8Array | undefined {
perfLogger.logStartReadFile(fileName);
const file = readFileWorker(fileName, /*encoding*/ undefined, /*buffer*/ true);
perfLogger.logStopReadFile();
return file;
}

function writeFile(fileName: string, data: string, writeByteOrderMark?: boolean): void {
perfLogger.logEvent("WriteFile: " + fileName);
// If a BOM is required, emit one
Expand Down
3 changes: 3 additions & 0 deletions src/compiler/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@
"commandLineParser.ts",
"moduleNameResolver.ts",

"wasm/parser.ts",
"wasm/transform.ts",

"binder.ts",
"symbolWalker.ts",
"checker.ts",
Expand Down
11 changes: 8 additions & 3 deletions src/compiler/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3624,6 +3624,7 @@ namespace ts {
fileExists(path: string): boolean;

readFile(path: string): string | undefined;
readFileBuffer(path: string): Uint8Array | undefined;
trace?(s: string): void;
}

Expand Down Expand Up @@ -5675,6 +5676,7 @@ namespace ts {
emitBOM?: boolean;
emitDecoratorMetadata?: boolean;
experimentalDecorators?: boolean;
experimentalWasmModules?: boolean;
forceConsistentCasingInFileNames?: boolean;
/*@internal*/generateCpuProfile?: string;
/*@internal*/help?: boolean;
Expand Down Expand Up @@ -5834,7 +5836,8 @@ namespace ts {
* Used on extensions that doesn't define the ScriptKind but the content defines it.
* Deferred extensions are going to be included in all project contexts.
*/
Deferred = 7
Deferred = 7,
Wasm = 8,
}

export const enum ScriptTarget {
Expand Down Expand Up @@ -6109,6 +6112,7 @@ namespace ts {
// readFile function is used to read arbitrary text files on disk, i.e. when resolution procedure needs the content of 'package.json'
// to determine location of bundled typings for node module
readFile(fileName: string): string | undefined;
readFileBuffer(fileName: string): Uint8Array | undefined;
trace?(s: string): void;
directoryExists?(directoryName: string): boolean;
/**
Expand Down Expand Up @@ -6177,7 +6181,8 @@ namespace ts {
Js = ".js",
Jsx = ".jsx",
Json = ".json",
TsBuildInfo = ".tsbuildinfo"
TsBuildInfo = ".tsbuildinfo",
Wasm = ".wasm",
}

export interface ResolvedModuleWithFailedLookupLocations {
Expand Down Expand Up @@ -7601,7 +7606,7 @@ namespace ts {
* });
* ```
*/
onEmitNode?(hint: EmitHint, node: Node | undefined, emitCallback: (hint: EmitHint, node: Node | undefined) => void): void;
onEmitNode?(hint: EmitHint, node: Node | undefined, emitCallback: (hint: EmitHint, node: Node | undefined) => void, getTextPos: () => number): void;

/**
* A hook used to check if an emit notification is required for a node.
Expand Down
Loading