Merge branch 'master' of https://github.com/rollup/rollup into sync-f…

…bf806ac

.github/workflows/build-and-tests.yml

@@ -179,7 +179,6 @@ jobs:
         with:
           toolchain: nightly-2023-10-05
           targets: ${{ matrix.settings.target }}
-          components: rustfmt, clippy
       - name: Cache cargo
         uses: actions/cache@v3
         with:

.github/workflows/repl-artefacts.yml

@@ -28,7 +28,6 @@ jobs:
         with:
           toolchain: nightly-2023-10-05
           targets: x86_64-unknown-linux-gnu
-          components: rustfmt, clippy
       - name: Cache cargo
         uses: actions/cache@v3
         with:

CHANGELOG.md

@@ -1,5 +1,31 @@
 # rollup changelog
 
+## 4.2.0
+
+_2023-10-31_
+
+### Features
+
+- Run parsing in multiple threads and introduce `parseAstAsync` helper function (#5202)
+
+### Pull Requests
+
+- [#5202](https://github.com/rollup/rollup/pull/5202): perf: introduce `parseAstAsync` and parallelize parsing AST (@sapphi-red)
+
+## 4.1.6
+
+_2023-10-31_
+
+### Bug Fixes
+
+- Fix a bug where emtpy block comments in certain positions could freeze Rollup (#5231)
+
+### Pull Requests
+
+- [#5228](https://github.com/rollup/rollup/pull/5228): build: ensure rust toolchain components for linting are installed (@jerome-benoit)
+- [#5231](https://github.com/rollup/rollup/pull/5231): Render emtpy block comments after tree-shaken statements (@lukastaegert)
+- [#5232](https://github.com/rollup/rollup/pull/5232): Revert specifying rustfmt and clippy in toolchain file as it breaks REPL build (@lukastaegert)
+
 ## 4.1.5
 
 _2023-10-28_

browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rollup/browser",
-  "version": "4.1.5",
+  "version": "4.2.0",
   "description": "Next-generation ES module bundler browser build",
   "main": "dist/rollup.browser.js",
   "module": "dist/es/rollup.browser.js",

browser/src/wasm.ts

@@ -1,2 +1,12 @@
 // eslint-disable-next-line import/no-unresolved
 export { parse, xxhashBase64Url } from '../../wasm/bindings_wasm.js';
+
+// eslint-disable-next-line import/no-unresolved
+import { parse } from '../../wasm/bindings_wasm.js';
+export async function parseAsync(
+	code: string,
+	allowReturnOutsideFunction: boolean,
+	_signal?: AbortSignal | undefined | null
+) {
+	return parse(code, allowReturnOutsideFunction);
+}

docs/javascript-api/index.md

@@ -369,3 +369,34 @@ assert.deepEqual(
 	}
 );
 ```
+
+There is also an asynchronous version that parses in a different thread in the non-wasm builds of Rollup:
+
+```js
+import { parseAstAsync } from 'rollup/parseAst';
+import assert from 'node:assert';
+
+assert.deepEqual(
+	await parseAstAsync('return 42;', { allowReturnOutsideFunction: true }),
+	{
+		type: 'Program',
+		start: 0,
+		end: 10,
+		body: [
+			{
+				type: 'ReturnStatement',
+				start: 0,
+				end: 10,
+				argument: {
+					type: 'Literal',
+					start: 7,
+					end: 9,
+					raw: '42',
+					value: 42
+				}
+			}
+		],
+		sourceType: 'module'
+	}
+);
+```

native.d.ts

@@ -4,4 +4,5 @@
 /* auto-generated by NAPI-RS */
 
 export function parse(code: string, allowReturnOutsideFunction: boolean): Buffer
+export function parseAsync(code: string, allowReturnOutsideFunction: boolean, signal?: AbortSignal | undefined | null): Promise<Buffer>
 export function xxhashBase64Url(input: Uint8Array): string

native.js

@@ -46,9 +46,10 @@ If this is important to you, please consider supporting Rollup to make a native
 
 const packageBase = imported.musl && isMusl() ? imported.musl : imported.base;
 const localName = `./rollup.${packageBase}.node`;
-const { parse, xxhashBase64Url } = require(
+const { parse, parseAsync, xxhashBase64Url } = require(
 	existsSync(join(__dirname, localName)) ? localName : `@rollup/rollup-${packageBase}`
 );
 
 module.exports.parse = parse;
+module.exports.parseAsync = parseAsync;
 module.exports.xxhashBase64Url = xxhashBase64Url;

native.wasm.js

@@ -1,4 +1,6 @@
 const { parse, xxhashBase64Url } = require('./wasm-node/bindings_wasm.js');
 
 exports.parse = parse;
+exports.parseAsync = async (code, allowReturnOutsideFunction, _signal) =>
+	parse(code, allowReturnOutsideFunction);
 exports.xxhashBase64Url = xxhashBase64Url;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rollup",
-  "version": "4.1.5",
+  "version": "4.2.0",
   "description": "Next-generation ES module bundler",
   "main": "dist/rollup.js",
   "module": "dist/es/rollup.js",

rust/bindings_napi/src/lib.rs

@@ -9,11 +9,45 @@ use parse_ast::parse_ast;
 #[global_allocator]
 static ALLOC: mimalloc_rust::GlobalMiMalloc = mimalloc_rust::GlobalMiMalloc;
 
+pub struct ParseTask {
+  pub code: String,
+  pub allow_return_outside_function: bool,
+}
+
+#[napi]
+impl Task for ParseTask {
+  type Output = Buffer;
+  type JsValue = Buffer;
+
+  fn compute(&mut self) -> Result<Self::Output> {
+    Ok(parse_ast(self.code.clone(), self.allow_return_outside_function).into())
+  }
+
+  fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
+    Ok(output)
+  }
+}
+
 #[napi]
 pub fn parse(code: String, allow_return_outside_function: bool) -> Buffer {
   parse_ast(code, allow_return_outside_function).into()
 }
 
+#[napi]
+pub fn parse_async(
+  code: String,
+  allow_return_outside_function: bool,
+  signal: Option<AbortSignal>,
+) -> AsyncTask<ParseTask> {
+  AsyncTask::with_optional_signal(
+    ParseTask {
+      code,
+      allow_return_outside_function,
+    },
+    signal,
+  )
+}
+
 #[napi]
 pub fn xxhash_base64_url(input: Uint8Array) -> String {
   xxhash::xxhash_base64_url(&input)

src/Module.ts

@@ -68,7 +68,7 @@ import {
 	logShimmedExport,
 	logSyntheticNamedExportsNeedNamespaceExport
 } from './utils/logs';
-import { parseAst } from './utils/parseAst';
+import { parseAst, parseAstAsync } from './utils/parseAst';
 import {
 	doAttributesDiffer,
 	getAttributesFromImportExportDeclaration
@@ -785,7 +785,7 @@ export default class Module {
 		return { source, usesTopLevelAwait };
 	}
 
-	setSource({
+	async setSource({
 		ast,
 		code,
 		customTransformCache,
@@ -799,7 +799,7 @@ export default class Module {
 	}: TransformModuleJSON & {
 		resolvedIds?: ResolvedIdMap;
 		transformFiles?: EmittedFile[] | undefined;
-	}): void {
+	}): Promise<void> {
 		if (code.startsWith('#!')) {
 			const shebangEndPosition = code.indexOf('\n');
 			this.shebang = code.slice(2, shebangEndPosition);
@@ -831,7 +831,7 @@ export default class Module {
 		this.transformDependencies = transformDependencies;
 		this.customTransformCache = customTransformCache;
 		this.updateOptions(moduleOptions);
-		const moduleAst = ast ?? this.tryParse();
+		const moduleAst = ast ?? (await this.tryParseAsync());
 
 		timeEnd('generate ast', 3);
 		timeStart('analyze ast', 3);
@@ -1334,6 +1334,14 @@ export default class Module {
 			return this.error(logModuleParseError(error_, this.id), error_.pos);
 		}
 	}
+
+	private async tryParseAsync(): Promise<ProgramAst> {
+		try {
+			return (await parseAstAsync(this.info.code!)) as ProgramAst;
+		} catch (error_: any) {
+			return this.error(logModuleParseError(error_, this.id), error_.pos);
+		}
+	}
 }
 
 // if there is a cyclic import in the reexport chain, we should not

src/ModuleLoader.ts

@@ -317,10 +317,10 @@ export class ModuleLoader {
 				for (const emittedFile of cachedModule.transformFiles)
 					this.pluginDriver.emitFile(emittedFile);
 			}
-			module.setSource(cachedModule);
+			await module.setSource(cachedModule);
 		} else {
 			module.updateOptions(sourceDescription);
-			module.setSource(
+			await module.setSource(
 				await transform(sourceDescription, module, this.pluginDriver, this.options.onLog)
 			);
 		}

src/rollup/types.d.ts

@@ -206,6 +206,16 @@ export type ParseAst = (
 	options?: { allowReturnOutsideFunction?: boolean }
 ) => AstNode;
 
+// declare AbortSignal here for environments without DOM lib or @types/node
+declare global {
+	interface AbortSignal {}
+}
+
+export type ParseAstAsync = (
+	input: string,
+	options?: { allowReturnOutsideFunction?: boolean; signal?: AbortSignal }
+) => Promise<AstNode>;
+
 export interface PluginContext extends MinimalPluginContext {
 	addWatchFile: (id: string) => void;
 	cache: PluginCache;

src/utils/logs.ts

@@ -824,7 +824,8 @@ export function logModuleParseError(error: Error, moduleId: string): RollupLog {
 		cause: error,
 		code: PARSE_ERROR,
 		id: moduleId,
-		message
+		message,
+		stack: error.stack
 	};
 }
 

src/utils/parseAst.ts

@@ -1,10 +1,21 @@
-import { parse } from '../../native';
-import type { ParseAst } from '../rollup/types';
+import { parse, parseAsync } from '../../native';
+import type { ParseAst, ParseAstAsync } from '../rollup/types';
 import { convertProgram } from './convert-ast';
 import getReadStringFunction from './getReadStringFunction';
 
 export const parseAst: ParseAst = (input, { allowReturnOutsideFunction = false } = {}) => {
 	const astBuffer = parse(input, allowReturnOutsideFunction);
 	const readString = getReadStringFunction(astBuffer);
-	return convertProgram(astBuffer.buffer, readString);
+	const result = convertProgram(astBuffer.buffer, readString);
+	return result;
+};
+
+export const parseAstAsync: ParseAstAsync = async (
+	input,
+	{ allowReturnOutsideFunction = false, signal } = {}
+) => {
+	const astBuffer = await parseAsync(input, allowReturnOutsideFunction, signal);
+	const readString = getReadStringFunction(astBuffer);
+	const result = convertProgram(astBuffer.buffer, readString);
+	return result;
 };

src/utils/parseAstType.d.ts

@@ -1,3 +1,4 @@
-import type { ParseAst } from '../rollup/types';
+import type { ParseAst, ParseAstAsync } from '../rollup/types';
 
 export const parseAst: ParseAst;
+export const parseAstAsync: ParseAstAsync;

src/utils/renderHelpers.ts

@@ -82,7 +82,7 @@ export function findFirstLineBreakOutsideComment(code: string): [number, number]
 		// With our assumption, '/' always starts a comment. Determine comment type:
 		charCodeAfterSlash = code.charCodeAt(start + 1);
 		if (charCodeAfterSlash === 47 /*"/"*/) return [start, lineBreakPos + 1];
-		start = code.indexOf('*/', start + 3) + 2;
+		start = code.indexOf('*/', start + 2) + 2;
 		if (start > lineBreakPos) {
 			lineBreakPos = code.indexOf('\n', start);
 		}

test/form/samples/empty-block-comment/_config.js

@@ -0,0 +1,3 @@
+module.exports = defineTest({
+	description: 'renders empty block comments after tree-shaken statements (#5230)'
+});

test/form/samples/empty-block-comment/_expected.js

@@ -0,0 +1,4 @@
+function instance() {
+}
+
+export { instance };

test/form/samples/empty-block-comment/main.js

@@ -0,0 +1,3 @@
+export function instance() {
+		const b = 2; /**/
+}

test/misc/parse-ast.js

@@ -1,5 +1,5 @@
 const assert = require('node:assert');
-const { parseAst } = require('../../dist/parseAst');
+const { parseAst, parseAstAsync } = require('../../dist/parseAst');
 
 describe('parseAst', () => {
 	it('parses an AST', async () => {
@@ -109,3 +109,37 @@ describe('parseAst', () => {
 		assert.ok(key !== value);
 	});
 });
+
+describe('parseAstAsync', () => {
+	it('parses an AST', async () => {
+		assert.deepStrictEqual(await parseAstAsync('console.log("ok")'), {
+			type: 'Program',
+			start: 0,
+			end: 17,
+			body: [
+				{
+					type: 'ExpressionStatement',
+					start: 0,
+					end: 17,
+					expression: {
+						type: 'CallExpression',
+						start: 0,
+						end: 17,
+						arguments: [{ type: 'Literal', start: 12, end: 16, raw: '"ok"', value: 'ok' }],
+						callee: {
+							type: 'MemberExpression',
+							start: 0,
+							end: 11,
+							computed: false,
+							object: { type: 'Identifier', start: 0, end: 7, name: 'console' },
+							optional: false,
+							property: { type: 'Identifier', start: 8, end: 11, name: 'log' }
+						},
+						optional: false
+					}
+				}
+			],
+			sourceType: 'module'
+		});
+	});
+});