Add a worker pool

src/swc/__tests__/options.test.ts

@@ -130,6 +130,32 @@ describe("parserArgs", () => {
       expect(mockExit).toHaveBeenCalledWith(2);
       expect(mockConsoleError).toHaveBeenCalledTimes(2);
     });
+
+    it("--workers exits on non-numeric values", async () => {
+      const args = [
+        "node",
+        "/path/to/node_modules/swc-cli/bin/swc.js",
+        "--workers",
+        "not-a-number",
+        "src",
+      ];
+      await parserArgs(args);
+      expect(mockExit).toHaveBeenCalledWith(2);
+      expect(mockConsoleError).toHaveBeenCalledTimes(2);
+    });
+
+    it("--workers exits on non-integer values", async () => {
+      const args = [
+        "node",
+        "/path/to/node_modules/swc-cli/bin/swc.js",
+        "--workers",
+        "1.5",
+        "src",
+      ];
+      await parserArgs(args);
+      expect(mockExit).toHaveBeenCalledWith(2);
+      expect(mockConsoleError).toHaveBeenCalledTimes(2);
+    });
   });
 
   describe("--source-maps", () => {

src/swc/dir.ts

@@ -1,10 +1,10 @@
-import slash from "slash";
 import { existsSync, promises } from "fs";
-import { dirname, relative, join } from "path";
+import { dirname, resolve } from "path";
+import Piscina from "piscina";
 import { CompileStatus } from "./constants";
 import { CliOptions } from "./options";
-import { compile, exists } from "./util";
-import { outputResult } from "./compile";
+import { exists, getDest } from "./util";
+import handleCompile from "./dirWorker";
 import {
   globSources,
   isCompilableExtension,
@@ -26,53 +26,8 @@ declare module "fs" {
 
 const { mkdir, rmdir, rm, copyFile, unlink } = promises;
 
-const cwd = process.cwd();
 const recursive = { recursive: true };
 
-/**
- * Removes the leading directory, including all parent relative paths
- */
-function stripComponents(filename: string) {
-  const components = filename.split("/").slice(1);
-  if (!components.length) {
-    return filename;
-  }
-  while (components[0] === "..") {
-    components.shift();
-  }
-  return components.join("/");
-}
-
-function getDest(filename: string, outDir: string, ext?: string) {
-  const relativePath = slash(relative(cwd, filename));
-  let base = stripComponents(relativePath);
-  if (ext) {
-    base = base.replace(/\.\w*$/, ext);
-  }
-  return join(outDir, base);
-}
-
-async function handleCompile(
-  filename: string,
-  outDir: string,
-  sync: boolean,
-  swcOptions: Options
-) {
-  const dest = getDest(filename, outDir, ".js");
-  const sourceFileName = slash(relative(dirname(dest), filename));
-
-  const options = { ...swcOptions, sourceFileName };
-
-  const result = await compile(filename, options, sync, dest);
-
-  if (result) {
-    await outputResult(result, filename, dest, options);
-    return CompileStatus.Compiled;
-  } else {
-    return CompileStatus.Omitted;
-  }
-}
-
 async function handleCopy(filename: string, outDir: string) {
   const dest = getDest(filename, outDir);
   const dir = dirname(dest);
@@ -126,7 +81,12 @@ async function initialCompilation(cliOptions: CliOptions, swcOptions: Options) {
   if (sync) {
     for (const filename of compilable) {
       try {
-        const result = await handleCompile(filename, outDir, sync, swcOptions);
+        const result = await handleCompile({
+          filename,
+          outDir,
+          sync,
+          swcOptions,
+        });
         results.set(filename, result);
       } catch (err: any) {
         console.error(err.message);
@@ -143,10 +103,16 @@ async function initialCompilation(cliOptions: CliOptions, swcOptions: Options) {
       }
     }
   } else {
+    const workers = new Piscina({
+      filename: resolve(__dirname, "./dirWorker.js"),
+      maxThreads: cliOptions.workers,
+      concurrentTasksPerWorker: 2,
+    });
+
     await Promise.all([
       Promise.allSettled(
-        compilable.map(file =>
-          handleCompile(file, outDir, sync, swcOptions).catch(err => {
+        compilable.map(filename =>
+          workers.run({ filename, outDir, sync, swcOptions }).catch(err => {
             console.error(err.message);
             throw err;
           })
@@ -264,12 +230,12 @@ async function watchCompilation(cliOptions: CliOptions, swcOptions: Options) {
       if (isCompilableExtension(filename, extensions)) {
         try {
           const start = process.hrtime();
-          const result = await handleCompile(
+          const result = await handleCompile({
             filename,
             outDir,
             sync,
-            swcOptions
-          );
+            swcOptions,
+          });
           if (!quiet && result === CompileStatus.Compiled) {
             const end = process.hrtime(start);
             console.log(

src/swc/dirWorker.ts

@@ -0,0 +1,28 @@
+import slash from "slash";
+import { dirname, relative } from "path";
+import { CompileStatus } from "./constants";
+import { compile, getDest } from "./util";
+import { outputResult } from "./compile";
+
+import type { Options } from "@swc/core";
+
+export default async function handleCompile(opts: {
+  filename: string;
+  outDir: string;
+  sync: boolean;
+  swcOptions: Options;
+}) {
+  const dest = getDest(opts.filename, opts.outDir, ".js");
+  const sourceFileName = slash(relative(dirname(dest), opts.filename));
+
+  const options = { ...opts.swcOptions, sourceFileName };
+
+  const result = await compile(opts.filename, options, opts.sync, dest);
+
+  if (result) {
+    await outputResult(result, opts.filename, dest, options);
+    return CompileStatus.Compiled;
+  } else {
+    return CompileStatus.Omitted;
+  }
+}

src/swc/options.ts

@@ -102,6 +102,11 @@ export const initProgram = () => {
     collect
   );
 
+  program.option(
+    "--workers [number]",
+    "The number of workers to use for parallel processing"
+  );
+
   program.option(
     "--log-watch-compilation",
     "Log a message when a watched file is successfully compiled",
@@ -157,6 +162,7 @@ export interface CliOptions {
    * Invoke swc using transformSync. It's useful for debugging.
    */
   readonly sync: boolean;
+  readonly workers: number | undefined;
   readonly sourceMapTarget?: string;
   readonly filename: string;
   readonly filenames: string[];
@@ -207,6 +213,16 @@ export default function parserArgs(args: string[]) {
     );
   }
 
+  let workers: number | undefined;
+  if (opts.workers != null) {
+    workers = parseFloat(opts.workers);
+    if (!Number.isInteger(workers) || workers < 0) {
+      errors.push(
+        "--workers must be a positive integer (found " + opts.workers + ")"
+      );
+    }
+  }
+
   if (errors.length) {
     console.error("swc:");
     for (const error of errors) {
@@ -265,6 +281,7 @@ export default function parserArgs(args: string[]) {
     filename: opts.filename,
     filenames,
     sync: !!opts.sync,
+    workers,
     sourceMapTarget: opts.sourceMapTarget,
     extensions: opts.extensions || DEFAULT_EXTENSIONS,
     watch: !!opts.watch,

src/swc/util.ts

@@ -1,7 +1,7 @@
 import * as swc from "@swc/core";
 import slash from "slash";
 import { mkdirSync, writeFileSync, promises } from "fs";
-import { dirname, relative } from "path";
+import { dirname, join, relative } from "path";
 
 export async function exists(path: string): Promise<boolean> {
   let pathExists = true;
@@ -125,3 +125,28 @@ export function assertCompilationResult<T>(
     );
   }
 }
+
+/**
+ * Removes the leading directory, including all parent relative paths
+ */
+function stripComponents(filename: string) {
+  const components = filename.split("/").slice(1);
+  if (!components.length) {
+    return filename;
+  }
+  while (components[0] === "..") {
+    components.shift();
+  }
+  return components.join("/");
+}
+
+const cwd = process.cwd();
+
+export function getDest(filename: string, outDir: string, ext?: string) {
+  const relativePath = slash(relative(cwd, filename));
+  let base = stripComponents(relativePath);
+  if (ext) {
+    base = base.replace(/\.\w*$/, ext);
+  }
+  return join(outDir, base);
+}