refactor: optimize bundling performance, pre-transpile client code so that sites don't have to

.gitignore

@@ -26,6 +26,7 @@ packages/create-docusaurus/lib/
 packages/lqip-loader/lib/
 packages/docusaurus/lib/
 packages/docusaurus-*/lib/*
+packages/docusaurus-*/lib-pretranspiled/*
 packages/eslint-plugin/lib/
 packages/stylelint-copyright/lib/
 

__tests__/validate-tsconfig.test.ts

@@ -34,13 +34,15 @@ const tsconfigSchema = Joi.object({
   extends: Joi.valid(
     '../../tsconfig.base.json',
     '../../tsconfig.base.client.json',
+    '../../tsconfig.base.client.pretranspiled.json',
   ),
   compilerOptions: Joi.object({
     rootDir: Joi.valid('src').required(),
-    outDir: Joi.valid('lib').required(),
+    outDir: Joi.valid('lib', 'lib-pretranspiled').required(),
     tsBuildInfoFile: Joi.valid(
       'lib/.tsbuildinfo',
       'lib/.tsbuildinfo-client',
+      'lib/.tsbuildinfo-client-pretranspiled',
       'lib/.tsbuildinfo-worker',
     ),
   }).unknown(),

admin/scripts/copyUntypedFiles.js

@@ -11,6 +11,7 @@ import chokidar from 'chokidar';
 
 const srcDir = path.join(process.cwd(), 'src');
 const libDir = path.join(process.cwd(), 'lib');
+const libPretranspiledDir = path.join(process.cwd(), 'lib-pretranspiled');
 
 const ignoredPattern = /(?:__tests__|\.tsx?$)/;
 
@@ -20,6 +21,14 @@ async function copy() {
       return !ignoredPattern.test(testedPath);
     },
   });
+
+  if (await fs.pathExists(libPretranspiledDir)) {
+    await fs.copy(srcDir, libPretranspiledDir, {
+      filter(testedPath) {
+        return !ignoredPattern.test(testedPath);
+      },
+    });
+  }
 }
 
 if (process.argv.includes('--watch')) {

packages/docusaurus-theme-classic/src/index.ts

@@ -140,7 +140,8 @@ export default function themeClassic(
     name: 'docusaurus-theme-classic',
 
     getThemePath() {
-      return '../lib/theme';
+      // TODO POC
+      return '../lib-pretranspiled/theme';
     },
 
     getTypeScriptThemePath() {

packages/docusaurus-theme-classic/src/theme/CodeBlock/CopyButton/index.tsx

@@ -7,19 +7,23 @@
 
 import React, {useCallback, useState, useRef, useEffect} from 'react';
 import clsx from 'clsx';
-import copy from 'copy-text-to-clipboard';
 import {translate} from '@docusaurus/Translate';
 import type {Props} from '@theme/CodeBlock/CopyButton';
 import IconCopy from '@theme/Icon/Copy';
 import IconSuccess from '@theme/Icon/Success';
 
 import styles from './styles.module.css';
 
+async function copyToClipboard(text: string) {
+  const copy = (await import('copy-text-to-clipboard')).default;
+  copy(text);
+}
+
 export default function CopyButton({code, className}: Props): JSX.Element {
   const [isCopied, setIsCopied] = useState(false);
   const copyTimeout = useRef<number | undefined>(undefined);
   const handleCopyCode = useCallback(() => {
-    copy(code);
+    copyToClipboard(code);
     setIsCopied(true);
     copyTimeout.current = window.setTimeout(() => {
       setIsCopied(false);

packages/docusaurus-theme-classic/tsconfig.client.pretranspiled.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.base.client.pretranspiled.json",
+  "compilerOptions": {
+    "rootDir": "src",
+    "outDir": "lib-pretranspiled",
+    "tsBuildInfoFile": "lib/.tsbuildinfo-client-pretranspiled"
+  },
+  "include": [
+    "src/nprogress.ts",
+    "src/prism-include-languages.ts",
+    "src/theme",
+    "src/*.d.ts"
+  ],
+  "exclude": ["**/__tests__/**"]
+}

packages/docusaurus-theme-classic/tsconfig.json

@@ -1,6 +1,9 @@
 {
   "extends": "../../tsconfig.base.json",
-  "references": [{"path": "./tsconfig.client.json"}],
+  "references": [
+    {"path": "./tsconfig.client.json"},
+    {"path": "./tsconfig.client.pretranspiled.json"}
+  ],
   "compilerOptions": {
     "noEmit": false,
     "incremental": true,

packages/docusaurus-theme-common/tsconfig.json

@@ -1,5 +1,5 @@
 {
-  "extends": "../../tsconfig.base.client.json",
+  "extends": "../../tsconfig.base.client.pretranspiled.json",
   "compilerOptions": {
     "rootDir": "src",
     "outDir": "lib",

packages/docusaurus-utils-common/tsconfig.json

@@ -1,14 +1,15 @@
 {
-  "extends": "../../tsconfig.base.json",
+  "extends": "../../tsconfig.base.client.json",
   "compilerOptions": {
-    "noEmit": false,
-    "incremental": true,
+    "rootDir": "src",
+    "outDir": "lib",
     "tsBuildInfoFile": "lib/.tsbuildinfo",
     "sourceMap": true,
     "declarationMap": true,
-    "rootDir": "src",
-    "outDir": "lib",
-    "noEmitHelpers": false
+    "importHelpers": true,
+    "noEmitHelpers": false,
+    "moduleResolution": "NodeNext",
+    "module": "NodeNext"
   },
   "include": ["src"],
   "exclude": ["**/__tests__/**"]

packages/docusaurus/src/webpack/__tests__/base.test.ts

@@ -10,19 +10,29 @@
 import _ from 'lodash';
 import * as utils from '@docusaurus/utils/lib/webpackUtils';
 import {posixPath} from '@docusaurus/utils';
-import {excludeJS, clientDir, createBaseConfig} from '../base';
+import {excludeJS, CoreClientDir, createBaseConfig} from '../base';
 import type {Props} from '@docusaurus/types';
 
 describe('babel transpilation exclude logic', () => {
-  it('always transpiles client dir files', () => {
+  it('never transpiles client dir files', () => {
     const clientFiles = [
       'App.js',
       'clientEntry.js',
       'serverEntry.js',
       path.join('exports', 'Link.js'),
     ];
     clientFiles.forEach((file) => {
-      expect(excludeJS(path.join(clientDir, file))).toBe(false);
+      expect(excludeJS(path.join(CoreClientDir, file))).toBe(true);
+    });
+  });
+
+  it('always transpiles copy-text-to-clipboard', () => {
+    const clientFiles = [
+      '/whatever/copy-text-to-clipboard/lib.js',
+      '/whatever/node_modules/copy-text-to-clipboard/lib.js',
+    ];
+    clientFiles.forEach((file) => {
+      expect(excludeJS(file)).toBe(false);
     });
   });
 
@@ -40,11 +50,11 @@
   it('transpiles docusaurus npm packages even in node_modules', () => {
     const moduleFiles = [
       '/website/node_modules/docusaurus-theme-search/theme/Navbar/index.js',
-      'node_modules/@docusaurus/theme-classic/theme/Layout.js',
+      'node_modules/@docusaurus/some-theme/lib/theme/Layout.js',
       '/docusaurus/website/node_modules/@docusaurus/theme-search-algolia/theme/SearchBar.js',
     ];
     moduleFiles.forEach((file) => {
       expect(excludeJS(file)).toBe(false);
     });
   });
 

packages/docusaurus/src/webpack/base.ts

@@ -21,27 +21,78 @@ import type {Props} from '@docusaurus/types';
 
 const CSS_REGEX = /\.css$/i;
 const CSS_MODULE_REGEX = /\.module\.css$/i;
-export const clientDir = path.join(__dirname, '..', 'client');
+
+export const CoreClientDir = path.join(__dirname, '..', 'client');
+function isCoreClientModule(modulePath: string): boolean {
+  return modulePath.startsWith(CoreClientDir);
+}
 
 const LibrariesToTranspile = [
   'copy-text-to-clipboard', // Contains optional catch binding, incompatible with recent versions of Edge
 ];
-
 const LibrariesToTranspileRegex = new RegExp(
   LibrariesToTranspile.map((libName) => `(node_modules/${libName})`).join('|'),
 );
+function isLibraryToTranspileModule(modulePath: string): boolean {
+  return LibrariesToTranspileRegex.test(modulePath);
+}
 
-export function excludeJS(modulePath: string): boolean {
-  // Always transpile client dir
-  if (modulePath.startsWith(clientDir)) {
+// Returns true when a node_module package has docusaurus in its name
+// Not super robust logic, but it's what we use historically
+// /node_modules/docusaurus-xyz/node_modules/other/index.js => false
+// /node_modules/other/node_modules/docusaurus-xyz/index.js => true
+function isInDocusaurusNamedPackage(modulePath: string): boolean {
+  return /docusaurus(?:(?!node_modules).)*\.jsx?$/.test(modulePath);
+}
+
+function shouldBeTranspiled(modulePath: string): boolean {
+  // Never transpile Docusaurus core client dir
+  // It's already pre-transpiled
+  if (isCoreClientModule(modulePath)) {
     return false;
   }
-  // Don't transpile node_modules except any docusaurus npm package
-  return (
-    modulePath.includes('node_modules') &&
-    !/docusaurus(?:(?!node_modules).)*\.jsx?$/.test(modulePath) &&
-    !LibrariesToTranspileRegex.test(modulePath)
-  );
+
+  if (isLibraryToTranspileModule(modulePath)) {
+    return true;
+  }
+
+  // A TS module should always be transpiled
+  if (modulePath.endsWith('.ts') || modulePath.endsWith('.tsx')) {
+    return true;
+  }
+
+  // TODO POC pretranspiled modules
+  if (
+    modulePath.includes('docusaurus-') ||
+    modulePath.includes('docusaurus-theme-classic') ||
+    modulePath.includes('@docusaurus/theme-classic') ||
+    modulePath.includes('docusaurus-theme-common') ||
+    modulePath.includes('@docusaurus/theme-common') ||
+    modulePath.includes('docusaurus-utils-common') ||
+    modulePath.includes('@docusaurus/utils-common')
+  ) {
+    return false;
+  }
+
+  if (modulePath.includes('node_modules')) {
+    // we only transpile Docusaurus-named packages
+    // If it's a JS/React package it's usually pre-transpiled
+    return isInDocusaurusNamedPackage(modulePath);
+  }
+  // if it's not a dependency, we always transpile it
+  else {
+    return true;
+  }
+}
+
+export function excludeJS(modulePath: string): boolean {
+  const excluded = !shouldBeTranspiled(modulePath);
+  /*
+  modulePath.includes('docusaurus-') &&
+    console.log('excluded?', excluded ? '✅' : '⚠️', modulePath);
+
+   */
+  return excluded;
 }
 
 export async function createBaseConfig({

packages/docusaurus/src/webpack/utils.ts

@@ -30,6 +30,7 @@ export function formatStatsErrorMessage(
   statsJson: ReturnType<webpack.Stats['toJson']> | undefined,
 ): string | undefined {
   if (statsJson?.errors?.length) {
+    console.log({statsJsonErrors: statsJson.errors});
     // TODO formatWebpackMessages does not print stack-traces
     // Also the error causal chain is lost here
     // We log the stacktrace inside serverEntry.tsx for now (not ideal)

packages/docusaurus/tsconfig.client.json

@@ -1,5 +1,5 @@
 {
-  "extends": "../../tsconfig.base.client.json",
+  "extends": "../../tsconfig.base.client.pretranspiled.json",
   "compilerOptions": {
     "rootDir": "src",
     "outDir": "lib",

tsconfig.base.client.json

@@ -6,6 +6,7 @@
     "incremental": true,
     "moduleResolution": "bundler",
     "module": "esnext",
-    "target": "esnext"
+    "target": "es2022",
+    "jsx": "react-jsx"
   }
 }

tsconfig.base.client.pretranspiled.json

@@ -0,0 +1,7 @@
+{
+  "extends": "./tsconfig.base.client.json",
+  "compilerOptions": {
+    "target": "es2022",
+    "jsx": "react-jsx"
+  }
+}