fix(core): fix i18n sites SSG memory leak - require.cache (facebook#1…

…0599)

.gitignore

@@ -14,7 +14,7 @@ package-lock.json
 .eslintcache
 
 yarn-error.log
-build
+website/build
 coverage
 .docusaurus
 .cache-loader

packages/docusaurus/src/client/serverEntry.tsx

@@ -18,7 +18,7 @@ import {
 } from './BrokenLinksContext';
 import type {PageCollectedData, AppRenderer} from '../common';
 
-const render: AppRenderer = async ({pathname}) => {
+const render: AppRenderer['render'] = async ({pathname}) => {
   await preload(pathname);
 
   const modules = new Set<string>();

packages/docusaurus/src/commands/build/build.ts

@@ -0,0 +1,122 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import fs from 'fs-extra';
+import logger, {PerfLogger} from '@docusaurus/logger';
+import {mapAsyncSequential} from '@docusaurus/utils';
+import {loadContext, type LoadContextParams} from '../../server/site';
+import {loadI18n} from '../../server/i18n';
+import {buildLocale, type BuildLocaleParams} from './buildLocale';
+
+export type BuildCLIOptions = Pick<
+  LoadContextParams,
+  'config' | 'locale' | 'outDir'
+> & {
+  bundleAnalyzer?: boolean;
+  minify?: boolean;
+  dev?: boolean;
+};
+
+export async function build(
+  siteDirParam: string = '.',
+  cliOptions: Partial<BuildCLIOptions> = {},
+): Promise<void> {
+  process.env.BABEL_ENV = 'production';
+  process.env.NODE_ENV = 'production';
+  process.env.DOCUSAURUS_CURRENT_LOCALE = cliOptions.locale;
+  if (cliOptions.dev) {
+    logger.info`Building in dev mode`;
+    process.env.BABEL_ENV = 'development';
+    process.env.NODE_ENV = 'development';
+  }
+
+  const siteDir = await fs.realpath(siteDirParam);
+
+  ['SIGINT', 'SIGTERM'].forEach((sig) => {
+    process.on(sig, () => process.exit());
+  });
+
+  const locales = await PerfLogger.async('Get locales to build', () =>
+    getLocalesToBuild({siteDir, cliOptions}),
+  );
+
+  if (locales.length > 1) {
+    logger.info`Website will be built for all these locales: ${locales}`;
+  }
+
+  await PerfLogger.async(`Build`, () =>
+    mapAsyncSequential(locales, async (locale) => {
+      await tryToBuildLocale({siteDir, locale, cliOptions});
+    }),
+  );
+
+  logger.info`Use code=${'npm run serve'} command to test your build locally.`;
+}
+
+async function getLocalesToBuild({
+  siteDir,
+  cliOptions,
+}: {
+  siteDir: string;
+  cliOptions: BuildCLIOptions;
+}): Promise<[string, ...string[]]> {
+  if (cliOptions.locale) {
+    return [cliOptions.locale];
+  }
+
+  const context = await loadContext({
+    siteDir,
+    outDir: cliOptions.outDir,
+    config: cliOptions.config,
+    locale: cliOptions.locale,
+    localizePath: cliOptions.locale ? false : undefined,
+  });
+  const i18n = await loadI18n(context.siteConfig, {
+    locale: cliOptions.locale,
+  });
+  if (i18n.locales.length > 1) {
+    logger.info`Website will be built for all these locales: ${i18n.locales}`;
+  }
+
+  // We need the default locale to always be the 1st in the list. If we build it
+  // last, it would "erase" the localized sites built in sub-folders
+  return [
+    i18n.defaultLocale,
+    ...i18n.locales.filter((locale) => locale !== i18n.defaultLocale),
+  ];
+}
+
+async function tryToBuildLocale(params: BuildLocaleParams) {
+  try {
+    await PerfLogger.async(`${logger.name(params.locale)}`, async () => {
+      // Note: I tried to run buildLocale in worker_threads (still sequentially)
+      // It didn't work and I got SIGSEGV / SIGBUS errors
+      // See https://x.com/sebastienlorber/status/1848413716372480338
+      await runBuildLocaleTask(params);
+    });
+  } catch (err) {
+    throw new Error(
+      logger.interpolate`Unable to build website for locale name=${params.locale}.`,
+      {
+        cause: err,
+      },
+    );
+  }
+}
+
+async function runBuildLocaleTask(params: BuildLocaleParams) {
+  // Note: I tried to run buildLocale task in worker_threads (sequentially)
+  // It didn't work and I got SIGSEGV / SIGBUS errors
+  // Goal was to isolate memory of each localized site build
+  // See also https://x.com/sebastienlorber/status/1848413716372480338
+  //
+  // Running in child_process worked but is more complex and requires
+  // specifying the memory of the child process + weird logging issues to fix
+  //
+  // Note in the future we could try to enable concurrent localized site builds
+  await buildLocale(params);
+}

packages/docusaurus/src/commands/build.ts → packages/docusaurus/src/commands/build/buildLocale.ts

@@ -10,130 +10,34 @@ import path from 'path';
 import _ from 'lodash';
 import {compile} from '@docusaurus/bundler';
 import logger, {PerfLogger} from '@docusaurus/logger';
-import {mapAsyncSequential} from '@docusaurus/utils';
-import {loadSite, loadContext, type LoadContextParams} from '../server/site';
-import {handleBrokenLinks} from '../server/brokenLinks';
-import {createBuildClientConfig} from '../webpack/client';
-import createServerConfig from '../webpack/server';
+import {loadSite} from '../../server/site';
+import {handleBrokenLinks} from '../../server/brokenLinks';
+import {createBuildClientConfig} from '../../webpack/client';
+import createServerConfig from '../../webpack/server';
 import {
   createConfigureWebpackUtils,
   executePluginsConfigureWebpack,
-} from '../webpack/configure';
-import {loadI18n} from '../server/i18n';
-import {executeSSG} from '../ssg/ssgExecutor';
+} from '../../webpack/configure';
+import {executeSSG} from '../../ssg/ssgExecutor';
 import type {
   ConfigureWebpackUtils,
   LoadedPlugin,
   Props,
 } from '@docusaurus/types';
-import type {SiteCollectedData} from '../common';
+import type {SiteCollectedData} from '../../common';
+import {BuildCLIOptions} from './build';
 
-export type BuildCLIOptions = Pick<
-  LoadContextParams,
-  'config' | 'locale' | 'outDir'
-> & {
-  bundleAnalyzer?: boolean;
-  minify?: boolean;
-  dev?: boolean;
-};
-
-export async function build(
-  siteDirParam: string = '.',
-  cliOptions: Partial<BuildCLIOptions> = {},
-): Promise<void> {
-  process.env.BABEL_ENV = 'production';
-  process.env.NODE_ENV = 'production';
-  process.env.DOCUSAURUS_CURRENT_LOCALE = cliOptions.locale;
-  if (cliOptions.dev) {
-    logger.info`Building in dev mode`;
-    process.env.BABEL_ENV = 'development';
-    process.env.NODE_ENV = 'development';
-  }
-
-  const siteDir = await fs.realpath(siteDirParam);
-
-  ['SIGINT', 'SIGTERM'].forEach((sig) => {
-    process.on(sig, () => process.exit());
-  });
-
-  async function tryToBuildLocale({locale}: {locale: string}) {
-    try {
-      await PerfLogger.async(`${logger.name(locale)}`, () =>
-        buildLocale({
-          siteDir,
-          locale,
-          cliOptions,
-        }),
-      );
-    } catch (err) {
-      throw new Error(
-        logger.interpolate`Unable to build website for locale name=${locale}.`,
-        {
-          cause: err,
-        },
-      );
-    }
-  }
-
-  const locales = await PerfLogger.async('Get locales to build', () =>
-    getLocalesToBuild({siteDir, cliOptions}),
-  );
-
-  if (locales.length > 1) {
-    logger.info`Website will be built for all these locales: ${locales}`;
-  }
-
-  await PerfLogger.async(`Build`, () =>
-    mapAsyncSequential(locales, async (locale) => {
-      await tryToBuildLocale({locale});
-    }),
-  );
-
-  logger.info`Use code=${'npm run serve'} command to test your build locally.`;
-}
-
-async function getLocalesToBuild({
-  siteDir,
-  cliOptions,
-}: {
+export type BuildLocaleParams = {
   siteDir: string;
-  cliOptions: BuildCLIOptions;
-}): Promise<[string, ...string[]]> {
-  if (cliOptions.locale) {
-    return [cliOptions.locale];
-  }
-
-  const context = await loadContext({
-    siteDir,
-    outDir: cliOptions.outDir,
-    config: cliOptions.config,
-    locale: cliOptions.locale,
-    localizePath: cliOptions.locale ? false : undefined,
-  });
-  const i18n = await loadI18n(context.siteConfig, {
-    locale: cliOptions.locale,
-  });
-  if (i18n.locales.length > 1) {
-    logger.info`Website will be built for all these locales: ${i18n.locales}`;
-  }
-
-  // We need the default locale to always be the 1st in the list. If we build it
-  // last, it would "erase" the localized sites built in sub-folders
-  return [
-    i18n.defaultLocale,
-    ...i18n.locales.filter((locale) => locale !== i18n.defaultLocale),
-  ];
-}
+  locale: string;
+  cliOptions: Partial<BuildCLIOptions>;
+};
 
-async function buildLocale({
+export async function buildLocale({
   siteDir,
   locale,
   cliOptions,
-}: {
-  siteDir: string;
-  locale: string;
-  cliOptions: Partial<BuildCLIOptions>;
-}): Promise<void> {
+}: BuildLocaleParams): Promise<void> {
   // Temporary workaround to unlock the ability to translate the site config
   // We'll remove it if a better official API can be designed
   // See https://github.com/facebook/docusaurus/issues/4542

packages/docusaurus/src/commands/deploy.ts

@@ -12,7 +12,7 @@ import logger from '@docusaurus/logger';
 import shell from 'shelljs';
 import {hasSSHProtocol, buildSshUrl, buildHttpsUrl} from '@docusaurus/utils';
 import {loadContext, type LoadContextParams} from '../server/site';
-import {build} from './build';
+import {build} from './build/build';
 
 export type DeployCLIOptions = Pick<
   LoadContextParams,

packages/docusaurus/src/commands/serve.ts

@@ -14,7 +14,7 @@ import serveHandler from 'serve-handler';
 import openBrowser from 'react-dev-utils/openBrowser';
 import {applyTrailingSlash} from '@docusaurus/utils-common';
 import {loadSiteConfig} from '../server/config';
-import {build} from './build';
+import {build} from './build/build';
 import {getHostPort, type HostPortOptions} from '../server/getHostPort';
 import type {LoadContextParams} from '../server/site';
 

packages/docusaurus/src/common.d.ts

@@ -15,9 +15,13 @@ export type AppRenderResult = {
   collectedData: PageCollectedData;
 };
 
-export type AppRenderer = (params: {
-  pathname: string;
-}) => Promise<AppRenderResult>;
+export type AppRenderer = {
+  render: (params: {pathname: string}) => Promise<AppRenderResult>;
+
+  // It's important to shut down the app renderer
+  // Otherwise Node.js require cache leaks memory
+  shutdown: () => Promise<void>;
+};
 
 export type PageCollectedData = {
   // TODO Docusaurus v4 refactor: helmet state is non-serializable

packages/docusaurus/src/index.ts

@@ -5,7 +5,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-export {build} from './commands/build';
+export {build} from './commands/build/build';
 export {clear} from './commands/clear';
 export {deploy} from './commands/deploy';
 export {externalCommand} from './commands/external';