feat(core): expose opt-in env variable for SSG thread recycling (#11166)

slorber · web-flow · commit 8061f2267be8 · 2025-05-09T17:40:58.000+02:00
diff --git a/packages/docusaurus/src/ssg/ssgEnv.ts b/packages/docusaurus/src/ssg/ssgEnv.ts
@@ -27,3 +27,14 @@ export const SSGWorkerThreadTaskSize: number = process.env
   .DOCUSAURUS_SSG_WORKER_THREAD_TASK_SIZE
   ? parseInt(process.env.DOCUSAURUS_SSG_WORKER_THREAD_TASK_SIZE, 10)
   : 10; // TODO need fine-tuning
+
+// Controls worker thread recycling behavior (maxMemoryLimitBeforeRecycle)
+// See https://github.com/facebook/docusaurus/pull/11166
+// See https://github.com/facebook/docusaurus/issues/11161
+export const SSGWorkerThreadRecyclerMaxMemory: number | undefined = process.env
+  .DOCUSAURUS_SSG_WORKER_THREAD_RECYCLER_MAX_MEMORY
+  ? parseInt(process.env.DOCUSAURUS_SSG_WORKER_THREAD_RECYCLER_MAX_MEMORY, 10)
+  : // TODO we should probably provide a default value here
+    //  2gb is a quite reasonable max that should work well even for large sites
+    //  we'd rather ask community feedback first
+    undefined;
diff --git a/packages/docusaurus/src/ssg/ssgExecutor.ts b/packages/docusaurus/src/ssg/ssgExecutor.ts
@@ -12,7 +12,11 @@ import _ from 'lodash';
 import logger, {PerfLogger} from '@docusaurus/logger';
 import {createSSGParams} from './ssgParams';
 import {renderHashRouterTemplate} from './ssgTemplate';
-import {SSGWorkerThreadCount, SSGWorkerThreadTaskSize} from './ssgEnv';
+import {
+  SSGWorkerThreadCount,
+  SSGWorkerThreadRecyclerMaxMemory,
+  SSGWorkerThreadTaskSize,
+} from './ssgEnv';
 import {generateHashRouterEntrypoint} from './ssgUtils';
 import {createGlobalSSGResult} from './ssgGlobalResult';
 import {executeSSGInlineTask} from './ssgWorkerInline';
@@ -124,6 +128,16 @@ const createPooledSSGExecutor: CreateSSGExecutor = async ({
         runtime: 'worker_threads',
         isolateWorkers: false,
         workerData: {params},
+
+        // WORKER MEMORY MANAGEMENT
+        // Allows containing SSG memory leaks with a thread recycling workaround
+        // See https://github.com/facebook/docusaurus/pull/11166
+        // See https://github.com/facebook/docusaurus/issues/11161
+        maxMemoryLimitBeforeRecycle: SSGWorkerThreadRecyclerMaxMemory,
+        resourceLimits: {
+          // For some reason I can't figure out how to limit memory on a worker
+          // See https://x.com/sebastienlorber/status/1920781195618513143
+        },
       });
     },
   );
