Skip to content

Commit 33811e3

Browse files
slorberslorber
authored
perf(core): optimize SSG collected data memory and worker thread communication (#11162)
1 parent 53fa0ec commit 33811e3

6 files changed

Lines changed: 105 additions & 46 deletions

File tree

‎packages/docusaurus-logger/src/perfLogger.ts‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,22 @@ function createPerfLogger(): PerfLoggerAPI {
7272
}
7373
};
7474

75-
const formatMemory = (memory: Memory): string => {
76-
const fmtHead = (bytes: number) =>
77-
logger.cyan(`${(bytes / 1000000).toFixed(0)}mb`);
75+
const formatBytesToMb = (bytes: number) =>
76+
logger.cyan(`${(bytes / 1024 / 1024).toFixed(0)}mb`);
77+
78+
const formatMemoryDelta = (memory: Memory): string => {
7879
return logger.dim(
79-
`(${fmtHead(memory.before.heapUsed)} -> ${fmtHead(
80+
`(Heap ${formatBytesToMb(memory.before.heapUsed)} -> ${formatBytesToMb(
8081
memory.after.heapUsed,
82+
)} / Total ${formatBytesToMb(memory.after.heapTotal)})`,
83+
);
84+
};
85+
86+
const formatMemoryCurrent = (): string => {
87+
const memory = getMemory();
88+
return logger.dim(
89+
`(Heap ${formatBytesToMb(memory.heapUsed)} / Total ${formatBytesToMb(
90+
memory.heapTotal,
8191
)})`,
8292
);
8393
};
@@ -103,7 +113,7 @@ function createPerfLogger(): PerfLoggerAPI {
103113
console.log(
104114
`${PerfPrefix}${formatStatus(error)} ${label} - ${formatDuration(
105115
duration,
106-
)} - ${formatMemory(memory)}`,
116+
)} - ${formatMemoryDelta(memory)}`,
107117
);
108118
};
109119

@@ -144,7 +154,9 @@ function createPerfLogger(): PerfLoggerAPI {
144154
};
145155

146156
const log: PerfLoggerAPI['log'] = (label: string) =>
147-
console.log(`${PerfPrefix} ${applyParentPrefix(label)}`);
157+
console.log(
158+
`${PerfPrefix} ${applyParentPrefix(label)} - ${formatMemoryCurrent()}`,
159+
);
148160

149161
const async: PerfLoggerAPI['async'] = async (label, asyncFn) => {
150162
const finalLabel = applyParentPrefix(label);

‎packages/docusaurus/src/client/serverEntry.tsx‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ import {
1616
createStatefulBrokenLinks,
1717
BrokenLinksProvider,
1818
} from './BrokenLinksContext';
19-
import {toPageCollectedMetadata} from './serverHelmetUtils';
20-
import type {PageCollectedData, AppRenderer} from '../common';
19+
import {toPageCollectedMetadataInternal} from './serverHelmetUtils';
20+
import type {AppRenderer, PageCollectedDataInternal} from '../common';
2121

2222
const render: AppRenderer['render'] = async ({
2323
pathname,
@@ -47,15 +47,15 @@ const render: AppRenderer['render'] = async ({
4747

4848
const {helmet} = helmetContext as FilledContext;
4949

50-
const metadata = toPageCollectedMetadata({helmet});
50+
const metadata = toPageCollectedMetadataInternal({helmet});
5151

5252
// TODO Docusaurus v4 remove with deprecated postBuild({head}) API
5353
// the returned collectedData must be serializable to run in workers
5454
if (v4RemoveLegacyPostBuildHeadAttribute) {
5555
metadata.helmet = null;
5656
}
5757

58-
const collectedData: PageCollectedData = {
58+
const collectedData: PageCollectedDataInternal = {
5959
metadata,
6060
anchors: statefulBrokenLinks.getCollectedAnchors(),
6161
links: statefulBrokenLinks.getCollectedLinks(),

‎packages/docusaurus/src/client/serverHelmetUtils.tsx‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
*/
77

88
import type {ReactElement} from 'react';
9-
import type {PageCollectedMetadata} from '../common';
9+
import type {PageCollectedMetadataInternal} from '../common';
1010
import type {HelmetServerState} from 'react-helmet-async';
1111

1212
type BuildMetaTag = {name?: string; content?: string};
@@ -30,11 +30,11 @@ function isNoIndexTag(tag: BuildMetaTag): boolean {
3030
);
3131
}
3232

33-
export function toPageCollectedMetadata({
33+
export function toPageCollectedMetadataInternal({
3434
helmet,
3535
}: {
3636
helmet: HelmetServerState;
37-
}): PageCollectedMetadata {
37+
}): PageCollectedMetadataInternal {
3838
const tags = getBuildMetaTags(helmet);
3939
const noIndex = tags.some(isNoIndexTag);
4040

‎packages/docusaurus/src/common.d.ts‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import type {RouteBuildMetadata} from '@docusaurus/types';
1313

1414
export type AppRenderResult = {
1515
html: string;
16-
collectedData: PageCollectedData;
16+
collectedData: PageCollectedDataInternal;
1717
};
1818

1919
export type AppRenderer = {
@@ -40,23 +40,43 @@ export type RouteBuildMetadataInternal = {
4040
script: string;
4141
};
4242

43-
// This data structure must remain serializable!
44-
// See why: https://github.com/facebook/docusaurus/pull/10826
4543
export type PageCollectedMetadata = {
4644
public: RouteBuildMetadata;
47-
internal: RouteBuildMetadataInternal;
4845
// TODO Docusaurus v4 remove legacy unserializable helmet data structure
4946
// See https://github.com/facebook/docusaurus/pull/10850
5047
helmet: HelmetServerState | null;
5148
};
5249

50+
// This data structure must remain serializable!
51+
// See why: https://github.com/facebook/docusaurus/pull/10826
52+
export type PageCollectedMetadataInternal = PageCollectedMetadata & {
53+
internal: {
54+
htmlAttributes: string;
55+
bodyAttributes: string;
56+
title: string;
57+
meta: string;
58+
link: string;
59+
script: string;
60+
};
61+
};
62+
63+
export type PageCollectedDataInternal = {
64+
metadata: PageCollectedMetadataInternal;
65+
modules: string[];
66+
links: string[];
67+
anchors: string[];
68+
};
69+
70+
// Keep this data structure as small as possible
71+
// See https://github.com/facebook/docusaurus/pull/11162
5372
export type PageCollectedData = {
5473
metadata: PageCollectedMetadata;
5574
links: string[];
5675
anchors: string[];
57-
modules: string[];
5876
};
5977

78+
// Keep this data structure as small as possible
79+
// See https://github.com/facebook/docusaurus/pull/11162
6080
export type SiteCollectedData = {
6181
[pathname: string]: PageCollectedData;
6282
};

‎packages/docusaurus/src/ssg/ssgExecutor.ts‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -38,16 +38,13 @@ const createSimpleSSGExecutor: CreateSSGExecutor = async ({
3838
}) => {
3939
return {
4040
run: () => {
41-
return PerfLogger.async(
42-
'Generate static files (current thread)',
43-
async () => {
44-
const ssgResults = await executeSSGInlineTask({
45-
pathnames,
46-
params,
47-
});
48-
return createGlobalSSGResult(ssgResults);
49-
},
50-
);
41+
return PerfLogger.async('SSG (current thread)', async () => {
42+
const ssgResults = await executeSSGInlineTask({
43+
pathnames,
44+
params,
45+
});
46+
return createGlobalSSGResult(ssgResults);
47+
});
5148
},
5249

5350
destroy: async () => {
@@ -111,7 +108,7 @@ const createPooledSSGExecutor: CreateSSGExecutor = async ({
111108
}
112109

113110
const pool = await PerfLogger.async(
114-
`Create SSG pool - ${logger.cyan(numberOfThreads)} threads`,
111+
`Create SSG thread pool - ${logger.cyan(numberOfThreads)} threads`,
115112
async () => {
116113
const Tinypool = await import('tinypool').then((m) => m.default);
117114

@@ -134,23 +131,26 @@ const createPooledSSGExecutor: CreateSSGExecutor = async ({
134131
const pathnamesChunks = _.chunk(pathnames, SSGWorkerThreadTaskSize);
135132

136133
// Tiny wrapper for type-safety
137-
const submitTask: ExecuteSSGWorkerThreadTask = (task) => pool.run(task);
134+
const submitTask: ExecuteSSGWorkerThreadTask = async (task) => {
135+
const result = await pool.run(task);
136+
// Note, we don't use PerfLogger.async() because all tasks are submitted
137+
// immediately at once and queued, while results are received progressively
138+
PerfLogger.log(`Result for task ${logger.name(task.id)}`);
139+
return result;
140+
};
138141

139142
return {
140143
run: async () => {
141-
const results = await PerfLogger.async(
142-
`Generate static files (${numberOfThreads} worker threads)`,
143-
async () => {
144-
return Promise.all(
145-
pathnamesChunks.map((taskPathnames, taskIndex) => {
146-
return submitTask({
147-
id: taskIndex + 1,
148-
pathnames: taskPathnames,
149-
});
150-
}),
151-
);
152-
},
153-
);
144+
const results = await PerfLogger.async(`Thread pool`, async () => {
145+
return Promise.all(
146+
pathnamesChunks.map((taskPathnames, taskIndex) => {
147+
return submitTask({
148+
id: taskIndex + 1,
149+
pathnames: taskPathnames,
150+
});
151+
}),
152+
);
153+
});
154154
const allResults = results.flat();
155155
return createGlobalSSGResult(allResults);
156156
},

‎packages/docusaurus/src/ssg/ssgRenderer.ts‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,18 @@ import {SSGConcurrency} from './ssgEnv';
2222
import {writeStaticFile} from './ssgUtils';
2323
import {createSSGRequire} from './ssgNodeRequire';
2424
import type {SSGParams} from './ssgParams';
25-
import type {AppRenderer, AppRenderResult} from '../common';
25+
import type {
26+
AppRenderer,
27+
PageCollectedData,
28+
PageCollectedDataInternal,
29+
} from '../common';
2630
import type {HtmlMinifier} from '@docusaurus/bundler';
2731

2832
export type SSGSuccess = {
2933
success: true;
3034
pathname: string;
3135
result: {
32-
collectedData: AppRenderResult['collectedData'];
36+
collectedData: PageCollectedData;
3337
warnings: string[];
3438
// html: we don't include it on purpose!
3539
// we don't need to aggregate all html contents in memory!
@@ -144,6 +148,26 @@ export async function loadSSGRenderer({
144148
};
145149
}
146150

151+
// We reduce the page collected data structure after the HTML file is written
152+
// Some data (modules, metadata.internal) is only useful to create the HTML file
153+
// It's not useful to aggregate that collected data in memory
154+
// Keep this data structure as small as possible
155+
// See https://github.com/facebook/docusaurus/pull/11162
156+
function reduceCollectedData(
157+
pageCollectedData: PageCollectedDataInternal,
158+
): PageCollectedData {
159+
// We re-create the object from scratch
160+
// We absolutely want to avoid TS duck typing
161+
return {
162+
anchors: pageCollectedData.anchors,
163+
metadata: {
164+
public: pageCollectedData.metadata.public,
165+
helmet: pageCollectedData.metadata.helmet,
166+
},
167+
links: pageCollectedData.links,
168+
};
169+
}
170+
147171
async function generateStaticFile({
148172
pathname,
149173
appRenderer,
@@ -176,11 +200,14 @@ async function generateStaticFile({
176200
content: minifierResult.code,
177201
params,
178202
});
203+
204+
const collectedData = reduceCollectedData(appRenderResult.collectedData);
205+
179206
return {
180207
success: true,
181208
pathname,
182209
result: {
183-
collectedData: appRenderResult.collectedData,
210+
collectedData,
184211
// As of today, only the html minifier can emit SSG warnings
185212
warnings: minifierResult.warnings,
186213
},

0 commit comments

Comments
 (0)