Skip to content

Commit eb43ae5

Browse files
bjohansebasbjohansebas
authored
perf(javascript): reduce minifier token buffer memory (#22329)
* perf(javascript): reduce minifier token buffer memory * test(javascript): type runtime comment pattern fixtures * test(javascript): restore Unicode line separators in parser fixture
1 parent 586ef39 commit eb43ae5

6 files changed

Lines changed: 156 additions & 30 deletions

File tree

‎.changeset/073-javascript-comment-patterns.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"webpack": patch
3+
---
4+
5+
Reduce JavaScript minifier buffer memory, preserve token positions for large inputs, and reuse comment filter patterns.

‎lib/javascript/jsMinify.js‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,11 +108,19 @@ const jsMinify = async (
108108
}
109109
if (condition === "all") return () => true;
110110
if (condition === "some") {
111+
const pattern = /@preserve|@lic|@cc_on|^\**!/i;
111112
return (node, comment) =>
112113
(comment.type === "comment2" || comment.type === "comment1") &&
113-
/@preserve|@lic|@cc_on|^\**!/i.test(comment.value);
114+
pattern.test(comment.value);
114115
}
115-
return (node, comment) => new RegExp(condition).test(comment.value);
116+
// Compile once, but only when asked: a malformed pattern must still be
117+
// harmless when there are no comments to test against it.
118+
/** @type {RegExp | undefined} */
119+
let pattern;
120+
return (node, comment) => {
121+
if (pattern === undefined) pattern = new RegExp(condition);
122+
return pattern.test(comment.value);
123+
};
116124
};
117125

118126
const format = { beautify: false, ...options.format, ...options.output };

‎lib/javascript/syntax-printer.js‎

Lines changed: 48 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1668,10 +1668,10 @@ const createTerserTree = ({ ast, parse }) => {
16681668
const EMPTY = [];
16691669
// Kept across calls, as a worker minifies one asset after another: every
16701670
// read is at an offset this source set, so what an earlier one left is inert.
1671-
let byStart = new Int32Array(0);
1672-
let byEnd = new Int32Array(0);
1673-
// Each token's offsets and kind, by index: a source holds fewer tokens
1674-
// than characters, so these grow with the two above.
1671+
let startIndexBuffer = new Int32Array(0);
1672+
let endIndexBuffer = new Int32Array(0);
1673+
// Each token's offsets and kind grow with the token count, not source size:
1674+
// comments and long identifiers or strings need no extra token slots.
16751675
let startBuffer = new Int32Array(0);
16761676
let endBuffer = new Int32Array(0);
16771677
let kindBuffer = new Int8Array(0);
@@ -1690,18 +1690,47 @@ const createTerserTree = ({ ast, parse }) => {
16901690
let inTemplateText = false;
16911691
/** @type {boolean[]} */
16921692
const braces = [];
1693-
if (byStart.length <= source.length) {
1694-
byStart = new Int32Array(source.length + 1);
1695-
byEnd = new Int32Array(source.length + 1);
1696-
startBuffer = new Int32Array(source.length + 1);
1697-
endBuffer = new Int32Array(source.length + 1);
1698-
kindBuffer = new Int8Array(source.length + 1);
1693+
if (startIndexBuffer.length <= source.length) {
1694+
startIndexBuffer = new Int32Array(source.length + 1);
1695+
endIndexBuffer = new Int32Array(source.length + 1);
16991696
}
1700-
const starts = startBuffer;
1701-
const ends = endBuffer;
1702-
const kinds = kindBuffer;
1703-
const tokenStart = byStart;
1704-
const tokenEnd = byEnd;
1697+
let starts = startBuffer;
1698+
let ends = endBuffer;
1699+
let kinds = kindBuffer;
1700+
const byStart = startIndexBuffer;
1701+
const byEnd = endIndexBuffer;
1702+
// Keep this call's indices alive through conversion, but do not cache a
1703+
// large source's buffers, even if parsing or conversion declines it.
1704+
if (source.length > 1 << 23) {
1705+
startIndexBuffer = new Int32Array(0);
1706+
endIndexBuffer = new Int32Array(0);
1707+
startBuffer = new Int32Array(0);
1708+
endBuffer = new Int32Array(0);
1709+
kindBuffer = new Int8Array(0);
1710+
}
1711+
/**
1712+
* @returns {void} grow the token tables without retaining oversized inputs
1713+
*/
1714+
const growTokens = () => {
1715+
const capacity = Math.min(
1716+
source.length + 1,
1717+
Math.max(4096, starts.length * 2)
1718+
);
1719+
const nextStarts = new Int32Array(capacity);
1720+
const nextEnds = new Int32Array(capacity);
1721+
const nextKinds = new Int8Array(capacity);
1722+
nextStarts.set(starts);
1723+
nextEnds.set(ends);
1724+
nextKinds.set(kinds);
1725+
starts = nextStarts;
1726+
ends = nextEnds;
1727+
kinds = nextKinds;
1728+
if (source.length <= 1 << 23) {
1729+
startBuffer = starts;
1730+
endBuffer = ends;
1731+
kindBuffer = kinds;
1732+
}
1733+
};
17051734
// Whitespace to the spec that terser's tokenizer refuses.
17061735
if (source.includes("\u1680")) return undefined;
17071736
/** @type {EstreeNode} */
@@ -1739,7 +1768,7 @@ const createTerserTree = ({ ast, parse }) => {
17391768
// `}` closing a substitution, through the next `${` or backtick.
17401769
if (inTemplateText) {
17411770
ends[count - 1] = token.end;
1742-
tokenEnd[token.end] = count;
1771+
byEnd[token.end] = count;
17431772
// terser's template token carries the text it holds.
17441773
if (label === "template" || label === "invalidTemplate") {
17451774
values[count - 1] = token.value;
@@ -1791,13 +1820,14 @@ const createTerserTree = ({ ast, parse }) => {
17911820
// A word after a dot is a property name to terser, even a keyword.
17921821
if (kind === WORD && previousWasDot) kind = -WORD;
17931822
previousWasDot = label === "." || label === "?.";
1823+
if (count === starts.length) growTokens();
17941824
starts[count] = token.start;
17951825
ends[count] = token.end;
17961826
kinds[count] = kind;
17971827
values.push(value);
17981828
count++;
1799-
tokenStart[token.start] = count;
1800-
tokenEnd[token.end] = count;
1829+
byStart[token.start] = count;
1830+
byEnd[token.end] = count;
18011831
}
18021832
});
18031833
} catch (_err) {
@@ -1841,15 +1871,6 @@ const createTerserTree = ({ ast, parse }) => {
18411871
return low;
18421872
};
18431873

1844-
// Too large a source is not kept past this call.
1845-
if (source.length > 1 << 23) {
1846-
byStart = new Int32Array(0);
1847-
byEnd = new Int32Array(0);
1848-
startBuffer = new Int32Array(0);
1849-
endBuffer = new Int32Array(0);
1850-
kindBuffer = new Int8Array(0);
1851-
}
1852-
18531874
// The comments in each gap, gap `i` being the one before token `i`.
18541875
// Sized up front: filled out of order, it would otherwise go sparse.
18551876
/** @type {(Token[] | undefined)[]} */

‎test/benchmarkCases/js-minifier-unit/index.bench.mjs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import fs from "fs";
2+
import { createRequire } from "module";
3+
4+
const require = createRequire(import.meta.url);
5+
6+
const jsMinify = require("../../../lib/javascript/jsMinify.js");
7+
8+
const source = fs.readFileSync(require.resolve("lodash/lodash.js"), "utf8");
9+
10+
/**
11+
* @param {import("tinybench").Bench} bench bench
12+
* @returns {void}
13+
*/
14+
export default (bench) => {
15+
// Includes compression and output: token-table savings must survive the
16+
// whole minifier, including the AST and source buffers it also needs.
17+
bench.add('unit benchmark "js-minifier-unit", lodash', async () => {
18+
await jsMinify(
19+
{ "lodash.js": source },
20+
undefined,
21+
{ printer: true, compress: { passes: 2 } },
22+
true
23+
);
24+
});
25+
};

‎test/unitCases/jsMinify.unittest.js‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,47 @@ describe("jsMinify", () => {
130130
await expect(jsMinify({ "broken.js": "function (" })).rejects.toThrow();
131131
});
132132

133+
for (const printer of [false, true]) {
134+
it(`should match string comment patterns like RegExp conditions (printer: ${printer})`, async () => {
135+
const input = {
136+
"comments.js":
137+
"/*! @license first */ first();\n" +
138+
"/* ordinary */ second();\n" +
139+
"/*! @license last */ last();"
140+
};
141+
const actual = await jsMinify(
142+
input,
143+
INPUT_MAP,
144+
// Terser accepts string patterns at runtime, but its types exclude them.
145+
{ printer, format: { comments: /** @type {EXPECTED_ANY} */ ("first|last") } },
146+
{ condition: /** @type {EXPECTED_ANY} */ ("@license") }
147+
);
148+
const expected = await jsMinify(
149+
input,
150+
INPUT_MAP,
151+
{ printer, format: { comments: /first|last/ } },
152+
{ condition: /@license/ }
153+
);
154+
expect(actual).toEqual(expected);
155+
expect(actual.extractedComments).toHaveLength(2);
156+
});
157+
158+
it(`should defer invalid comment patterns until a comment is tested (printer: ${printer})`, async () => {
159+
// Exercise a malformed runtime pattern outside the published type union.
160+
const invalidPattern = /** @type {EXPECTED_ANY} */ ("[");
161+
const options = { printer, format: { comments: invalidPattern } };
162+
const input = { "plain.js": "run();" };
163+
expect(await jsMinify(input, undefined, options, { condition: invalidPattern })).toEqual(
164+
await jsMinify(input, undefined, { printer, format: { comments: false } }, false)
165+
);
166+
const commented = { "commented.js": "/* comment */ run();" };
167+
await expect(jsMinify(commented, undefined, options, false)).rejects.toThrow(SyntaxError);
168+
await expect(
169+
jsMinify(commented, undefined, { printer }, { condition: invalidPattern })
170+
).rejects.toThrow(SyntaxError);
171+
});
172+
}
173+
133174
it("should claim JavaScript assets only", () => {
134175
expect(jsMinify.filter("app.js")).toBe(true);
135176
expect(jsMinify.filter("app.mjs?v=1")).toBe(true);

‎test/unitCases/syntax-printer-parse.unittest.js‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ const CASES = [
167167
["regular expressions read twice", "{}\n/a/.test(b); if (c) /d/.test(e); x = yield_ => /f/;"],
168168
["literals", "a = [1, 1.5, .5, 1e3, 0x10, 0b1, 0o7, 1_000, 10n, 0x1Fn, 1e400, 'x', \"y\", null, true, false, /re/giu];"],
169169
["identifiers beyond ASCII", "var été = 1, \u{1d49c} = 2; sink(été, \u{1d49c});"],
170-
["line breaks of every kind", "a = 1\r\nb = 2\rc = 3
d = 4
e = /* x\r\ny */ 5;"],
170+
["line breaks of every kind", "a = 1\r\nb = 2\rc = 3\u2028d = 4\u2029e = /* x\r\ny */ 5;"],
171171
["a shebang and banners", "#!/usr/bin/env node\n/*! banner */\n// line\nsink();"],
172172
["an empty source", ""],
173173
["only comments", "/* a */ // b\n"],
@@ -263,6 +263,32 @@ describe("syntax-printer parse", () => {
263263
});
264264
}
265265

266+
it("should preserve token positions when releasing oversized buffers", () => {
267+
if (!loaded) return;
268+
const convert = createTerserTree(terser);
269+
const options = { filename: "large.js" };
270+
const body = "const value = (first + second); use(value);";
271+
for (const source of [body, body + " ".repeat((1 << 23) + 1), body]) {
272+
const theirs = terser.parse.parse(source, options);
273+
const ours = convert(source, options);
274+
expect(ours).toBeDefined();
275+
expect(firstDifference(terser.ast, theirs, ours)).toBeUndefined();
276+
}
277+
});
278+
279+
it("should grow token tables across files and template boundaries", () => {
280+
if (!loaded) return;
281+
const convert = createTerserTree(terser);
282+
const options = { filename: "tokens.js" };
283+
for (const count of [1, 800, 1600, 2]) {
284+
const source = "consume(`before${value}after`, /pattern/g);\n".repeat(count);
285+
const theirs = terser.parse.parse(source, options);
286+
const ours = convert(source, options);
287+
expect(ours).toBeDefined();
288+
expect(firstDifference(terser.ast, theirs, ours)).toBeUndefined();
289+
}
290+
});
291+
266292
for (const [name, source, module] of DECLINED) {
267293
it(`should leave to terser: ${name}`, () => {
268294
if (!loaded) return;

0 commit comments

Comments
 (0)