Skip to content

Commit dcaf957

Browse files
authored
chore: optimize fixtures in a bounded worker pool (#2265)
## Summary - optimize regression fixtures in worker threads orchestrated by Tinypool - cap optimization at two concurrent workers on memory-constrained CI runners - move fixture reads, synchronous optimization, checksumming, and writes into the worker - avoid retaining fixture contents or optimized SVGs on the main thread - report optimization, rendering, and comparison durations separately ## Motivation `optimize()` is synchronous, so the previous `os.cpus().length * 2` async functions did not provide CPU parallelism. They only allowed multiple fixture reads to complete around the active optimization, increasing peak memory without parallelizing the expensive work. This uses actual worker threads while bounding the number of concurrent large ASTs to two. Tasks contain only file paths, and workers return only checksums and byte counts. ## Benchmark A local run over 256 fixtures sampled across the regression corpus: - one Tinypool worker: 11.7 seconds - two Tinypool workers: 6.6 seconds - wall-time reduction: about 43% The benchmark excludes the corpus's seven extreme 20–90 MB fixtures. The two-worker cap is intentionally conservative for those files and the 8 GiB CI environment. ## Verification - `pnpm test` — 1,554 passed, 9 skipped - `pnpm typecheck` - `pnpm exec eslint test/regression/optimize.js test/regression/optimize-worker.js` - `pnpm exec prettier --check test/regression/optimize.js test/regression/optimize-worker.js package.json pnpm-lock.yaml` - `git diff --check` - Tinypool worker smoke test verified optimization output, checksum, and byte metrics The full regression corpus was not run locally because it is host-dependent and includes several very large fixtures.
1 parent a354293 commit dcaf957

7 files changed

Lines changed: 98 additions & 45 deletions

File tree

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@
121121
"rimraf": "^6.1.3",
122122
"rollup": "^4.62.4",
123123
"tar-stream": "^3.2.0",
124+
"tinypool": "^2.1.0",
124125
"typescript": "^7.0.2",
125126
"vitest": "^4.1.10"
126127
}

‎pnpm-lock.yaml‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎test/regression/delta.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ function writeReport(deltaReport) {
6363
console.log(`${HEADING}
6464
Files Affected: ${filesAffected.toLocaleString()} / ${totalFiles.toLocaleString()}
6565
Bytes Saved Δ: ${toDisplayString(bytesDelta, (n) => bytesToHumanReadable(n))}
66-
Time Taken Δ: ${toDisplayString(secsDelta, (n) => secsToHumanReadable(n))}
66+
Optimization Time Δ: ${toDisplayString(secsDelta, (n) => secsToHumanReadable(n))}
6767
Peak Memory Alloc Δ: ${toDisplayString(memDelta, (n) => bytesToHumanReadable(n, 'KiB'))}`);
6868
}
6969

‎test/regression/lib.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ export async function printReport(report) {
5454
5555
▶ Metrics
5656
Bytes Saved: ${bytesToHumanReadable(report.metrics.bytesSaved)}
57-
Time Taken: ${secsToHumanReadable(report.metrics.timeTakenSecs)}
57+
Optimization Time: ${secsToHumanReadable(report.metrics.timeTakenSecs)}
5858
Peak Memory Alloc: ${bytesToHumanReadable(report.metrics.peakMemoryAlloc, 'KiB')}${
5959
shouldHaveMatched.length !== 0
6060
? picocolors.red(

‎test/regression/optimize-worker.js‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import crypto from 'node:crypto';
2+
import fs from 'node:fs/promises';
3+
import path from 'node:path';
4+
import { optimize } from '../../lib/svgo.js';
5+
6+
/** @type {import('../../lib/types.js').Config} */
7+
const SVGO_OPTS = { floatPrecision: 4 };
8+
9+
/**
10+
* @typedef OptimizeTask
11+
* @property {string} name
12+
* @property {string} originalPath
13+
* @property {string} optimizedPath
14+
*
15+
* @typedef OptimizeResult
16+
* @property {string} name
17+
* @property {number} bytesSaved
18+
* @property {string} checksum
19+
*/
20+
21+
/**
22+
* @param {OptimizeTask} task
23+
* @returns {Promise<OptimizeResult>}
24+
*/
25+
export default async function optimizeFixture(task) {
26+
const original = await fs.readFile(task.originalPath, 'utf8');
27+
const optimized = optimize(original, SVGO_OPTS).data;
28+
29+
await fs.mkdir(path.dirname(task.optimizedPath), { recursive: true });
30+
await fs.writeFile(task.optimizedPath, optimized);
31+
32+
return {
33+
name: task.name,
34+
bytesSaved:
35+
Buffer.byteLength(original, 'utf8') -
36+
Buffer.byteLength(optimized, 'utf8'),
37+
checksum: crypto.createHash('md5').update(optimized).digest('hex'),
38+
};
39+
}

‎test/regression/optimize.js‎

Lines changed: 46 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,24 @@
11
import fs from 'node:fs/promises';
22
import os from 'node:os';
33
import path from 'node:path';
4+
import Tinypool from 'tinypool';
45
import {
56
REGRESSION_FIXTURES_PATH,
67
REGRESSION_OPTIMIZED_PATH,
78
writeReport,
89
} from './regression-io.js';
9-
import { md5sum, pathToPosix } from './lib.js';
10-
import { optimize } from '../../lib/svgo.js';
10+
import { pathToPosix } from './lib.js';
1111

12-
/** @type {import('../../lib/types.js').Config} */
13-
const SVGO_OPTS = { floatPrecision: 4 };
12+
// Keep concurrent large ASTs bounded on memory-constrained CI runners.
13+
const DEFAULT_OPTIMIZE_WORKERS = Math.min(os.availableParallelism(), 2);
14+
const workerUrl = new URL('./optimize-worker.js', import.meta.url);
1415

1516
/**
16-
* @param {string[]} list
17-
* @returns {Promise<Partial<import('./regression-io.js').TestReport>>}
17+
* @param {ReadonlyArray<string>} list
18+
* @returns {Promise<Pick<import('./regression-io.js').TestReport, 'metrics' | 'checksums'>>}
1819
*/
1920
const optimizeFixtures = async (list) => {
21+
const started = performance.now();
2022
const totalFiles = list.length;
2123
let processed = 0;
2224

@@ -30,46 +32,43 @@ const optimizeFixtures = async (list) => {
3032
checksums: {},
3133
};
3234

33-
/**
34-
* @param {string} name
35-
*/
36-
const processFile = async (name) => {
37-
const original = await fs.readFile(
38-
path.join(REGRESSION_FIXTURES_PATH, name),
39-
'utf-8',
40-
);
41-
const optimized = optimize(original, SVGO_OPTS).data;
42-
const namePosix = pathToPosix(name);
43-
report.checksums[namePosix] = md5sum(optimized);
44-
45-
const prevFileSize = Buffer.byteLength(original, 'utf8');
46-
const resultFileSize = Buffer.byteLength(optimized, 'utf8');
47-
report.metrics.bytesSaved += prevFileSize - resultFileSize;
48-
49-
const file = path.join(REGRESSION_OPTIMIZED_PATH, name);
50-
await fs.mkdir(path.dirname(file), { recursive: true });
51-
await fs.writeFile(file, optimized);
52-
53-
if (process.stdout.isTTY) {
54-
process.stdout.clearLine(0);
55-
process.stdout.write(
56-
`\rOptimized ${(++processed).toLocaleString()} of ${totalFiles.toLocaleString()}…`,
35+
const workerCount = Math.min(DEFAULT_OPTIMIZE_WORKERS, list.length);
36+
if (workerCount !== 0) {
37+
const pool = new Tinypool({
38+
filename: workerUrl.href,
39+
minThreads: workerCount,
40+
maxThreads: workerCount,
41+
});
42+
/** @type {import('./optimize-worker.js').OptimizeResult[]} */
43+
let results;
44+
try {
45+
results = await Promise.all(
46+
list.map(async (name) => {
47+
const result = await pool.run({
48+
name,
49+
originalPath: path.join(REGRESSION_FIXTURES_PATH, name),
50+
optimizedPath: path.join(REGRESSION_OPTIMIZED_PATH, name),
51+
});
52+
if (process.stdout.isTTY) {
53+
process.stdout.clearLine(0);
54+
process.stdout.write(
55+
`\rOptimized ${(++processed).toLocaleString()} of ${totalFiles.toLocaleString()}…`,
56+
);
57+
}
58+
return result;
59+
}),
5760
);
61+
} finally {
62+
await pool.destroy();
5863
}
59-
};
6064

61-
const worker = async () => {
62-
let item;
63-
while ((item = list.pop())) {
64-
await processFile(item);
65+
for (const result of results) {
66+
report.checksums[pathToPosix(result.name)] = result.checksum;
67+
report.metrics.bytesSaved += result.bytesSaved;
6568
}
66-
};
67-
68-
await Promise.all(
69-
Array.from(new Array(os.cpus().length * 2), () => worker()),
70-
);
69+
}
7170

72-
report.metrics.timeTakenSecs = process.uptime();
71+
report.metrics.timeTakenSecs = (performance.now() - started) / 1000;
7372
report.metrics.peakMemoryAlloc = process.resourceUsage().maxRSS;
7473
return report;
7574
};
@@ -81,7 +80,12 @@ const optimizeFixtures = async (list) => {
8180
});
8281
const list = (await filesPromise).filter((name) => name.endsWith('.svg'));
8382
const report = await optimizeFixtures(list);
84-
console.log();
83+
if (process.stdout.isTTY) {
84+
console.log();
85+
}
86+
console.info(
87+
`Optimized SVGs in ${report.metrics.timeTakenSecs.toFixed(2)}s`,
88+
);
8589
await writeReport(report);
8690
} catch (error) {
8791
console.error(error);

‎test/regression/regression-io.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import { getCommitRef } from './lib.js';
2222
*
2323
* @typedef Metrics
2424
* @property {number} bytesSaved Total bytes saved throughout this test run.
25-
* @property {number} timeTakenSecs Total time taken throughout this test run.
25+
* @property {number} timeTakenSecs Time taken to optimize all fixtures.
2626
* @property {number} peakMemoryAlloc
2727
* Peak memory allocation throughout this test run in kibibytes (1,024 bytes).
2828
*

0 commit comments

Comments
 (0)