|
| 1 | +import path from 'node:path'; |
| 2 | +import fs from 'node:fs/promises'; |
| 3 | +import { fileURLToPath } from 'node:url'; |
| 4 | +import { cli } from 'cleye'; |
| 5 | +import { createFixture } from 'fs-fixture'; |
| 6 | +import { tmpdir as tsxCacheDirectory } from '../../src/utils/temporary-directory.js'; |
| 7 | +import { isFeatureSupported, type Version } from '../../src/utils/node-features.js'; |
| 8 | +import type { SpecifierStyle } from './utils/generate-fixture.js'; |
| 9 | +import { scenarios, type Scenario } from './utils/scenarios.js'; |
| 10 | +import { resolveTsx, type TsxImplementation } from './utils/resolve-tsx.js'; |
| 11 | +import { resolveNode, type NodeBinary } from './utils/resolve-node.js'; |
| 12 | +import { runOnce, type RunResult } from './utils/run.js'; |
| 13 | +import { mean, standardDeviation, linearFit } from './utils/stats.js'; |
| 14 | + |
| 15 | +const specifierStyles: SpecifierStyle[] = ['ts', 'js', 'extensionless']; |
| 16 | +const scaleCounts = [10, 100, 300, 1000]; |
| 17 | + |
| 18 | +const argv = cli({ |
| 19 | + name: 'benchmark', |
| 20 | + parameters: ['[scenarios...]'], |
| 21 | + help: { |
| 22 | + description: 'Measures tsx startup across scenarios, Node versions, and project sizes', |
| 23 | + }, |
| 24 | + flags: { |
| 25 | + compare: { |
| 26 | + type: [String], |
| 27 | + alias: 'c', |
| 28 | + description: 'tsx to compare: npm version (e.g. 4.21.0) or path to a tsx checkout', |
| 29 | + default: () => [] as string[], |
| 30 | + }, |
| 31 | + node: { |
| 32 | + type: [String], |
| 33 | + alias: 'n', |
| 34 | + description: 'Additional Node version to test (repeatable); downloaded via get-node', |
| 35 | + default: () => [] as string[], |
| 36 | + }, |
| 37 | + modules: { |
| 38 | + type: Number, |
| 39 | + alias: 'm', |
| 40 | + description: 'Module count (ignored with --scale)', |
| 41 | + default: 1000, |
| 42 | + }, |
| 43 | + specifier: { |
| 44 | + type: (style: SpecifierStyle) => { |
| 45 | + if (!specifierStyles.includes(style)) { |
| 46 | + throw new Error(`Invalid specifier style: ${style}`); |
| 47 | + } |
| 48 | + return style; |
| 49 | + }, |
| 50 | + alias: 's', |
| 51 | + description: `Import specifier style for esm-ts: ${specifierStyles.join(', ')}`, |
| 52 | + default: 'ts' as SpecifierStyle, |
| 53 | + }, |
| 54 | + runs: { |
| 55 | + type: Number, |
| 56 | + alias: 'r', |
| 57 | + description: 'Timed runs per cell', |
| 58 | + default: 5, |
| 59 | + }, |
| 60 | + cold: { |
| 61 | + type: Boolean, |
| 62 | + description: 'Clear the tsx transform cache before every run', |
| 63 | + default: false, |
| 64 | + }, |
| 65 | + scale: { |
| 66 | + type: Boolean, |
| 67 | + description: `Sweep module counts ${scaleCounts.join('/')} and report per-module cost + fixed tax`, |
| 68 | + default: false, |
| 69 | + }, |
| 70 | + json: { |
| 71 | + type: Boolean, |
| 72 | + description: 'Emit raw per-run results as JSON', |
| 73 | + default: false, |
| 74 | + }, |
| 75 | + }, |
| 76 | +}); |
| 77 | + |
| 78 | +const { |
| 79 | + compare, node: nodeVersions, modules, specifier, runs, cold, scale, json, |
| 80 | +} = argv.flags; |
| 81 | + |
| 82 | +const selectedScenarios = ( |
| 83 | + argv._.scenarios.length > 0 |
| 84 | + ? argv._.scenarios.map((name) => { |
| 85 | + const scenario = scenarios.find(s => s.name === name); |
| 86 | + if (!scenario) { |
| 87 | + throw new Error(`Unknown scenario "${name}". Available: ${scenarios.map(s => s.name).join(', ')}`); |
| 88 | + } |
| 89 | + return scenario; |
| 90 | + }) |
| 91 | + : scenarios.filter(scenario => scenario.default) |
| 92 | +); |
| 93 | + |
| 94 | +const moduleCounts = scale ? scaleCounts : [modules]; |
| 95 | +const log = (message = '') => process.stderr.write(`${message}\n`); |
| 96 | +const out = (message = '') => process.stdout.write(`${message}\n`); |
| 97 | + |
| 98 | +const clearTransformCache = () => fs.rm(tsxCacheDirectory, { |
| 99 | + recursive: true, |
| 100 | + force: true, |
| 101 | +}); |
| 102 | + |
| 103 | +type Row = { |
| 104 | + scenario: string; |
| 105 | + nodeVersion: string; |
| 106 | + impl: string; |
| 107 | + moduleCount: number; |
| 108 | + results: RunResult[]; |
| 109 | +}; |
| 110 | + |
| 111 | +const measureCell = async ( |
| 112 | + scenario: Scenario, |
| 113 | + node: NodeBinary, |
| 114 | + cliPath: string, |
| 115 | + fixturePath: string, |
| 116 | +): Promise<RunResult[]> => { |
| 117 | + const entryPath = path.join(fixturePath, scenario.entry); |
| 118 | + const args = scenario.runner === 'tsx' ? [cliPath, entryPath] : [entryPath]; |
| 119 | + const usesCache = scenario.runner === 'tsx'; |
| 120 | + |
| 121 | + // Reset cache before warmup: avoids stale-file skew and cross-cell pollution |
| 122 | + if (usesCache) { |
| 123 | + await clearTransformCache(); |
| 124 | + } |
| 125 | + await runOnce(node.path, args, fixturePath); |
| 126 | + |
| 127 | + const results: RunResult[] = []; |
| 128 | + for (let run = 0; run < runs; run += 1) { |
| 129 | + if (usesCache && cold) { |
| 130 | + await clearTransformCache(); |
| 131 | + } |
| 132 | + results.push(await runOnce(node.path, args, fixturePath)); |
| 133 | + } |
| 134 | + return results; |
| 135 | +}; |
| 136 | + |
| 137 | +// Resolve Node binaries (current + any --node), de-duplicated by version |
| 138 | +const nodeBinaries: NodeBinary[] = []; |
| 139 | +const resolvedBinaries = await Promise.all([ |
| 140 | + resolveNode(), |
| 141 | + ...nodeVersions.map(version => resolveNode(version)), |
| 142 | +]); |
| 143 | +for (const binary of resolvedBinaries) { |
| 144 | + if (!nodeBinaries.some(existing => existing.version === binary.version)) { |
| 145 | + nodeBinaries.push(binary); |
| 146 | + } |
| 147 | +} |
| 148 | + |
| 149 | +// tsx implementations: local + comparisons (npm/path) |
| 150 | +await using installRoot = await createFixture(); |
| 151 | +const implementations: TsxImplementation[] = [ |
| 152 | + { |
| 153 | + name: 'local', |
| 154 | + cliPath: fileURLToPath(new URL('../../dist/cli.mjs', import.meta.url)), |
| 155 | + }, |
| 156 | + ...await Promise.all( |
| 157 | + compare.map(comparison => resolveTsx(comparison, installRoot.path)), |
| 158 | + ), |
| 159 | +]; |
| 160 | + |
| 161 | +const rows: Row[] = []; |
| 162 | +const skipped: string[] = []; |
| 163 | + |
| 164 | +// Measurements are intentionally sequential: concurrent child processes |
| 165 | +// would contend for CPU and corrupt the timings. |
| 166 | +for (const scenario of selectedScenarios) { |
| 167 | + // Build each fixture once per module count; reuse across nodes and cells |
| 168 | + const fixtures = await Promise.all(moduleCounts.map( |
| 169 | + moduleCount => createFixture(scenario.build(moduleCount, specifier)), |
| 170 | + )); |
| 171 | + await using _scenarioFixtures = { |
| 172 | + [Symbol.asyncDispose]: async () => { |
| 173 | + await Promise.all(fixtures.map(fixture => fixture.rm())); |
| 174 | + }, |
| 175 | + }; |
| 176 | + |
| 177 | + for (const node of nodeBinaries) { |
| 178 | + if ( |
| 179 | + scenario.supportedNodeVersions |
| 180 | + && !isFeatureSupported( |
| 181 | + scenario.supportedNodeVersions, |
| 182 | + node.version.split('.').map(Number) as Version, |
| 183 | + ) |
| 184 | + ) { |
| 185 | + skipped.push(`${scenario.name} on Node ${node.version} (unsupported)`); |
| 186 | + continue; |
| 187 | + } |
| 188 | + |
| 189 | + // node-runner scenarios (baseline/native) ignore --compare |
| 190 | + const cells = ( |
| 191 | + scenario.runner === 'node' |
| 192 | + ? [{ |
| 193 | + name: 'node', |
| 194 | + cliPath: '', |
| 195 | + }] |
| 196 | + : implementations |
| 197 | + ); |
| 198 | + for (const { name, cliPath } of cells) { |
| 199 | + for (const [index, moduleCount] of moduleCounts.entries()) { |
| 200 | + log(` ${scenario.name} · Node ${node.version} · ${name} · ${moduleCount} modules`); |
| 201 | + const results = await measureCell(scenario, node, cliPath, fixtures[index].path); |
| 202 | + rows.push({ |
| 203 | + scenario: scenario.name, |
| 204 | + nodeVersion: node.version, |
| 205 | + impl: name, |
| 206 | + moduleCount, |
| 207 | + results, |
| 208 | + }); |
| 209 | + } |
| 210 | + } |
| 211 | + } |
| 212 | +} |
| 213 | + |
| 214 | +if (json) { |
| 215 | + process.stdout.write(`${JSON.stringify({ |
| 216 | + meta: { |
| 217 | + runs, |
| 218 | + cold, |
| 219 | + scale, |
| 220 | + specifier, |
| 221 | + }, |
| 222 | + rows, |
| 223 | + }, null, 2)}\n`); |
| 224 | +} else { |
| 225 | + renderTables(); |
| 226 | +} |
| 227 | + |
| 228 | +function renderTables() { |
| 229 | + out(); |
| 230 | + out(`${runs} runs/cell · ${cold ? 'cold' : 'warm'} cache · specifier=${specifier}`); |
| 231 | + |
| 232 | + for (const scenario of selectedScenarios) { |
| 233 | + const scenarioRows = rows.filter(row => row.scenario === scenario.name); |
| 234 | + if (scenarioRows.length === 0) { |
| 235 | + continue; |
| 236 | + } |
| 237 | + out(`\n## ${scenario.name} — ${scenario.description}`); |
| 238 | + |
| 239 | + const byNode = new Map<string, Row[]>(); |
| 240 | + for (const row of scenarioRows) { |
| 241 | + const list = byNode.get(row.nodeVersion) ?? []; |
| 242 | + list.push(row); |
| 243 | + byNode.set(row.nodeVersion, list); |
| 244 | + } |
| 245 | + |
| 246 | + for (const [nodeVersion, nodeRows] of byNode) { |
| 247 | + if (scale) { |
| 248 | + out(`Node ${nodeVersion} — scale ${scaleCounts.join('/')}:`); |
| 249 | + const byImpl = new Map<string, Row[]>(); |
| 250 | + for (const row of nodeRows) { |
| 251 | + const list = byImpl.get(row.impl) ?? []; |
| 252 | + list.push(row); |
| 253 | + byImpl.set(row.impl, list); |
| 254 | + } |
| 255 | + for (const [impl, implRows] of byImpl) { |
| 256 | + const points = implRows.map(row => ({ |
| 257 | + x: row.moduleCount, |
| 258 | + y: mean(row.results.map(result => result.wallMs)), |
| 259 | + })); |
| 260 | + const { slope, intercept } = linearFit(points); |
| 261 | + const detail = points |
| 262 | + .slice() |
| 263 | + .sort((a, b) => a.x - b.x) |
| 264 | + .map(point => `${point.x}→${point.y.toFixed(0)}ms`) |
| 265 | + .join(' '); |
| 266 | + out(` ${impl.padEnd(20)} ${(slope * 1000).toFixed(1)}µs/module +${intercept.toFixed(0)}ms fixed [${detail}]`); |
| 267 | + } |
| 268 | + } else { |
| 269 | + const [count] = moduleCounts; |
| 270 | + out(`Node ${nodeVersion} — ${count} modules:`); |
| 271 | + const localMean = mean( |
| 272 | + (nodeRows.find(row => row.impl === 'local') ?? nodeRows[0]) |
| 273 | + .results.map(result => result.wallMs), |
| 274 | + ); |
| 275 | + for (const row of nodeRows) { |
| 276 | + // wall is the headline signal; RSS + load/eval split carry little |
| 277 | + // independent signal on synthetic trees (see README) — JSON only. |
| 278 | + const wall = row.results.map(result => result.wallMs); |
| 279 | + const wallMean = mean(wall); |
| 280 | + const relative = ( |
| 281 | + row.impl === 'local' || scenario.runner === 'node' |
| 282 | + ? '' |
| 283 | + : ` (${(wallMean / localMean).toFixed(2)}x)` |
| 284 | + ); |
| 285 | + const name = row.impl.padEnd(20); |
| 286 | + const meanColumn = `mean ${wallMean.toFixed(0)}±${standardDeviation(wall).toFixed(0)}ms`.padEnd(18); |
| 287 | + const minColumn = `min ${Math.min(...wall).toFixed(0)}ms`; |
| 288 | + out(` ${name}${meanColumn}${minColumn}${relative}`); |
| 289 | + } |
| 290 | + } |
| 291 | + } |
| 292 | + } |
| 293 | + |
| 294 | + if (skipped.length > 0) { |
| 295 | + out(`\nSkipped:\n${skipped.map(entry => ` ${entry}`).join('\n')}`); |
| 296 | + } |
| 297 | +} |
0 commit comments