Skip to content

Commit f141408

Browse files
steipetesteipete
authored
fix(test): bound local full-suite RAM (#100437)
1 parent 8ccacd9 commit f141408

6 files changed

Lines changed: 22 additions & 72 deletions

File tree

‎scripts/lib/vitest-local-scheduling.d.mts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ export type VitestHostInfo = {
22
cpuCount?: number;
33
loadAverage1m?: number;
44
totalMemoryBytes?: number;
5+
freeMemoryBytes?: number;
56
};
67

78
export type LocalVitestScheduling = {
@@ -25,10 +26,6 @@ export function resolveLocalVitestScheduling(
2526
system?: VitestHostInfo,
2627
pool?: "forks" | "threads",
2728
): LocalVitestScheduling;
28-
export function shouldUseLargeLocalFullSuiteProfile(
29-
env?: Record<string, string | undefined>,
30-
system?: VitestHostInfo,
31-
): boolean;
3229
export function resolveLocalFullSuiteProfile(
3330
env?: Record<string, string | undefined>,
3431
system?: VitestHostInfo,

‎scripts/lib/vitest-local-scheduling.mjs‎

Lines changed: 6 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,7 @@
33

44
import os from "node:os";
55

6-
const DEFAULT_LOCAL_FULL_SUITE_PARALLELISM = 4;
7-
const LARGE_LOCAL_FULL_SUITE_PARALLELISM = 10;
8-
const DEFAULT_LOCAL_FULL_SUITE_VITEST_WORKERS = 1;
9-
const LARGE_LOCAL_FULL_SUITE_VITEST_WORKERS = 2;
6+
const MAX_LOCAL_FULL_SUITE_PARALLELISM = 10;
107

118
const clamp = (value, min, max) => Math.max(min, Math.min(max, value));
129

@@ -189,42 +186,12 @@ export function resolveLocalVitestScheduling(
189186
};
190187
}
191188

192-
export function shouldUseLargeLocalFullSuiteProfile(
193-
env = process.env,
194-
system = detectVitestHostInfo(),
195-
) {
196-
if (isCiLikeEnv(env)) {
197-
return false;
198-
}
199-
const scheduling = resolveLocalVitestScheduling(env, system, "threads");
200-
return scheduling.maxWorkers >= 5 && !scheduling.throttledBySystem;
201-
}
202-
203189
export function resolveLocalFullSuiteProfile(env = process.env, system = detectVitestHostInfo()) {
204-
if (!isSystemThrottleDisabled(env)) {
205-
const memoryPressureLimit = resolveMemoryPressureWorkerLimit(system);
206-
if (memoryPressureLimit === 1) {
207-
return {
208-
shardParallelism: 1,
209-
vitestMaxWorkers: 1,
210-
};
211-
}
212-
if (memoryPressureLimit === 2) {
213-
return {
214-
shardParallelism: 2,
215-
vitestMaxWorkers: 1,
216-
};
217-
}
218-
}
219-
220-
if (shouldUseLargeLocalFullSuiteProfile(env, system)) {
221-
return {
222-
shardParallelism: LARGE_LOCAL_FULL_SUITE_PARALLELISM,
223-
vitestMaxWorkers: LARGE_LOCAL_FULL_SUITE_VITEST_WORKERS,
224-
};
225-
}
190+
const scheduling = resolveLocalVitestScheduling(env, system, "threads");
226191
return {
227-
shardParallelism: DEFAULT_LOCAL_FULL_SUITE_PARALLELISM,
228-
vitestMaxWorkers: DEFAULT_LOCAL_FULL_SUITE_VITEST_WORKERS,
192+
// Each shard is a separate Vitest process with its own module graph. Spend the
193+
// host worker budget once across shards instead of multiplying it inside them.
194+
shardParallelism: Math.min(scheduling.maxWorkers, MAX_LOCAL_FULL_SUITE_PARALLELISM),
195+
vitestMaxWorkers: 1,
229196
};
230197
}

‎scripts/test-projects.mjs‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,6 @@ async function main() {
313313
);
314314
}
315315
if (concurrency > 1) {
316-
const localFullSuiteProfile = resolveLocalFullSuiteProfile(baseEnv);
317316
const shardTimings = readShardTimings(process.cwd(), baseEnv);
318317
const parallelSpecs = applyDefaultParallelVitestWorkerBudget(
319318
applyParallelVitestCachePaths(orderFullSuiteSpecsForParallelRun(runSpecs, shardTimings), {
@@ -322,16 +321,6 @@ async function main() {
322321
}),
323322
baseEnv,
324323
);
325-
if (
326-
!isCiLikeEnv(baseEnv) &&
327-
!baseEnv.OPENCLAW_TEST_PROJECTS_PARALLEL &&
328-
!baseEnv.OPENCLAW_VITEST_MAX_WORKERS &&
329-
!baseEnv.OPENCLAW_TEST_WORKERS &&
330-
localFullSuiteProfile.shardParallelism === 10 &&
331-
localFullSuiteProfile.vitestMaxWorkers === 2
332-
) {
333-
console.error("[test] using host-aware local full-suite profile: shards=10 workers=2");
334-
}
335324
console.error(
336325
`[test] running ${parallelSpecs.length} Vitest shards with parallelism ${concurrency}`,
337326
);

‎src/scripts/test-projects.test.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ const {
104104
cpuCount?: number;
105105
loadAverage1m?: number;
106106
totalMemoryBytes?: number;
107+
freeMemoryBytes?: number;
107108
},
108109
) => number;
109110
};
@@ -583,7 +584,7 @@ describe("test-projects args", () => {
583584
).toBe(3);
584585
});
585586

586-
it("uses a bounded local default for full-suite project parallelism", () => {
587+
it("uses the global host worker budget for full-suite project parallelism", () => {
587588
expect(
588589
resolveParallelFullSuiteConcurrency(
589590
58,
@@ -596,7 +597,7 @@ describe("test-projects args", () => {
596597
totalMemoryBytes: 16 * 1024 ** 3,
597598
},
598599
),
599-
).toBe(4);
600+
).toBe(2);
600601
});
601602

602603
it("gives parallel Vitest shards separate filesystem module caches", () => {

‎test/scripts/test-projects.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3820,7 +3820,7 @@ describe("scripts/test-projects full-suite sharding", () => {
38203820
]);
38213821
});
38223822

3823-
it("uses the large host-aware local profile on roomy local hosts", () => {
3823+
it("uses the global host worker budget for roomy local hosts", () => {
38243824
expect(
38253825
resolveParallelFullSuiteConcurrency(
38263826
61,
@@ -3831,7 +3831,7 @@ describe("scripts/test-projects full-suite sharding", () => {
38313831
totalMemoryBytes: 48 * 1024 ** 3,
38323832
},
38333833
),
3834-
).toBe(10);
3834+
).toBe(6);
38353835
});
38363836

38373837
it("keeps CI full-suite runs serial even on roomy hosts", () => {

‎test/scripts/vitest-local-scheduling.test.ts‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import {
44
resolveLocalVitestEnv,
55
resolveLocalFullSuiteProfile,
66
resolveLocalVitestScheduling,
7-
shouldUseLargeLocalFullSuiteProfile,
87
} from "../../scripts/lib/vitest-local-scheduling.mjs";
98

109
describe("vitest local full-suite profile", () => {
@@ -33,7 +32,7 @@ describe("vitest local full-suite profile", () => {
3332
});
3433
});
3534

36-
it("selects the large local profile on roomy hosts that are not throttled", () => {
35+
it("spends the host worker budget once across full-suite shards", () => {
3736
const env = {};
3837
const hostInfo = {
3938
cpuCount: 14,
@@ -46,38 +45,35 @@ describe("vitest local full-suite profile", () => {
4645
fileParallelism: true,
4746
throttledBySystem: false,
4847
});
49-
expect(shouldUseLargeLocalFullSuiteProfile(env, hostInfo)).toBe(true);
5048
expect(resolveLocalFullSuiteProfile(env, hostInfo)).toEqual({
51-
shardParallelism: 10,
52-
vitestMaxWorkers: 2,
49+
shardParallelism: 6,
50+
vitestMaxWorkers: 1,
5351
});
5452
});
5553

56-
it("keeps the smaller local profile when the host is already throttled", () => {
54+
it("reduces full-suite shard concurrency when the host is already throttled", () => {
5755
const hostInfo = {
5856
cpuCount: 14,
5957
loadAverage1m: 14,
6058
totalMemoryBytes: 48 * 1024 ** 3,
6159
freeMemoryBytes: 32 * 1024 ** 3,
6260
};
6361

64-
expect(shouldUseLargeLocalFullSuiteProfile({}, hostInfo)).toBe(false);
6562
expect(resolveLocalFullSuiteProfile({}, hostInfo)).toEqual({
66-
shardParallelism: 4,
63+
shardParallelism: 1,
6764
vitestMaxWorkers: 1,
6865
});
6966
});
7067

71-
it("never selects the large local profile in CI", () => {
68+
it("caps full-suite process fanout on the largest hosts", () => {
7269
const hostInfo = {
73-
cpuCount: 14,
70+
cpuCount: 64,
7471
loadAverage1m: 0,
75-
totalMemoryBytes: 48 * 1024 ** 3,
72+
totalMemoryBytes: 512 * 1024 ** 3,
7673
};
7774

78-
expect(shouldUseLargeLocalFullSuiteProfile({ CI: "true" }, hostInfo)).toBe(false);
79-
expect(resolveLocalFullSuiteProfile({ CI: "true" }, hostInfo)).toEqual({
80-
shardParallelism: 4,
75+
expect(resolveLocalFullSuiteProfile({}, hostInfo)).toEqual({
76+
shardParallelism: 10,
8177
vitestMaxWorkers: 1,
8278
});
8379
});

0 commit comments

Comments
 (0)