Skip to content

Commit d7a66f5

Browse files
Add bun test --shard=M/N for splitting tests across CI jobs (#29366)
## What does this PR do? Adds `bun test --shard=<index>/<count>` for splitting test files across multiple CI runners / GitHub Actions jobs. ```sh # In a matrix of 3 jobs: bun test --shard=1/3 bun test --shard=2/3 bun test --shard=3/3 ``` ``` --shard=2/3: running 3/10 test files f01.test.ts: (pass) t f04.test.ts: (pass) t f07.test.ts: (pass) t ``` ### How it works 1. Discovered test files are **sorted by path** so every machine sees the same ordering regardless of filesystem iteration order. 2. Files are distributed **round-robin**: file at sorted index `i` belongs to shard `(i % count) + 1`. This keeps shards balanced to within one file of each other. 3. The shard index is **1-based** (`1 <= index <= count`), matching Jest, Vitest, and Playwright. 4. Composes with `--changed` (sharding is applied after the changed-files filter) and `--randomize` (shuffle happens after shard selection, within the shard). 5. If a shard ends up with zero files (e.g. 2 test files, `--shard=5/5`), it exits `0` with a status line rather than erroring with "No tests found!". Invalid inputs (`0/3`, `4/3`, `1/0`, `abc`, `1/`, `/3`) produce a clear error and exit non-zero. ## How did you verify your code works? - `test/cli/test/test-shard.test.ts` — 13 tests covering partitioning (union == all files, intersection == ∅), determinism across runs, `1/1` running everything, the summary line, empty-shard exit code, and all invalid-input cases. - `bun bd test test/cli/test/test-changed.test.ts` still passes (touched the shared `pass_with_no_tests_from_filter` variable). - Gate: `0/13` pass without the `src/` changes, `13/13` with. Fixes #4587 --------- Co-authored-by: robobun <[email protected]> Co-authored-by: Jarred Sumner <[email protected]>
1 parent 36d3f60 commit d7a66f5

5 files changed

Lines changed: 296 additions & 5 deletions

File tree

‎completions/bun.zsh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,6 +696,7 @@ _bun_test_completion() {
696696
'--todo[Include tests that are marked with "test.todo()"]' \
697697
'--coverage[Generate a coverage profile]' \
698698
'--bail[Exit the test suite after <NUMBER> failures. If you do not specify a number, it defaults to 1.]:bail' \
699+
'--shard[Run a subset of test files, e.g. 1/3. Useful for splitting tests across CI jobs.]:shard' \
699700
'--test-name-pattern[Run only tests with a name that matches the given regex]:pattern' \
700701
'-t[Run only tests with a name that matches the given regex]:pattern' &&
701702
ret=0

‎src/cli.zig‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,10 @@ pub const Command = struct {
375375
/// Otherwise the value is a git ref (commit, branch, tag) to diff
376376
/// against.
377377
changed: ?[]const u8 = null,
378+
/// `bun test --shard=M/N`. When set, test files are sorted by path
379+
/// and only every Nth file (starting from M-1) is run. index is
380+
/// 1-based; both are validated at parse time so `1 <= index <= count`.
381+
shard: ?struct { index: u32, count: u32 } = null,
378382

379383
reporters: struct {
380384
dots: bool = false,

‎src/cli/Arguments.zig‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ pub const test_only_params = [_]ParamType{
247247
clap.parseParam("--parallel <NUMBER>? Run test files in parallel using N worker processes. Implies --isolate. Defaults to CPU core count.") catch unreachable,
248248
clap.parseParam("--parallel-delay <NUMBER> Milliseconds the first --parallel worker must be busy before spawning the rest. 0 spawns all immediately. Default 5.") catch unreachable,
249249
clap.parseParam("--test-worker (internal) Run as a --parallel worker, receiving files over IPC.") catch unreachable,
250+
clap.parseParam("--shard <STR> Run a subset of test files, e.g. '--shard=1/3' runs the first of three shards. Useful for splitting tests across multiple CI jobs.") catch unreachable,
250251
};
251252
pub const test_params = test_only_params ++ runtime_params_ ++ transpiler_params_ ++ base_params_;
252253

@@ -613,6 +614,31 @@ pub fn parse(allocator: std.mem.Allocator, ctx: Command.Context, comptime cmd: C
613614
if (args.option("--changed")) |since| {
614615
ctx.test_options.changed = since;
615616
}
617+
if (args.option("--shard")) |shard| {
618+
const sep = bun.strings.indexOfChar(shard, '/') orelse {
619+
Output.prettyErrorln("<r><red>error<r>: --shard expects <d>'<r>index/count<d>'<r>, e.g. --shard=1/3", .{});
620+
Global.exit(1);
621+
};
622+
const index_str = shard[0..sep];
623+
const count_str = shard[sep + 1 ..];
624+
const index = std.fmt.parseInt(u32, index_str, 10) catch {
625+
Output.prettyErrorln("<r><red>error<r>: --shard index must be a positive integer, got \"{s}\"", .{index_str});
626+
Global.exit(1);
627+
};
628+
const count = std.fmt.parseInt(u32, count_str, 10) catch {
629+
Output.prettyErrorln("<r><red>error<r>: --shard count must be a positive integer, got \"{s}\"", .{count_str});
630+
Global.exit(1);
631+
};
632+
if (count == 0) {
633+
Output.prettyErrorln("<r><red>error<r>: --shard count must be greater than 0", .{});
634+
Global.exit(1);
635+
}
636+
if (index == 0 or index > count) {
637+
Output.prettyErrorln("<r><red>error<r>: --shard index must be between 1 and {d}, got {d}", .{ count, index });
638+
Global.exit(1);
639+
}
640+
ctx.test_options.shard = .{ .index = index, .count = count };
641+
}
616642
ctx.test_options.update_snapshots = args.flag("--update-snapshots");
617643
ctx.test_options.run_todo = args.flag("--todo");
618644
ctx.test_options.only = args.flag("--only");

‎src/cli/test_command.zig‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1631,13 +1631,17 @@ pub const TestCommand = struct {
16311631
defer ctx.allocator.free(all_test_files);
16321632
const search_count = scanner.search_count;
16331633

1634-
var pass_with_no_tests_from_changed = false;
1634+
// When --changed or --shard filters the discovered test files
1635+
// down to zero, the "No tests found!" error path is suppressed
1636+
// and the run exits 0 — an empty shard or an unchanged tree
1637+
// is not a misconfiguration.
1638+
var pass_with_no_tests_from_filter = false;
16351639
var changed_module_graph_files: []const []const u8 = &.{};
16361640
defer {
16371641
for (changed_module_graph_files) |p| ctx.allocator.free(p);
16381642
ctx.allocator.free(changed_module_graph_files);
16391643
}
1640-
const test_files: []PathString = if (ctx.test_options.changed) |changed_since| brk: {
1644+
var test_files: []PathString = if (ctx.test_options.changed) |changed_since| brk: {
16411645
// If the Scanner found nothing, fall through to the existing
16421646
// "no tests found" error path rather than treating it as a
16431647
// --changed success.
@@ -1650,13 +1654,13 @@ pub const TestCommand = struct {
16501654
changed_module_graph_files = result.module_graph_files;
16511655
if (result.test_files.len == 0 and result.changed_count == 0) {
16521656
Output.prettyError("<r><d>--changed:<r> no changed files, nothing to run\n", .{});
1653-
pass_with_no_tests_from_changed = true;
1657+
pass_with_no_tests_from_filter = true;
16541658
} else if (result.test_files.len == 0) {
16551659
Output.prettyError(
16561660
"<r><d>--changed:<r> {d} changed file{s}, but no test files are affected\n",
16571661
.{ result.changed_count, if (result.changed_count == 1) "" else "s" },
16581662
);
1659-
pass_with_no_tests_from_changed = true;
1663+
pass_with_no_tests_from_filter = true;
16601664
} else {
16611665
Output.prettyError(
16621666
"<r><d>--changed:<r> {d} changed file{s}, running {d}/{d} test file{s}\n",
@@ -1673,6 +1677,52 @@ pub const TestCommand = struct {
16731677
break :brk result.test_files;
16741678
} else all_test_files;
16751679

1680+
// --shard=M/N: sort the test files for determinism, then keep only
1681+
// every Nth file starting at M-1. This round-robin distribution
1682+
// keeps shards roughly balanced regardless of how many files there
1683+
// are, and is stable across runs and machines as long as the set of
1684+
// test files is the same.
1685+
//
1686+
// Only runs when there are files to shard — if the scanner or
1687+
// --changed already produced an empty list, fall through to the
1688+
// existing "No tests found!" / --changed messaging rather than
1689+
// printing a confusing "running 0/0 test files".
1690+
if (ctx.test_options.shard) |shard| if (test_files.len > 0) {
1691+
std.sort.pdq(PathString, test_files, {}, struct {
1692+
pub fn lessThan(_: void, a: PathString, b: PathString) bool {
1693+
return strings.order(a.slice(), b.slice()) == .lt;
1694+
}
1695+
}.lessThan);
1696+
1697+
var write: usize = 0;
1698+
for (test_files, 0..) |file, i| {
1699+
if (i % shard.count == shard.index - 1) {
1700+
test_files[write] = file;
1701+
write += 1;
1702+
}
1703+
}
1704+
1705+
Output.prettyError(
1706+
"<r><d>--shard={d}/{d}:<r> running {d}/{d} test file{s}\n",
1707+
.{
1708+
shard.index,
1709+
shard.count,
1710+
write,
1711+
test_files.len,
1712+
if (test_files.len == 1) "" else "s",
1713+
},
1714+
);
1715+
Output.flush();
1716+
1717+
if (write == 0) {
1718+
// There were test files, but fewer than the shard count so
1719+
// this shard got none. That's fine — not a "no tests
1720+
// found" error.
1721+
pass_with_no_tests_from_filter = true;
1722+
}
1723+
test_files = test_files[0..write];
1724+
};
1725+
16761726
// Normally the watcher is only enabled when there are test files to
16771727
// run; `bun test --watch` with nothing matching should still exit.
16781728
// With --changed we always want to keep watching as long as any test
@@ -1775,7 +1825,7 @@ pub const TestCommand = struct {
17751825

17761826
var failed_to_find_any_tests = false;
17771827

1778-
if (test_files.len == 0 and !pass_with_no_tests_from_changed) {
1828+
if (test_files.len == 0 and !pass_with_no_tests_from_filter) {
17791829
failed_to_find_any_tests = true;
17801830

17811831
// "bun test" - positionals[0] == "test"

‎test/cli/test/test-shard.test.ts‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
import { describe, expect, test } from "bun:test";
2+
import { bunEnv, bunExe, tempDir } from "harness";
3+
4+
// `bun test --shard=M/N` splits discovered test files across N shards.
5+
// Files are sorted by path first for determinism, then distributed
6+
// round-robin: file i goes to shard (i % N) + 1.
7+
8+
function makeFixture(name: string, fileCount: number) {
9+
const files: Record<string, string> = {};
10+
for (let i = 0; i < fileCount; i++) {
11+
const id = String(i).padStart(2, "0");
12+
files[`f${id}.test.ts`] = `import { test } from "bun:test"; test("t", () => { console.log("RAN f${id}"); });`;
13+
}
14+
return tempDir(name, files);
15+
}
16+
17+
async function runShard(cwd: string, shard: string, extra: string[] = []) {
18+
await using proc = Bun.spawn({
19+
cmd: [bunExe(), "test", `--shard=${shard}`, ...extra],
20+
env: bunEnv,
21+
cwd,
22+
stdout: "pipe",
23+
stderr: "pipe",
24+
});
25+
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
26+
const ran = stdout
27+
.split("\n")
28+
.filter(l => l.startsWith("RAN "))
29+
.map(l => l.slice(4))
30+
.sort();
31+
return { ran, stderr, exitCode };
32+
}
33+
34+
describe.concurrent("--shard", () => {
35+
test("partitions test files across shards with no overlap or gaps", async () => {
36+
using dir = makeFixture("shard-partition", 10);
37+
const cwd = String(dir);
38+
39+
const results = await Promise.all(["1/3", "2/3", "3/3"].map(s => runShard(cwd, s)));
40+
for (const r of results) {
41+
expect(r.stderr).toContain("--shard=");
42+
expect(r.exitCode).toBe(0);
43+
}
44+
45+
const all = results.flatMap(r => r.ran).sort();
46+
// Every file ran exactly once across all shards.
47+
expect(all).toEqual(["f00", "f01", "f02", "f03", "f04", "f05", "f06", "f07", "f08", "f09"]);
48+
49+
// No overlap between shards.
50+
const seen = new Set<string>();
51+
for (const r of results) {
52+
for (const f of r.ran) {
53+
expect(seen.has(f)).toBe(false);
54+
seen.add(f);
55+
}
56+
}
57+
58+
// Round-robin over the sorted list: shard M gets indices M-1, M-1+N, ...
59+
expect(results[0].ran).toEqual(["f00", "f03", "f06", "f09"]);
60+
expect(results[1].ran).toEqual(["f01", "f04", "f07"]);
61+
expect(results[2].ran).toEqual(["f02", "f05", "f08"]);
62+
});
63+
64+
test("is deterministic across repeated runs", async () => {
65+
using dir = makeFixture("shard-determinism", 12);
66+
const cwd = String(dir);
67+
68+
const [a, b] = await Promise.all([runShard(cwd, "2/4"), runShard(cwd, "2/4")]);
69+
expect(a.ran).toEqual(["f01", "f05", "f09"]);
70+
expect(a.ran).toEqual(b.ran);
71+
expect(a.exitCode).toBe(0);
72+
expect(b.exitCode).toBe(0);
73+
});
74+
75+
test("composes with --randomize: shard selection is independent of the seed", async () => {
76+
// Shard selection sorts, picks, then --randomize shuffles only the
77+
// selected subset. This test verifies the SET of files in a shard
78+
// is unaffected by randomization — every seed (and no seed) yields
79+
// the same shard membership. Shuffle-order determinism under a
80+
// fixed seed is covered by test-randomize.test.ts.
81+
using dir = makeFixture("shard-randomize", 12);
82+
const cwd = String(dir);
83+
84+
const [plain, seeded1, seeded2, otherSeed] = await Promise.all([
85+
runShard(cwd, "2/4"),
86+
runShard(cwd, "2/4", ["--seed=123"]),
87+
runShard(cwd, "2/4", ["--seed=123"]),
88+
runShard(cwd, "2/4", ["--seed=999999"]),
89+
]);
90+
91+
expect(plain.ran).toEqual(["f01", "f05", "f09"]);
92+
expect(seeded1.ran).toEqual(plain.ran);
93+
expect(seeded2.ran).toEqual(plain.ran);
94+
expect(otherSeed.ran).toEqual(plain.ran);
95+
96+
expect(seeded1.stderr).toContain("--shard=2/4:");
97+
for (const r of [plain, seeded1, seeded2, otherSeed]) expect(r.exitCode).toBe(0);
98+
});
99+
100+
test("--shard=1/1 runs every test file", async () => {
101+
using dir = makeFixture("shard-one", 5);
102+
const cwd = String(dir);
103+
104+
const { ran, stderr, exitCode } = await runShard(cwd, "1/1");
105+
expect(stderr).toContain("--shard=1/1:");
106+
expect(ran).toEqual(["f00", "f01", "f02", "f03", "f04"]);
107+
expect(exitCode).toBe(0);
108+
});
109+
110+
test("prints the shard summary line", async () => {
111+
using dir = makeFixture("shard-summary", 7);
112+
const cwd = String(dir);
113+
114+
const { stderr, exitCode } = await runShard(cwd, "2/3");
115+
expect(stderr).toMatch(/--shard=2\/3: running \d+\/7 test files/);
116+
expect(exitCode).toBe(0);
117+
});
118+
119+
test("does not print shard summary when there are no test files", async () => {
120+
// With no test files at all, --shard should stay out of the way and
121+
// let the normal "No tests found!" error path handle it.
122+
using dir = tempDir("shard-no-files", {
123+
"not-a-test.ts": "export const x = 1;",
124+
});
125+
await using proc = Bun.spawn({
126+
cmd: [bunExe(), "test", "--shard=1/3"],
127+
env: bunEnv,
128+
cwd: String(dir),
129+
stdout: "pipe",
130+
stderr: "pipe",
131+
});
132+
const [, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
133+
expect(stderr).not.toContain("--shard=");
134+
expect(stderr).not.toContain("0/0");
135+
expect(stderr.toLowerCase()).toContain("no tests found");
136+
expect(exitCode).not.toBe(0);
137+
});
138+
139+
test("empty shard exits 0 without 'No tests found'", async () => {
140+
// 2 files, 5 shards → shards 3, 4, 5 get nothing.
141+
using dir = makeFixture("shard-empty", 2);
142+
const cwd = String(dir);
143+
144+
const { ran, stderr, exitCode } = await runShard(cwd, "5/5");
145+
expect(ran).toEqual([]);
146+
expect(stderr).toContain("--shard=5/5:");
147+
expect(stderr).toContain("running 0/2 test files");
148+
expect(stderr).not.toContain("No tests found");
149+
expect(stderr).not.toContain("did not match any test files");
150+
expect(exitCode).toBe(0);
151+
});
152+
153+
test.each([
154+
["0/3", "index must be between 1 and 3"],
155+
["4/3", "index must be between 1 and 3"],
156+
["1/0", "count must be greater than 0"],
157+
["abc", "expects"],
158+
["1/", "count must be a positive integer"],
159+
["/3", "index must be a positive integer"],
160+
["a/3", "index must be a positive integer"],
161+
["1/b", "count must be a positive integer"],
162+
])("rejects invalid --shard=%s", async (arg, needle) => {
163+
using dir = makeFixture("shard-invalid", 1);
164+
await using proc = Bun.spawn({
165+
cmd: [bunExe(), "test", `--shard=${arg}`],
166+
env: bunEnv,
167+
cwd: String(dir),
168+
stdout: "pipe",
169+
stderr: "pipe",
170+
});
171+
const [, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
172+
expect(stderr).toContain(needle);
173+
expect(exitCode).not.toBe(0);
174+
});
175+
176+
test("composes with --parallel: shard filters first, then workers run the subset", async () => {
177+
// 4 files; --shard=1/2 keeps f00 and f02. --parallel runs that subset.
178+
// This proves the file filter happens before the coordinator distributes;
179+
// worker distribution itself is covered by the JEST_WORKER_ID test in
180+
// parallel.test.ts.
181+
const files: Record<string, string> = {};
182+
for (let i = 0; i < 4; i++) {
183+
const id = String(i).padStart(2, "0");
184+
files[`f${id}.test.ts`] =
185+
`import {test} from "bun:test"; test("t", () => console.log("RAN f${id} WID="+process.env.JEST_WORKER_ID));`;
186+
}
187+
using dir = tempDir("shard-parallel", files);
188+
await using proc = Bun.spawn({
189+
cmd: [bunExe(), "test", "--shard=1/2", "--parallel=2"],
190+
env: { ...bunEnv, BUN_TEST_PARALLEL_SCALE_MS: "0" },
191+
cwd: String(dir),
192+
stdout: "pipe",
193+
stderr: "pipe",
194+
});
195+
const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
196+
const out = stdout + stderr;
197+
expect(stdout).toContain("PARALLEL");
198+
expect(stderr).toContain("--shard=1/2:");
199+
expect(stderr).toContain("running 2/4 test files");
200+
const ran = [...out.matchAll(/RAN (f\d\d) WID=(\S+)/g)].map(m => ({ file: m[1], wid: m[2] }));
201+
// Only shard-1 files ran, in any worker:
202+
expect(ran.map(r => r.file).sort()).toEqual(["f00", "f02"]);
203+
// JEST_WORKER_ID is the local worker (1..K), never undefined and never the
204+
// shard index. With 2 sharded files and lazy spawn, K may collapse to 1.
205+
for (const r of ran) {
206+
expect(["1", "2"]).toContain(r.wid);
207+
}
208+
expect(exitCode).toBe(0);
209+
});
210+
});

0 commit comments

Comments
 (0)