Skip to content

Commit aa247c5

Browse files
perf(ci): cache server tests individually across shards (#24966)
1 parent 557db6c commit aa247c5

8 files changed

Lines changed: 508 additions & 1 deletion

File tree

‎.github/workflows/ci.yml‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -420,7 +420,18 @@ jobs:
420420
env:
421421
T3CODE_TRANSFER_BUDGET_REPORT_PATH: ${{ runner.temp }}/t3code-transfer-budget.md
422422
T3CODE_TRANSFER_BUDGET_RESULT_PATH: ${{ runner.temp }}/thread-transfer-result.json
423-
run: vp run --filter t3 test --shard ${{ matrix.shard }}/${{ strategy.job-total }}
423+
run: >-
424+
vp run test:cached -- --shard ${{ matrix.shard }}/${{ strategy.job-total }}
425+
--uncached apps/server/integration/transferBudgetV2.integration.test.ts
426+
--verify-hits 1 --report "${{ runner.temp }}/test-cache-${{ matrix.shard }}.json" apps/server
427+
428+
- name: Upload test cache measurements
429+
if: always() && steps.changes.outputs.test_server != 'false'
430+
uses: actions/upload-artifact@v7
431+
with:
432+
name: test-cache-${{ matrix.shard }}
433+
path: ${{ runner.temp }}/test-cache-${{ matrix.shard }}.json
434+
if-no-files-found: error
424435

425436
# integration/transferBudgetV2.integration.test.ts writes the report; exactly one shard
426437
# produces these files. Gating the upload on their presence keeps a

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
"knip:production": "knip --production --preprocessor ./scripts/knip-schemas.ts",
3434
"lint:mobile": "node scripts/mobile-native-static-check.ts",
3535
"test": "vp run -r test",
36+
"test:cached": "node scripts/test-cached.ts",
3637
"test:perf:v2-wire": "vp test run apps/server/src/orchestration-v2/WireProjection.test.ts apps/server/src/orchestration-v2/threadHistoryPaging.test.ts apps/server/src/orchestration-v2/ThreadStream.test.ts apps/server/src/orchestration-v2/ThreadTransportPerformance.test.ts apps/server/src/orchestration-v2/ShellStream.test.ts apps/server/src/orchestration-v2/CommandPolicy.test.ts apps/server/src/orchestration-v2/runtimeLayer.test.ts apps/server/src/ws.test.ts packages/contracts/src/orchestrationV2.test.ts packages/client-runtime/src/operations/commands.test.ts packages/client-runtime/src/operations/commands.performance.test.ts packages/client-runtime/src/state/boundedThreadSnapshotHttp.test.ts packages/client-runtime/src/state/threadHistoryController.test.ts packages/client-runtime/src/state/threadHistoryMerge.test.ts packages/client-runtime/src/state/threads-sync.test.ts packages/client-runtime/src/state/shellReducer.test.ts",
3738
"test:resource-monitor": "cargo test --locked --manifest-path native/resource-monitor/Cargo.toml",
3839
"test:desktop-smoke": "vp run --filter @t3tools/desktop smoke-test",

‎scripts/lib/cached-tests.test.ts‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
// @effect-diagnostics nodeBuiltinImport:off - verifies native task caching in isolated subprocesses.
2+
import { describe, expect, it } from "vite-plus/test";
3+
import * as NodeChildProcess from "node:child_process";
4+
import * as NodeFSP from "node:fs/promises";
5+
import * as NodeOS from "node:os";
6+
import * as NodePath from "node:path";
7+
import * as NodeURL from "node:url";
8+
9+
import { cachedTestTasks, matchesTestPath } from "./cached-tests.ts";
10+
11+
describe("cachedTestTasks", () => {
12+
it("selects every test for a checkout-root path", () => {
13+
const root = NodePath.resolve(import.meta.dirname, "../..");
14+
for (const path of [".", root]) {
15+
expect(
16+
matchesTestPath("apps/server/src/auth/utils.test.ts", [
17+
NodePath.relative(root, NodePath.resolve(root, path)),
18+
]),
19+
).toBe(true);
20+
}
21+
expect(matchesTestPath("apps/server/src/auth/utils.test.ts", ["apps/server"])).toBe(true);
22+
expect(matchesTestPath("apps/server-other/a.test.ts", ["apps/server"])).toBe(false);
23+
});
24+
it("keeps a file's cache task stable when unrelated tests are added", () => {
25+
const first = cachedTestTasks(JSON.stringify({ files: ["src/b.test.ts"], groupSize: 1 }));
26+
const second = cachedTestTasks(
27+
JSON.stringify({ files: ["src/a.test.ts", "src/b.test.ts"], groupSize: 1 }),
28+
);
29+
const key = Object.keys(first).find((key) => key !== "test-cache")!;
30+
expect(second[key]).toEqual(first[key]);
31+
});
32+
33+
it("keeps shell metacharacters in filenames out of the command", () => {
34+
const tasks = cachedTestTasks(
35+
JSON.stringify({ files: ["src/a'$(touch injected).test.ts"], groupSize: 1 }),
36+
);
37+
const task = Object.entries(tasks).find(([key]) => key !== "test-cache")![1];
38+
expect(task.command).toBe("vp test run 'src/a'\"'\"'$(touch injected).test.ts'");
39+
});
40+
41+
it("rejects paths outside the checkout and empty groups", () => {
42+
for (const request of [
43+
{ files: ["../elsewhere.test.ts"], groupSize: 1 },
44+
{ files: ["/absolute.test.ts"], groupSize: 1 },
45+
{ files: ["src/a.test.ts"], groupSize: 0 },
46+
]) {
47+
expect(() => cachedTestTasks(JSON.stringify(request))).toThrow();
48+
}
49+
});
50+
51+
it("accepts spec files discovered by Vitest", () => {
52+
const tasks = cachedTestTasks(JSON.stringify({ files: ["src/example.spec.ts"], groupSize: 1 }));
53+
expect(
54+
Object.values(tasks).some((task) => task.command === "vp test run 'src/example.spec.ts'"),
55+
).toBe(true);
56+
});
57+
58+
it("does not add tasks without an explicit request", () => {
59+
expect(cachedTestTasks(undefined)).toEqual({});
60+
});
61+
62+
it("reuses individual results, invalidates filesystem inputs, and never caches failures", async () => {
63+
const directory = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "cached-tests-"));
64+
const root = NodePath.resolve(import.meta.dirname, "../..");
65+
const run = (files = ["a.test.ts", "b.test.ts"], uncached: Array<string> = []) =>
66+
new Promise<number | null>((resolve, reject) => {
67+
const child = NodeChildProcess.spawn(
68+
process.execPath,
69+
[NodePath.join(root, "node_modules/vite-plus/dist/bin.js"), "run", "test-cache"],
70+
{
71+
cwd: directory,
72+
env: {
73+
...process.env,
74+
NODE_OPTIONS: "",
75+
VP_REMOTE_CACHE: "off",
76+
T3_CACHED_TESTS: JSON.stringify({ files, groupSize: 1, uncached }),
77+
VITE_CACHE_PATH: NodePath.join(directory, "cache"),
78+
PATH: `${NodePath.join(root, "node_modules/.bin")}${NodePath.delimiter}${process.env.PATH}`,
79+
},
80+
stdio: "ignore",
81+
},
82+
);
83+
child.once("error", reject);
84+
child.once("close", resolve);
85+
});
86+
const summary = async () => {
87+
const value: { tasks: Array<{ result: Record<string, unknown> }> } = JSON.parse(
88+
await NodeFSP.readFile(NodePath.join(directory, "cache/v18/last-summary.json"), "utf8"),
89+
);
90+
return value.tasks.filter((task) => "CacheHit" in task.result).length;
91+
};
92+
try {
93+
await NodeFSP.mkdir(NodePath.join(directory, "node_modules"));
94+
await NodeFSP.symlink(
95+
NodePath.join(root, "node_modules/vite-plus"),
96+
NodePath.join(directory, "node_modules/vite-plus"),
97+
"dir",
98+
);
99+
await NodeFSP.writeFile(NodePath.join(directory, "package.json"), '{"type":"module"}');
100+
await NodeFSP.writeFile(
101+
NodePath.join(directory, "vite.config.ts"),
102+
`import {cachedTestTasks} from ${JSON.stringify(NodeURL.pathToFileURL(NodePath.join(root, "scripts/lib/cached-tests.ts")).href)}; export default {run:{tasks:cachedTestTasks(process.env.T3_CACHED_TESTS, import.meta.dirname)},test:{include:["*.test.ts"]}};`,
103+
);
104+
await NodeFSP.writeFile(
105+
NodePath.join(directory, "a.test.ts"),
106+
'import {test,expect} from "vite-plus/test"; import {readFileSync} from "node:fs"; test("fixture",()=>expect(readFileSync("fixture.txt","utf8")).toMatch(/^hello/));',
107+
);
108+
await NodeFSP.writeFile(
109+
NodePath.join(directory, "b.test.ts"),
110+
'import {test,expect} from "vite-plus/test"; test("independent",()=>expect(1).toBe(1));',
111+
);
112+
await NodeFSP.writeFile(NodePath.join(directory, "fixture.txt"), "hello");
113+
expect(await run()).toBe(0);
114+
expect(await run()).toBe(0);
115+
expect(await summary()).toBe(2);
116+
expect(await run(["b.test.ts"])).toBe(0);
117+
expect(await summary()).toBe(1);
118+
expect(await run()).toBe(0);
119+
expect(await summary()).toBe(2);
120+
await NodeFSP.writeFile(NodePath.join(directory, "fixture.txt"), "hello changed");
121+
expect(await run()).toBe(0);
122+
expect(await summary()).toBe(1);
123+
await NodeFSP.writeFile(NodePath.join(directory, "fixture.txt"), "broken");
124+
expect(await run()).toBe(1);
125+
expect(await run()).toBe(1);
126+
expect(await summary()).toBe(1);
127+
await NodeFSP.writeFile(NodePath.join(directory, "fixture.txt"), "hello again");
128+
await NodeFSP.writeFile(
129+
NodePath.join(directory, "b.test.ts"),
130+
'import {test} from "vite-plus/test";import {writeFileSync} from "node:fs";test("report",()=>writeFileSync("report.json","{}"));',
131+
);
132+
expect(await run(undefined, ["b.test.ts"])).toBe(0);
133+
await NodeFSP.unlink(NodePath.join(directory, "report.json"));
134+
expect(await run(undefined, ["b.test.ts"])).toBe(0);
135+
expect(await NodeFSP.readFile(NodePath.join(directory, "report.json"), "utf8")).toBe("{}");
136+
} finally {
137+
await NodeFSP.rm(directory, { recursive: true, force: true });
138+
}
139+
});
140+
});

‎scripts/lib/cached-tests.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// @effect-diagnostics nodeBuiltinImport:off - Vite loads this before dependency installation and service initialization.
2+
import * as NodeCrypto from "node:crypto";
3+
import * as NodeFS from "node:fs";
4+
import * as NodePath from "node:path";
5+
6+
interface CachedTestTask {
7+
command: string;
8+
cwd?: string;
9+
dependsOn?: Array<string>;
10+
cache:
11+
| false
12+
| {
13+
input: Array<string | { auto: true }>;
14+
output: Array<string>;
15+
};
16+
}
17+
18+
export function matchesTestPath(file: string, filters: Array<string>) {
19+
return (
20+
filters.length === 0 ||
21+
filters.some((filter) => filter === "" || file === filter || file.startsWith(`${filter}/`))
22+
);
23+
}
24+
25+
export function cachedTestTasks(
26+
raw: string | undefined,
27+
root = NodePath.resolve(import.meta.dirname, "../.."),
28+
): Record<string, CachedTestTask> {
29+
if (!raw) return {};
30+
const request: unknown = JSON.parse(raw);
31+
if (
32+
typeof request !== "object" ||
33+
request === null ||
34+
!("files" in request) ||
35+
!Array.isArray(request.files) ||
36+
!("groupSize" in request) ||
37+
typeof request.groupSize !== "number" ||
38+
!Number.isSafeInteger(request.groupSize) ||
39+
request.groupSize < 1
40+
) {
41+
throw new Error("Invalid cached test request");
42+
}
43+
const uncached = "uncached" in request ? request.uncached : [];
44+
if (!Array.isArray(uncached) || !uncached.every((file) => typeof file === "string")) {
45+
throw new Error("Invalid uncached test files");
46+
}
47+
const files: Array<string> = [];
48+
for (const file of request.files) {
49+
if (
50+
typeof file !== "string" ||
51+
NodePath.isAbsolute(file) ||
52+
file.split(/[\\/]/).includes("..") ||
53+
!/\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)
54+
) {
55+
throw new Error("Cached tests must be test files inside the checkout");
56+
}
57+
files.push(file);
58+
}
59+
const tasks: Record<string, CachedTestTask> = {};
60+
const sorted = [...new Set(files)].sort();
61+
const packages = new Map<string, Array<string>>();
62+
for (const file of sorted) {
63+
let directory = NodePath.dirname(NodePath.join(root, file));
64+
while (directory !== root && !NodeFS.existsSync(NodePath.join(directory, "package.json"))) {
65+
directory = NodePath.dirname(directory);
66+
}
67+
const entries = packages.get(directory) ?? [];
68+
entries.push(file);
69+
packages.set(directory, entries);
70+
}
71+
for (const [directory, entries] of packages) {
72+
const manifest: unknown = JSON.parse(
73+
NodeFS.readFileSync(NodePath.join(directory, "package.json"), "utf8"),
74+
);
75+
const command =
76+
directory !== root &&
77+
typeof manifest === "object" &&
78+
manifest !== null &&
79+
"scripts" in manifest &&
80+
typeof manifest.scripts === "object" &&
81+
manifest.scripts !== null &&
82+
"test" in manifest.scripts &&
83+
typeof manifest.scripts.test === "string"
84+
? manifest.scripts.test
85+
: "vp test run";
86+
if (!command.startsWith("vp test run")) {
87+
throw new Error(`Unsupported test command in ${directory}`);
88+
}
89+
const cacheable = entries.filter((file) => !uncached.includes(file));
90+
const groups: Array<Array<string>> = [];
91+
for (let index = 0; index < cacheable.length; index += request.groupSize) {
92+
groups.push(cacheable.slice(index, index + request.groupSize));
93+
}
94+
groups.push(...entries.filter((file) => uncached.includes(file)).map((file) => [file]));
95+
for (const group of groups) {
96+
const key = NodeCrypto.createHash("sha256").update(group.join("\0")).digest("hex");
97+
tasks[`test-cache:${key}`] = {
98+
command: `${command} ${group.map((file) => `'${NodePath.relative(directory, NodePath.join(root, file)).replaceAll("'", "'\"'\"'")}'`).join(" ")}`,
99+
...(directory === root ? {} : { cwd: NodePath.relative(root, directory) }),
100+
cache: uncached.includes(group[0]!)
101+
? false
102+
: {
103+
// Vitest reads and rewrites these caches while running otherwise unchanged tests.
104+
input: [
105+
{ auto: true },
106+
"!**/node_modules/.vite/**",
107+
"!**/node_modules/.vitest-cache/**",
108+
],
109+
output: [],
110+
},
111+
};
112+
}
113+
}
114+
return {
115+
...tasks,
116+
"test-cache": { command: 'node -e ""', dependsOn: Object.keys(tasks), cache: false },
117+
};
118+
}

‎scripts/lib/sharded-tests.test.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// @effect-diagnostics nodeBuiltinImport:off - verifies Vitest discovery in a disposable project.
2+
import { expect, it } from "vite-plus/test";
3+
import * as NodeFSP from "node:fs/promises";
4+
import * as NodeOS from "node:os";
5+
import * as NodePath from "node:path";
6+
import { shardedTestFiles } from "./sharded-tests.ts";
7+
8+
it("uses the configured sequencer and discovers every file exactly once", async () => {
9+
const root = await NodeFSP.realpath(
10+
await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "test-shards-")),
11+
);
12+
try {
13+
await NodeFSP.mkdir(NodePath.join(root, "node_modules"));
14+
await NodeFSP.symlink(
15+
NodePath.resolve(import.meta.dirname, "../../node_modules/vite-plus"),
16+
NodePath.join(root, "node_modules/vite-plus"),
17+
"dir",
18+
);
19+
await NodeFSP.writeFile(NodePath.join(root, "package.json"), '{"type":"module"}');
20+
await NodeFSP.writeFile(
21+
NodePath.join(root, "vite.config.ts"),
22+
`import {BaseSequencer} from "vite-plus/test/node";
23+
class Sequencer extends BaseSequencer {
24+
async shard(files) {
25+
const {index,count}=this.ctx.config.shard;
26+
return files.sort((a,b)=>a.moduleId.localeCompare(b.moduleId)).filter((_,i)=>i%count===index-1);
27+
}
28+
}
29+
export default {test:{include:["included/*.{test,spec}.ts"],sequence:{sequencer:Sequencer}}};`,
30+
);
31+
await NodeFSP.mkdir(NodePath.join(root, "included"));
32+
for (const file of [
33+
"included/a.test.ts",
34+
"included/b.test.ts",
35+
"included/c.test.ts",
36+
"included/d.spec.ts",
37+
"excluded.test.ts",
38+
]) {
39+
await NodeFSP.writeFile(NodePath.join(root, file), "export {};");
40+
}
41+
const first = await shardedTestFiles(root, "1/2");
42+
const second = await shardedTestFiles(root, "2/2");
43+
expect(first).toEqual(
44+
["a", "c"].map((file) => NodePath.join(root, `included/${file}.test.ts`)),
45+
);
46+
expect(second).toEqual([
47+
NodePath.join(root, "included/b.test.ts"),
48+
NodePath.join(root, "included/d.spec.ts"),
49+
]);
50+
expect(new Set([...first, ...second]).size).toBe(4);
51+
await expect(shardedTestFiles(root, "0/2")).rejects.toThrow("Shard must be");
52+
} finally {
53+
await NodeFSP.rm(root, { recursive: true, force: true });
54+
}
55+
});

‎scripts/lib/sharded-tests.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// @effect-diagnostics nodeBuiltinImport:off - Vitest discovery runs outside the application runtime.
2+
import * as NodePath from "node:path";
3+
import { createVitest } from "vite-plus/test/node";
4+
5+
export async function shardedTestFiles(root: string, shard: string) {
6+
const match = /^(\d+)\/(\d+)$/.exec(shard);
7+
const index = Number(match?.[1]);
8+
const count = Number(match?.[2]);
9+
if (!Number.isSafeInteger(index) || !Number.isSafeInteger(count) || index < 1 || index > count) {
10+
throw new Error("Shard must be index/count with 1 <= index <= count");
11+
}
12+
const context = await createVitest("test", { root, watch: false, shard });
13+
try {
14+
const files = await context.globTestSpecifications();
15+
const sequencer = new context.config.sequence.sequencer(context);
16+
const selected = await sequencer.shard(files);
17+
return selected.map((file) => NodePath.resolve(file.moduleId));
18+
} finally {
19+
await context.close();
20+
}
21+
}

0 commit comments

Comments
 (0)