Skip to content

Commit fc4297a

Browse files
edelaunaedelauna
committed
ci: stabilize extension test cache inputs
1 parent 1f4f1d3 commit fc4297a

2 files changed

Lines changed: 218 additions & 106 deletions

File tree

‎src/scripts/verify-coverage-cache-inputs.mjs‎

Lines changed: 144 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,42 @@ if (!pnpm) throw new Error("pnpm executable path is unavailable")
1111
const command = process.platform === "win32" ? process.execPath : pnpm
1212
const args = process.platform === "win32" ? [pnpm] : []
1313
const lanes = ["api", "core", "services", "misc", "tree-sitter"]
14+
const taskNames = lanes.flatMap((lane) => [`test:${lane}`, `test:coverage:${lane}`])
15+
// Data files owned by one lane's test area. A change here must invalidate only
16+
// the owner, because no test or source file imports across lane test areas.
17+
const laneOwnedDataFiles = {
18+
api: "src/api/providers/fetchers/__tests__/fixtures/ollama-model-details.json",
19+
core: "src/core/prompts/__tests__/__snapshots__/system-prompt/consistent-system-prompt.snap",
20+
misc: "src/__tests__/helpers/provider-stub.ts",
21+
"tree-sitter": "src/services/tree-sitter/__tests__/fixtures/sample-json.ts",
22+
}
23+
// Directories where a lane's own test run may create transient files. Litter
24+
// here must not change any other lane's hash.
25+
const laneProbeDirs = {
26+
api: "src/api/__tests__",
27+
core: "src/core/task/__tests__",
28+
services: "src/services/mcp/__tests__",
29+
misc: "src/integrations/misc/__tests__",
30+
"tree-sitter": "src/services/tree-sitter/__tests__",
31+
}
32+
const laneOwnedSpecFiles = {
33+
api: "src/api/providers/__tests__/anthropic.spec.ts",
34+
core: "src/core/prompts/__tests__/system-prompt.spec.ts",
35+
services: "src/services/mcp/__tests__/McpHub.spec.ts",
36+
misc: "src/utils/__tests__/safeWriteJson.test.ts",
37+
"tree-sitter": "src/services/tree-sitter/__tests__/wasm.spec.ts",
38+
}
1439
let probeRoot
1540

41+
// Probe the working tree so local runs validate uncommitted task edits. CI
42+
// runs on a clean tree, where this falls back to HEAD.
43+
const worktreeCommit = () => {
44+
const result = spawnSync("git", ["stash", "create"], { cwd: root, encoding: "utf8" })
45+
if (result.status !== 0) return "HEAD"
46+
const sha = result.stdout.trim()
47+
return sha || "HEAD"
48+
}
49+
1650
const git = (gitArgs) => {
1751
const result = spawnSync("git", gitArgs, { cwd: root, encoding: "utf8" })
1852
if (result.status !== 0) {
@@ -21,35 +55,36 @@ const git = (gitArgs) => {
2155
}
2256
}
2357

24-
const coverageTasks = () => {
58+
const turboTasks = () => {
2559
const result = spawnSync(
2660
command,
27-
[
28-
...args,
29-
"turbo",
30-
"--cwd",
31-
probeRoot,
32-
"run",
33-
...lanes.map((lane) => `test:coverage:${lane}`),
34-
"--filter=zoo-code",
35-
"--dry=json",
36-
"--no-daemon",
37-
],
61+
[...args, "turbo", "--cwd", probeRoot, "run", ...taskNames, "--filter=zoo-code", "--dry=json", "--no-daemon"],
3862
{ cwd: root, encoding: "utf8" },
3963
)
4064
if (result.status !== 0) {
4165
const details = [result.error?.message, result.signal, result.stderr, result.stdout].filter(Boolean).join("\n")
4266
throw new Error(details || `pnpm exited with status ${result.status ?? "unknown"}`)
4367
}
44-
const graph = JSON.parse(result.stdout)
45-
return lanes.map((lane) => {
46-
const task = graph.tasks.find(({ taskId }) => taskId === `zoo-code#test:coverage:${lane}`)
47-
if (!task) throw new Error(`Coverage lane missing from Turbo graph: ${lane}`)
68+
// pnpm may print warnings before the Turbo graph. The graph is the last
69+
// top-level JSON object in the output.
70+
const lines = result.stdout.split("\n")
71+
let jsonStart = -1
72+
for (let i = 0; i < lines.length; i++) {
73+
if (lines[i].startsWith("{")) jsonStart = i
74+
}
75+
if (jsonStart < 0) {
76+
const details = [result.error?.message, result.stderr, result.stdout].filter(Boolean).join("\n")
77+
throw new Error(details || "turbo dry run produced no JSON graph")
78+
}
79+
const graph = JSON.parse(lines.slice(jsonStart).join("\n"))
80+
return taskNames.map((taskName) => {
81+
const task = graph.tasks.find(({ taskId }) => taskId === `zoo-code#${taskName}`)
82+
if (!task) throw new Error(`Extension lane missing from Turbo graph: ${taskName}`)
4883
return task
4984
})
5085
}
5186

52-
const hashes = () => Object.fromEntries(coverageTasks().map((task) => [task.task.split(":").at(-1), task.hash]))
87+
const hashes = () => Object.fromEntries(turboTasks().map((task) => [task.task, task.hash]))
5388

5489
const withChangedFiles = (paths, run) => {
5590
const originals = paths.map((path) => [path, readFileSync(resolve(probeRoot, path), "utf8")])
@@ -62,7 +97,26 @@ const withChangedFiles = (paths, run) => {
6297
}
6398
}
6499

65-
const changedLanes = (before, after) => lanes.filter((lane) => before[lane] !== after[lane])
100+
const withCreatedFile = (path, run) => {
101+
const target = resolve(probeRoot, path)
102+
writeFileSync(target, "// cache-input-test\n")
103+
try {
104+
return run()
105+
} finally {
106+
rmSync(target, { force: true })
107+
}
108+
}
109+
110+
const changedTasks = (before, after) => taskNames.filter((task) => before[task] !== after[task])
111+
112+
const expectInvalidatesExactly = (task, changed) => {
113+
const expected = taskNames.filter((name) => name === task || name === `test:coverage:${task.slice(5)}`)
114+
const actual = changed.sort()
115+
if (actual.join(",") !== expected.sort().join(","))
116+
throw new Error(
117+
`${task} probe invalidated [${actual.join(", ") || "nothing"}] instead of [${expected.join(", ")}]`,
118+
)
119+
}
66120

67121
test("coverage cache input contract", async (context) => {
68122
probeRoot = mkdtempSync(resolve(tmpdir(), "zoo-code-coverage-cache-inputs-"))
@@ -87,13 +141,61 @@ test("coverage cache input contract", async (context) => {
87141
process.once("SIGTERM", onSigterm)
88142

89143
try {
90-
git(["worktree", "add", "--detach", probeRoot, "HEAD"])
144+
git(["worktree", "add", "--detach", probeRoot, worktreeCommit()])
91145
worktreeAdded = true
92146

93-
await context.test("coverage lane hashes ignore post-coverage verifier implementation", () => {
147+
await context.test("plain and coverage lanes share identical input sets", () => {
148+
const tasks = turboTasks()
149+
const inputsByTask = Object.fromEntries(tasks.map((task) => [task.task, task.inputs]))
150+
for (const lane of lanes) {
151+
const plain = Object.keys(inputsByTask[`test:${lane}`]).sort()
152+
const coverage = Object.keys(inputsByTask[`test:coverage:${lane}`]).sort()
153+
const extraInPlain = plain.filter((path) => !coverage.includes(path))
154+
const extraInCoverage = coverage.filter((path) => !plain.includes(path))
155+
if (extraInPlain.length > 0 || extraInCoverage.length > 0)
156+
throw new Error(
157+
`test:${lane} and test:coverage:${lane} input sets differ. ` +
158+
`Only in plain: [${extraInPlain.join(", ")}]. Only in coverage: [${extraInCoverage.join(", ")}]`,
159+
)
160+
}
161+
})
162+
163+
await context.test("lanes hash required inputs and exclude foreign test areas", () => {
164+
const tasks = turboTasks()
165+
for (const lane of lanes) {
166+
for (const taskName of [`test:${lane}`, `test:coverage:${lane}`]) {
167+
const task = tasks.find(({ task }) => task === taskName)
168+
const inputs = task.inputs
169+
const required = [
170+
"vitest.config.ts",
171+
`vitest.${lane}.config.ts`,
172+
"vitest.setup.ts",
173+
"utils/vitest-verbosity.ts",
174+
"__mocks__/vscode.js",
175+
"package.json",
176+
"tsconfig.json",
177+
]
178+
for (const path of required) {
179+
if (!Object.hasOwn(inputs, path))
180+
throw new Error(`${taskName} does not hash required input ${path}`)
181+
}
182+
for (const [owner, file] of Object.entries(laneOwnedSpecFiles)) {
183+
const packagePath = file.slice("src/".length)
184+
if (owner === lane) {
185+
if (!Object.hasOwn(inputs, packagePath))
186+
throw new Error(`${taskName} does not hash its own spec ${packagePath}`)
187+
} else if (Object.hasOwn(inputs, packagePath)) {
188+
throw new Error(`${taskName} hashes foreign spec ${packagePath}`)
189+
}
190+
}
191+
}
192+
}
193+
})
194+
195+
await context.test("lane hashes ignore post-coverage verifier implementation", () => {
94196
const before = hashes()
95197
const self = "scripts/verify-coverage-cache-inputs.mjs"
96-
for (const task of coverageTasks()) {
198+
for (const task of turboTasks()) {
97199
if (Object.hasOwn(task.inputs, self)) throw new Error(`${self} is an input of ${task.taskId}`)
98200
}
99201
for (const path of [
@@ -102,27 +204,37 @@ test("coverage cache input contract", async (context) => {
102204
"src/scripts/verify-lcov.mjs",
103205
]) {
104206
const after = withChangedFiles([path], hashes)
105-
const changed = changedLanes(before, after)
106-
if (changed.length !== 0) throw new Error(`${path} invalidated coverage lanes: ${changed.join(", ")}`)
207+
const changed = changedTasks(before, after)
208+
if (changed.length !== 0) throw new Error(`${path} invalidated extension lanes: ${changed.join(", ")}`)
107209
}
108210
})
109211

110-
await context.test("shared production changes invalidate every coverage lane that can import them", () => {
212+
await context.test("shared production changes invalidate every extension lane", () => {
111213
const before = hashes()
112214
const after = withChangedFiles(["src/utils/path.ts"], hashes)
113-
const changed = changedLanes(before, after)
215+
const changed = changedTasks(before, after)
114216

115-
if (changed.join(",") !== lanes.join(","))
217+
if (changed.join(",") !== taskNames.join(","))
116218
throw new Error(`Shared production change invalidated ${changed.join(", ") || "no lanes"}`)
117219
})
118220

119-
await context.test("lane-owned tests invalidate only their general coverage lane", () => {
120-
const before = hashes()
121-
const after = withChangedFiles(["src/api/providers/__tests__/anthropic.spec.ts"], hashes)
122-
const changed = changedLanes(before, after)
221+
await context.test("lane-owned data files invalidate only their owner", () => {
222+
for (const [lane, file] of Object.entries(laneOwnedDataFiles)) {
223+
const before = hashes()
224+
const after = withChangedFiles([file], hashes)
225+
const changed = changedTasks(before, after)
226+
expectInvalidatesExactly(`test:${lane}`, changed)
227+
}
228+
})
123229

124-
if (changed.join(",") !== "api")
125-
throw new Error(`API test change invalidated ${changed.join(", ") || "no lanes"}`)
230+
await context.test("transient files in a lane test area invalidate only that lane", () => {
231+
for (const [lane, dir] of Object.entries(laneProbeDirs)) {
232+
const probe = `${dir}/cache-boundary-probe.tmp`
233+
const before = hashes()
234+
const after = withCreatedFile(probe, hashes)
235+
const changed = changedTasks(before, after)
236+
expectInvalidatesExactly(`test:${lane}`, changed)
237+
}
126238
})
127239
} finally {
128240
process.off("SIGINT", onSigint)

0 commit comments

Comments
 (0)