Skip to content

Commit 6c1ddb3

Browse files
committed
fix(snapshot): heal silent empty-tree snapshots and widen index corruption recovery
1 parent d27f83c commit 6c1ddb3

3 files changed

Lines changed: 127 additions & 51 deletions

File tree

‎packages/opencode/src/snapshot/index.ts‎

Lines changed: 90 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -108,13 +108,20 @@ const layer: Layer.Layer<Service, never, FSUtil.Service | AppProcess.Service | C
108108
)
109109
const snapshotGitEnv = (env?: Record<string, string>) => ({ ...env, LC_ALL: "C", LANG: "C" })
110110

111+
// A corrupt index can silently produce the git empty tree for a non-empty worktree.
112+
const isSnapshotEmptyTree = (hash: string) =>
113+
hash === "4b825dc642cb6eb9a060e54bf8d69288fbee4904" ||
114+
hash === "6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321"
115+
111116
const git = Effect.fnUntraced(function* (
112117
cmd: string[],
113118
opts?: { cwd?: string; env?: Record<string, string>; stdin?: string },
114119
) {
115-
const result = Effect.suspend(() =>
120+
const writesIndex = ["add", "rm", "read-tree"].some((subcommand) => cmd.includes(subcommand))
121+
// Deleting a corrupt index is only safe ahead of commands that rebuild state from the worktree.
122+
const healsIndex = writesIndex || ["diff-files", "ls-files"].some((subcommand) => cmd.includes(subcommand))
123+
const execute = Effect.suspend(() =>
116124
Effect.gen(function* () {
117-
const writesIndex = ["add", "rm", "read-tree"].some((subcommand) => cmd.includes(subcommand))
118125
if (writesIndex && (yield* exists(indexLock))) {
119126
return {
120127
code: ChildProcessSpawner.ExitCode(128),
@@ -149,41 +156,52 @@ const layer: Layer.Layer<Service, never, FSUtil.Service | AppProcess.Service | C
149156
}),
150157
)
151158

152-
return yield* result.pipe(
153-
Effect.flatMap((result) =>
154-
isIndexLockContention(result) ? Effect.fail(result) : Effect.succeed(result),
155-
),
156-
Effect.retry({ schedule: indexLockRetry, while: isIndexLockContention }),
157-
Effect.catch((result) =>
158-
Effect.gen(function* () {
159-
if (isIndexLockContention(result)) {
160-
const ageMs = yield* indexLockAge()
161-
yield* Effect.logError("snapshot_index_lock_stuck", {
162-
indexLock,
163-
ageMs,
164-
remedy: "remove the lock manually only after confirming no git process owns it",
165-
})
166-
yield* Effect.sync(() =>
167-
process.stderr.write(
168-
`snapshot_index_lock_stuck index_lock=${indexLock} age_ms=${ageMs ?? "unknown"} remedy=manual-confirm-and-remove\n`,
169-
),
170-
)
171-
}
172-
return result
173-
}),
174-
),
159+
const settle = (effect: typeof execute) =>
160+
effect.pipe(
161+
Effect.flatMap((result) =>
162+
isIndexLockContention(result) ? Effect.fail(result) : Effect.succeed(result),
163+
),
164+
Effect.retry({ schedule: indexLockRetry, while: isIndexLockContention }),
165+
Effect.catch((result) =>
166+
Effect.gen(function* () {
167+
if (isIndexLockContention(result)) {
168+
const ageMs = yield* indexLockAge()
169+
yield* Effect.logError("snapshot_index_lock_stuck", {
170+
indexLock,
171+
ageMs,
172+
remedy: "remove the lock manually only after confirming no git process owns it",
173+
})
174+
yield* Effect.sync(() =>
175+
process.stderr.write(
176+
`snapshot_index_lock_stuck index_lock=${indexLock} age_ms=${ageMs ?? "unknown"} remedy=manual-confirm-and-remove\n`,
177+
),
178+
)
179+
}
180+
return result
181+
}),
182+
),
183+
)
184+
185+
return yield* settle(execute).pipe(
175186
Effect.flatMap((outcome) => {
176187
const corrupted =
177188
outcome.code !== 0 &&
178-
/index file corrupt|index file smaller than expected|bad index file sha1 signature/.test(
189+
healsIndex &&
190+
/index file corrupt|index file smaller than expected|bad index file sha1 signature|unexpected diff status|unknown file mode for .* in index/.test(
179191
outcome.stderr,
180192
)
181193
if (!corrupted) return Effect.succeed(outcome)
182194
return Effect.gen(function* () {
183-
// The snapshot gitdir is derived state; deleting a corrupt index only drops staged paths, which the retry restages.
195+
const removed = yield* fs.remove(path.join(state.gitdir, "index")).pipe(
196+
Effect.as(true),
197+
Effect.catch(() => Effect.succeed(false)),
198+
)
199+
if (!removed) {
200+
yield* Effect.logWarning("snapshot_index_corruption_detected", { gitdir: state.gitdir })
201+
return yield* settle(execute)
202+
}
184203
yield* Effect.logWarning("snapshot_index_corruption_recovered", { gitdir: state.gitdir })
185-
yield* fs.remove(path.join(state.gitdir, "index")).pipe(Effect.ignore)
186-
return yield* result
204+
return yield* settle(execute)
187205
})
188206
}),
189207
)
@@ -569,23 +587,49 @@ const layer: Layer.Layer<Service, never, FSUtil.Service | AppProcess.Service | C
569587
if (!(yield* setup(existed))) return
570588
return yield* withTransactionToken(
571589
Effect.gen(function* () {
572-
if (!(yield* add())) return
573-
const result = yield* git(args(["write-tree"]), { cwd: state.directory })
574-
if (result.code !== 0) {
575-
yield* Effect.logError("failed to write snapshot tree", {
576-
exitCode: result.code,
577-
stderr: result.stderr,
578-
})
579-
return
580-
}
581-
const hash = result.text.trim()
582-
if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(hash)) {
583-
yield* Effect.logError("snapshot git returned an invalid tree hash", {
584-
hash,
585-
stderr: result.stderr,
586-
})
587-
return
588-
}
590+
const capture = Effect.gen(function* () {
591+
if (!(yield* add())) return
592+
const result = yield* git(args(["write-tree"]), { cwd: state.directory })
593+
if (result.code !== 0) {
594+
yield* Effect.logError("failed to write snapshot tree", {
595+
exitCode: result.code,
596+
stderr: result.stderr,
597+
})
598+
return
599+
}
600+
const hash = result.text.trim()
601+
if (!/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(hash)) {
602+
yield* Effect.logError("snapshot git returned an invalid tree hash", {
603+
hash,
604+
stderr: result.stderr,
605+
})
606+
return
607+
}
608+
return hash
609+
})
610+
611+
// A corrupt index can silently produce the empty tree; rebuild it and recapture once before accepting the result.
612+
const hash = yield* capture.pipe(
613+
Effect.flatMap((captured) => {
614+
if (captured === undefined || !isSnapshotEmptyTree(captured)) return Effect.succeed(captured)
615+
return Effect.gen(function* () {
616+
yield* Effect.logWarning("snapshot_empty_tree_detected", {
617+
hash: captured,
618+
gitdir: state.gitdir,
619+
})
620+
yield* fs.remove(path.join(state.gitdir, "index")).pipe(Effect.ignore)
621+
const retry = yield* capture
622+
if (retry !== undefined && !isSnapshotEmptyTree(retry)) {
623+
yield* Effect.logWarning("snapshot_empty_tree_recovered", {
624+
hash: retry,
625+
gitdir: state.gitdir,
626+
})
627+
}
628+
return retry
629+
})
630+
}),
631+
)
632+
if (hash === undefined) return
589633
yield* Effect.logInfo("tracking", { hash, cwd: state.directory, git: state.gitdir })
590634
return hash
591635
}),

‎packages/opencode/test/fixture/snapshot-lock-worker.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type Message = {
2525
cleanup?: boolean
2626
patchAfterIgnore?: boolean
2727
corruptIndexOnce?: boolean
28+
truncateIndexOnce?: boolean
2829
}
2930

3031
const message: Message = JSON.parse(process.argv[2] ?? "{}")
@@ -99,7 +100,8 @@ const result = await Effect.runPromise(
99100
!message.deadToken &&
100101
!message.liveToken &&
101102
!message.foreignHostToken &&
102-
!message.corruptIndexOnce
103+
!message.corruptIndexOnce &&
104+
!message.truncateIndexOnce
103105
) {
104106
if (!message.reportToken) return { first }
105107
const gitdir = yield* Effect.promise(snapshotGitdir)
@@ -143,6 +145,17 @@ const result = await Effect.runPromise(
143145
}
144146
if (message.corruptIndexOnce) {
145147
yield* Effect.promise(() => fs.writeFile(path.join(gitdir, "index"), "corrupt"))
148+
const second = yield* snapshot.track()
149+
return { first, second, gitdir }
150+
}
151+
if (message.truncateIndexOnce) {
152+
yield* Effect.promise(async () => {
153+
const target = path.join(gitdir, "index")
154+
const bytes = await Bun.file(target).arrayBuffer()
155+
await Bun.write(target, bytes.slice(0, Math.max(12, Math.floor(bytes.byteLength / 2))))
156+
})
157+
const second = yield* snapshot.track()
158+
return { first, second, gitdir }
146159
}
147160
if (message.transientIndexLockMs) {
148161
setTimeout(() => void fs.rm(path.join(gitdir, "index.lock"), { force: true }), message.transientIndexLockMs)

‎packages/opencode/test/snapshot/snapshot-lock.test.ts‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ type WorkerMessage = {
2424
cleanup?: boolean
2525
patchAfterIgnore?: boolean
2626
corruptIndexOnce?: boolean
27+
truncateIndexOnce?: boolean
2728
}
2829

2930
type WorkerResult = {
@@ -428,9 +429,27 @@ describe("snapshot cross-process git lock", () => {
428429
const output = path.join(environment.data, "result.json")
429430
const result = await runWorker({ directory: repo.path, output, corruptIndexOnce: true }, environment.env)
430431
expect(result.code, result.stderr).toBe(0)
431-
expect(await Bun.file(output).json()).toMatchObject({
432-
first: expect.stringMatching(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/),
433-
second: expect.stringMatching(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/),
434-
})
432+
const json = await Bun.file(output).json()
433+
expect(json.first).toMatch(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/)
434+
expect(json.second).toMatch(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/)
435+
expect(json.second).not.toBe(json.first)
436+
expect(json.second).not.toBe("4b825dc642cb6eb9a060e54bf8d69288fbee4904")
437+
const names = await $`git --git-dir=${json.gitdir} ls-tree -r --name-only ${json.second}`.quiet().text()
438+
expect(names).toContain("changed.txt")
439+
}, 30_000)
440+
441+
test("recovers from a truncated snapshot index instead of publishing an empty tree", async () => {
442+
const repo = await tmpdir({ git: true })
443+
await using _repo = repo
444+
const environment = await testEnvironment()
445+
const output = path.join(environment.data, "result.json")
446+
const result = await runWorker({ directory: repo.path, output, truncateIndexOnce: true }, environment.env)
447+
expect(result.code, result.stderr).toBe(0)
448+
const json = await Bun.file(output).json()
449+
expect(json.second).toMatch(/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/)
450+
expect(json.second).not.toBe("4b825dc642cb6eb9a060e54bf8d69288fbee4904")
451+
expect(json.second).not.toBe("6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321")
452+
const names = await $`git --git-dir=${json.gitdir} ls-tree -r --name-only ${json.second}`.quiet().text()
453+
expect(names).toContain("changed.txt")
435454
}, 30_000)
436455
})

0 commit comments

Comments
 (0)