Skip to content

Commit 01f7b1c

Browse files
committed
fix(persistence): recover interrupted replacements during reads and startup
1 parent f6b1ee1 commit 01f7b1c

19 files changed

Lines changed: 836 additions & 186 deletions

‎apps/vscode-e2e/src/suite/restart-persistence.test.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,13 @@ async function runCreate(api: RooCodeAPI): Promise<void> {
5454
await waitUntilCompleted({ api, taskId })
5555
assert.strictEqual(sawMarker, true, `Completion should include ${MARKER}`)
5656

57-
// TaskCompleted fires before the final saveClineMessages() write is
58-
// necessarily durable on disk. Wait until the completion marker is
59-
// persisted in ui_messages.json before quitting, so the verify phase
60-
// never reads a task directory whose final write has not landed yet.
57+
// TaskCompleted and the UI marker do not acknowledge all three files.
58+
// Check both persisted histories and the store's published history item.
59+
// TaskHistoryStore publishes newly created cache entries only after writing them.
6160
const createdTaskId: string = taskId
6261
await waitFor(() => api.hasTaskUiMessageText(createdTaskId, MARKER))
62+
await waitForMarkedCompletion(api, createdTaskId)
63+
await waitFor(async () => (await api.getTaskHistoryItem(createdTaskId)) !== undefined)
6364

6465
const result: PhaseResult = {
6566
version: PHASE_RESULT_VERSION,
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# File replacement and restart model
2+
3+
`scripts/check-file-replacement-restart.ts` is an independent bounded submodel
4+
under `pnpm lifecycle:model-check`. It preserves the completion-readiness model's
5+
separate scope. It checks three files, one initial publication per file, one later
6+
replacement of any one file, and one process crash at every publication or
7+
replacement boundary. Exploration runs to a fixed point with a 10,000-state cap.
8+
9+
## Contract and production mapping
10+
11+
| Model action | Production boundary |
12+
| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
13+
| `publish-ui`, `publish-api`, `publish-history` | Successful writes of `ui_messages.json`, `api_conversation_history.json`, and `history_item.json` |
14+
| `acknowledge-all-three-files` | Explicit readiness conjunction in the restart E2E: `hasTaskUiMessageText`, `hasTaskApiConversationHistorySequence`, and `getTaskHistoryItem`. For a newly created task, `TaskHistoryStore.upsertCore` publishes its cache entry after `writeTaskFile` succeeds. This is a test readiness boundary, not a new public runtime event. |
15+
| `stage-replacement-*` | `safeWriteJson` writes and flushes a `.new_*.tmp` file under the advisory lock |
16+
| `atomic-commit` | Single temp-to-canonical rename |
17+
| `eperm-backup-rename`, `fallback-commit` | Windows-compatible EPERM fallback: canonical-to-backup, then temp-to-canonical |
18+
| `process-crash` | Process exits without rollback, cleanup, or lock release |
19+
| `lock-becomes-stale` | `proper-lockfile` staleness check using the shared 31-second window |
20+
| `fresh-host-startup`, `locked-reader-recovery` | `TaskHistoryStore.reconcile` recovers a missing history record before stat/read; `readFileWithMissingRetry` invokes the same recovery after a second ENOENT for UI/API reads |
21+
| `recovery-io-failure` | Recovery scan, access, stat, or restore fails; rejection preserves committed backups for a later retry |
22+
| `explicit-delete-under-lock` | `deleteJsonFile` removes matching replacements/backups before the canonical file, serialized with writers and recovery; store deletion remains best effort |
23+
24+
The UI marker can become visible while history publication is still pending.
25+
`Task.saveClineMessages` persists UI messages before updating metadata, and logs
26+
metadata failures independently. Neither `TaskCompleted` nor UI-marker visibility
27+
promises that all three files exist. The model explicitly reaches that partial
28+
publication state and permits interruption there. It requires restart discovery
29+
only after the readiness conjunction, not after an unfinished first write.
30+
31+
After acknowledgment, the latest committed version of each required file must
32+
remain in either the canonical file or its backup until explicit deletion.
33+
Completed fresh reads must return those committed versions and find the history
34+
record without a new write. A failed recovery must preserve every file version.
35+
Successful explicit deletion must leave no recoverable version to resurrect.
36+
The model requires reachable atomic and fallback successes, restart recovery,
37+
recovery after failure, partial publication, and reads after deletion.
38+
39+
The checker first runs the old two-rename protocol without reader recovery and
40+
requires a #2130 counterexample. It then exhaustively checks the fixed protocol.
41+
This is an abstract protocol check, not execution of the production writer.
42+
43+
## Refinement evidence
44+
45+
`src/utils/__tests__/safeWriteJson.interruption.test.ts` runs the real bundled
46+
writer in a child process, terminates it with SIGKILL, and exercises both commit
47+
paths before and after the replacement rename for all three filenames. Initial
48+
publication uses the production writer. UI/API tests call production readers;
49+
history-record cases initialize a fresh `TaskHistoryStore` before any recovery
50+
write. One case leaves the crash lock's timestamp unchanged, proving the fresh
51+
store can wait for staleness instead of requiring the test to age the lock.
52+
53+
`safeWriteJson.test.ts` injects directory scan, access, backup stat, and restore
54+
failures, checking backup preservation and retry. The store deletion tests cover
55+
single and batch deletion, leftover cleanup, unrelated-file preservation, and
56+
cleanup failure retaining the canonical copy. The existing two-extension-host
57+
restart E2E remains the production activation/rehydration boundary check.
58+
59+
## Assumptions and limits
60+
61+
This is process-crash safety on filesystems with atomic same-directory rename,
62+
not power-loss safety. Parent directories are not fsynced. Locks are advisory;
63+
all writers, recoverers, and history-file deleters must use the shared protocol.
64+
Live holders refresh their locks and must not be mistaken for crashed holders;
65+
filesystem clock behavior and lock compromise are outside the model. Recovery
66+
waits up to 35 lock retries at one-second intervals, then reports a lock failure.
67+
An indefinitely active writer or persistent filesystem fault can still prevent
68+
recovery; no liveness guarantee is claimed under such faults.
69+
70+
Recovery is abstracted as a locked restore and cleanup; repeated crashes during
71+
recovery, multiple orphaned versions, external file removal, arbitrary overlapping
72+
writers, and deletion interrupted between individual unlinks are not exhaustively
73+
modeled. Focused tests cover newest-backup selection and filesystem failures.
74+
Deletion does not prohibit an explicit later creation of a task at the same path
75+
or a stale writer that deliberately publishes new data after deletion. The
76+
no-resurrection property concerns recovery from leftovers after successful
77+
deletion. Lifecycle status/delegation reducers and completion-event semantics
78+
are unchanged.

‎docs/architecture/task-lifecycle-model.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,14 @@ CI runs `pnpm lifecycle:model-check` in `.github/workflows/code-qa.yml` after li
206206

207207
## Gap audit
208208

209+
The independent [file replacement/restart model](./file-replacement-restart-model.md)
210+
checks the process-crash replacement protocol separately from completion readiness.
211+
It models three task files, initial publication, one replacement, one crash,
212+
locked recovery on fresh reads/startup, and successful explicit deletion. Its
213+
contract, production mapping, old-protocol counterexample, refinement tests, and
214+
filesystem/lock assumptions are documented there. A passing completion-readiness
215+
model alone does not establish this file-level contract.
216+
209217
This section is a summary tracker. Issue links are historical provenance rather than specifications. Coverage status uses these evidence classes:
210218

211219
The [Task lifecycle verification GAP report](./task-lifecycle-gap-report.md) is the authoritative register. It holds the exhaustive repository inventory, the ranked stable burn-down register, the source-based methodology, and the follow-up checklist. This page remains the executable model-suite specification.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
"test:agent-evals:unit": "node --import tsx --test scripts/agent-evals/*.test.ts",
2121
"agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json",
2222
"test:mutation-ci": "node --test scripts/stryker-diff.test.mjs",
23-
"lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check",
23+
"lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-file-replacement-restart.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check",
2424
"webview-recovery:model-check": "node --import tsx scripts/check-webview-recovery.ts",
2525
"webview-recovery:check-types": "tsc -p scripts/tsconfig.webview-recovery.json",
2626
"fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts",
Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
/** Independent bounded abstraction; see docs/architecture/file-replacement-restart-model.md. */
2+
import assert from "node:assert/strict"
3+
4+
type Version = 0 | 1 | 2
5+
type Phase = "publishing" | "idle" | "temp" | "backup" | "committed" | "crashed" | "reading" | "read" | "deleted"
6+
interface FileState {
7+
canonical: Version
8+
backup: Version
9+
temp: Version
10+
committed: Version
11+
}
12+
interface State {
13+
files: FileState[]
14+
phase: Phase
15+
target: number
16+
acknowledged: boolean
17+
lock: "none" | "live" | "fresh-crashed" | "stale"
18+
readIndex: number
19+
recoveryFailed: boolean
20+
deleted: boolean
21+
}
22+
interface Edge {
23+
action: string
24+
state: State
25+
}
26+
const FILES = ["ui", "api", "history"] as const
27+
const MAX_STATES = 10_000
28+
const landmarks = new Set<string>()
29+
30+
function edges(state: State, fixed: boolean): Edge[] {
31+
const result: Edge[] = []
32+
function add(action: string, change: (next: State) => void) {
33+
const next = structuredClone(state)
34+
change(next)
35+
result.push({ action, state: next })
36+
}
37+
if (state.phase === "publishing") {
38+
// Separate file publications: UI visibility does not acknowledge history publication.
39+
for (const [index, file] of state.files.entries()) {
40+
if (!file.canonical)
41+
add(`publish-${FILES[index]}`, (next) => {
42+
next.files[index] = { canonical: 1, backup: 0, temp: 0, committed: 1 }
43+
if (next.files.every((entry) => entry.canonical)) next.phase = "idle"
44+
})
45+
}
46+
}
47+
if (state.phase === "idle" && !state.acknowledged) {
48+
add("acknowledge-all-three-files", (next) => {
49+
next.acknowledged = true
50+
})
51+
}
52+
if (state.phase === "idle" && state.acknowledged) {
53+
for (const index of [0, 1, 2])
54+
add(`stage-replacement-${FILES[index]}`, (next) => {
55+
next.target = index
56+
next.files[index].temp = 2
57+
next.phase = "temp"
58+
next.lock = "live"
59+
})
60+
}
61+
if (state.phase === "temp") {
62+
if (fixed)
63+
add("atomic-commit", (next) => {
64+
next.files[next.target].canonical = 2
65+
next.files[next.target].committed = 2
66+
next.files[next.target].temp = 0
67+
next.phase = "committed"
68+
})
69+
add(fixed ? "eperm-backup-rename" : "old-backup-rename", (next) => {
70+
const file = next.files[next.target]
71+
file.backup = file.canonical
72+
file.canonical = 0
73+
next.phase = "backup"
74+
})
75+
}
76+
if (state.phase === "backup")
77+
add("fallback-commit", (next) => {
78+
const file = next.files[next.target]
79+
file.canonical = file.temp
80+
file.committed = file.temp
81+
file.temp = 0
82+
next.phase = "committed"
83+
})
84+
// One process crash, at every publication/replacement boundary, including post-commit cleanup.
85+
if (["publishing", "idle", "temp", "backup", "committed"].includes(state.phase)) {
86+
add("process-crash", (next) => {
87+
next.phase = "crashed"
88+
next.lock = state.lock === "live" ? "fresh-crashed" : "none"
89+
})
90+
}
91+
if (state.lock === "fresh-crashed")
92+
add("lock-becomes-stale", (next) => {
93+
next.lock = "stale"
94+
})
95+
if (state.phase === "crashed")
96+
add("fresh-host-startup", (next) => {
97+
next.phase = "reading"
98+
})
99+
if (state.phase === "reading") {
100+
const file = state.files[state.readIndex]
101+
if (file.canonical || !fixed || (!file.backup && !file.temp)) {
102+
add(`read-${FILES[state.readIndex]}`, (next) => {
103+
next.readIndex++
104+
if (next.readIndex === FILES.length) next.phase = "read"
105+
})
106+
} else if (state.lock !== "fresh-crashed") {
107+
// Failed scan/access/stat/rename preserves every committed byte and permits retry.
108+
if (!state.recoveryFailed)
109+
add("recovery-io-failure", (next) => {
110+
next.recoveryFailed = true
111+
})
112+
add("locked-reader-recovery", (next) => {
113+
const recovering = next.files[next.readIndex]
114+
recovering.canonical = recovering.backup
115+
recovering.backup = 0
116+
recovering.temp = 0
117+
next.lock = "none"
118+
})
119+
}
120+
}
121+
if (["crashed", "read"].includes(state.phase) && state.lock !== "fresh-crashed") {
122+
add("explicit-delete-under-lock", (next) => {
123+
next.files = next.files.map(() => ({ canonical: 0, backup: 0, temp: 0, committed: 0 }))
124+
next.phase = "deleted"
125+
next.deleted = true
126+
next.lock = "none"
127+
})
128+
}
129+
if (state.phase === "deleted")
130+
add("read-after-delete", (next) => {
131+
next.phase = "read"
132+
})
133+
return result
134+
}
135+
136+
function violation(state: State): string | undefined {
137+
if (state.deleted && state.files.some((file) => file.canonical || file.backup)) return "deleted history resurrected"
138+
if (state.acknowledged && !state.deleted) {
139+
if (state.files.some((file) => file.canonical !== file.committed && file.backup !== file.committed)) {
140+
return "last committed content lost"
141+
}
142+
if (state.phase === "read" && state.files.some((file) => file.canonical !== file.committed)) {
143+
return "acknowledged task not discoverable/recoverable on fresh startup (#2130)"
144+
}
145+
}
146+
return undefined
147+
}
148+
149+
function explore(fixed: boolean): number {
150+
const initial: State = {
151+
files: FILES.map(() => ({ canonical: 0, backup: 0, temp: 0, committed: 0 })),
152+
phase: "publishing",
153+
target: 0,
154+
acknowledged: false,
155+
lock: "none",
156+
readIndex: 0,
157+
recoveryFailed: false,
158+
deleted: false,
159+
}
160+
const queue = [{ state: initial, trace: [] as string[] }]
161+
const seen = new Set([JSON.stringify(initial)])
162+
for (let index = 0; index < queue.length; index++) {
163+
const { state, trace } = queue[index]
164+
const error = violation(state)
165+
if (error) {
166+
if (!fixed && error.includes("#2130")) {
167+
console.log(`Old protocol #2130 counterexample: ${trace.join(" -> ")}`)
168+
return seen.size
169+
}
170+
throw new Error(`${error}: ${trace.join(" -> ")}\n${JSON.stringify(state)}`)
171+
}
172+
if (fixed) {
173+
if (state.phase === "crashed" && state.files[0].canonical && !state.files[2].canonical)
174+
landmarks.add("ui-visible-history-pending")
175+
if (state.phase === "read" && state.acknowledged && !state.deleted) landmarks.add("acknowledged-restart")
176+
if (state.phase === "read" && state.recoveryFailed && !state.deleted)
177+
landmarks.add("retry-after-recovery-failure")
178+
if (state.phase === "read" && state.deleted) landmarks.add("no-resurrection")
179+
if (state.phase === "committed" && !state.files[state.target].backup) landmarks.add("atomic-success")
180+
if (state.phase === "committed" && state.files[state.target].backup) landmarks.add("fallback-success")
181+
}
182+
for (const edge of edges(state, fixed)) {
183+
if (fixed && edge.action === "recovery-io-failure") assert.deepEqual(edge.state.files, state.files)
184+
const key = JSON.stringify(edge.state)
185+
if (seen.has(key)) continue
186+
seen.add(key)
187+
assert.ok(seen.size <= MAX_STATES, "Replacement/restart state budget exceeded")
188+
queue.push({ state: edge.state, trace: [...trace, edge.action] })
189+
}
190+
}
191+
assert.ok(fixed, "Old protocol must reproduce #2130")
192+
return seen.size
193+
}
194+
195+
explore(false)
196+
const checked = explore(true)
197+
assert.equal(landmarks.size, 6, `Unreachable replacement/restart landmarks: ${[...landmarks]}`)
198+
console.log(
199+
`File replacement/restart model passed: ${checked} states, 3 files, initial publication + 1 replacement, 1 crash, 6/6 landmarks`,
200+
)

0 commit comments

Comments
 (0)