Skip to content

Commit 8bc3d21

Browse files
committed
fix(antigravity): quarantine corrupted session checkpoints and auto-recover
1 parent a8ebd72 commit 8bc3d21

2 files changed

Lines changed: 94 additions & 1 deletion

File tree

‎apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,4 +192,56 @@ describe("watchAntigravityPromptOutcome", () => {
192192
});
193193
}).pipe(Effect.provide(NodeServices.layer), Effect.scoped),
194194
);
195+
196+
it.effect("detects corrupted checkpoint errors and quarantines the database", () =>
197+
Effect.gen(function* () {
198+
const fs = yield* FileSystem.FileSystem;
199+
const path = yield* Path.Path;
200+
const profileDirectory = yield* fs.makeTempDirectoryScoped();
201+
const sessionId = "a1b2c3d4-e5f6-4a1b-8c2d-3e4f5a6b7c8d";
202+
const conversations = path.join(profileDirectory, "antigravity-acp", "conversations");
203+
yield* fs.makeDirectory(conversations, { recursive: true });
204+
const dbPath = path.join(conversations, `${sessionId}.db`);
205+
206+
const database = new NodeSqlite.DatabaseSync(dbPath);
207+
try {
208+
database.exec(
209+
"CREATE TABLE IF NOT EXISTS steps (idx INTEGER PRIMARY KEY, step_type INTEGER, step_payload BLOB)",
210+
);
211+
} finally {
212+
database.close();
213+
}
214+
215+
const check = yield* watchAntigravityPromptOutcome(profileDirectory, sessionId);
216+
217+
const checkpointCorruptPayload = errorPayload(
218+
"agent executor error: could not find doneCh for checkpoint",
219+
);
220+
const corruptDb = new NodeSqlite.DatabaseSync(dbPath);
221+
try {
222+
corruptDb
223+
.prepare("INSERT INTO steps (idx, step_type, step_payload) VALUES (?, ?, ?)")
224+
.run(0, ERROR_MESSAGE, checkpointCorruptPayload);
225+
} finally {
226+
corruptDb.close();
227+
}
228+
229+
const failure = yield* check({ stopReason: "end_turn" });
230+
assert.deepEqual(failure, {
231+
class: "provider_error",
232+
message:
233+
"Antigravity session checkpoint was corrupted and has been quarantined. The session will restart.",
234+
code: "SESSION_CORRUPTED",
235+
retryable: true,
236+
});
237+
238+
// The original .db file should be quarantined (moved)
239+
const originalExists = yield* fs.exists(dbPath);
240+
assert.isFalse(originalExists);
241+
242+
const files = yield* fs.readDirectory(conversations);
243+
const quarantined = files.find((f) => f.startsWith(`${sessionId}.db.corrupt-`));
244+
assert.isDefined(quarantined);
245+
}).pipe(Effect.provide(NodeServices.layer), Effect.scoped),
246+
);
195247
});

‎apps/server/src/provider/acp/AntigravityTurnOutcome.ts‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,12 @@ const MAX_ERROR_PAYLOAD_BYTES = 16_384;
1919
// The Go harness's rendering of a Google API error, as the agent's api_errors.py reads it.
2020
const API_ERROR = /Error (\d{3}), Message: (.+?), Status: ([A-Z_]+)/s;
2121
const QUOTA_EXHAUSTED = /\breason:QUOTA_EXHAUSTED\b/;
22+
const CHECKPOINT_CORRUPTED = /could not find doneCh for checkpoint/i;
23+
24+
function isAntigravityCheckpointCorrupted(text: string): boolean {
25+
return CHECKPOINT_CORRUPTED.test(text);
26+
}
27+
2228
const RESET_TIMESTAMP = /\bquotaResetTimeStamp:([^\s\]]+)/;
2329
const RESET_DELAY = /\bquotaResetDelay:((?:\d+(?:\.\d+)?(?:ns|us|µs|μs|ms|h|m|s))+)(?=[\s\]])/;
2430
const GO_DURATION_PART = /(\d+(?:\.\d+)?)(ns|us|µs|μs|ms|h|m|s)/g;
@@ -107,6 +113,15 @@ function turnFailure(
107113
resetAt: resetAtFrom(text, now) ?? null,
108114
});
109115
}
116+
if (isAntigravityCheckpointCorrupted(text)) {
117+
return makeProviderFailure({
118+
class: "provider_error",
119+
message:
120+
"Antigravity session checkpoint was corrupted and has been quarantined. The session will restart.",
121+
code: "SESSION_CORRUPTED",
122+
retryable: true,
123+
});
124+
}
110125
if (apiError === null) {
111126
return makeProviderFailure({
112127
class: "provider_error",
@@ -123,6 +138,17 @@ function turnFailure(
123138
});
124139
}
125140

141+
async function quarantineAntigravitySessionDatabase(
142+
databasePath: string,
143+
timestamp: number,
144+
): Promise<string> {
145+
const quarantinePath = `${databasePath}.corrupt-${timestamp}`;
146+
await NodeFSP.rename(databasePath, quarantinePath).catch(() => {});
147+
await NodeFSP.rename(`${databasePath}-wal`, `${quarantinePath}-wal`).catch(() => {});
148+
await NodeFSP.rename(`${databasePath}-shm`, `${quarantinePath}-shm`).catch(() => {});
149+
return quarantinePath;
150+
}
151+
126152
const conversationSegments = ["antigravity-acp", "conversations"];
127153

128154
/**
@@ -189,6 +215,21 @@ export const watchAntigravityPromptOutcome = Effect.fn("AntigravityTurnOutcome.w
189215
Effect.orElseSucceed(() => undefined),
190216
)
191217
: undefined;
192-
return turnFailure(result.stopReason, step, yield* DateTime.now);
218+
const now = yield* DateTime.now;
219+
const failure = turnFailure(result.stopReason, step, now);
220+
if (failure?.code === "SESSION_CORRUPTED" && after !== undefined) {
221+
const timestamp = DateTime.toEpochMillis(now);
222+
yield* Effect.tryPromise(() =>
223+
quarantineAntigravitySessionDatabase(after.path, timestamp),
224+
).pipe(
225+
Effect.tap(() =>
226+
Effect.logWarning("Quarantined corrupted Antigravity session database", {
227+
path: after.path,
228+
}),
229+
),
230+
Effect.orElseSucceed(() => undefined),
231+
);
232+
}
233+
return failure;
193234
});
194235
});

0 commit comments

Comments
 (0)