Skip to content

Commit 5366b6d

Browse files
authored
refactor: store gateway restart recovery in SQLite (#110014)
* refactor: move restart sentinel state to sqlite * fix: satisfy restart sentinel architecture gates * test: avoid cold startup migration timeout * fix: bound durable restart notice retries * test: type restart notice contention mock * fix: detect legacy restart sentinel before reads * fix: keep delivery attempt result internal * fix: narrow restart sentinel preflight guard
1 parent b75b62f commit 5366b6d

30 files changed

Lines changed: 3713 additions & 629 deletions

docs/gateway/restart-recovery.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,14 @@ SQLite before the process exits. After boot the gateway posts the outcome back
159159
to the originating chat and dispatches a one-shot continuation turn so the
160160
agent picks up exactly where it left off, on the same channel and thread.
161161

162+
The sentinel's typed SQLite columns are authoritative for restart handling;
163+
its `payload_json` value is a replay/debug shadow only. Runtime reads, writes,
164+
and clears SQLite state without a file fallback. During the storage cutover, a
165+
bounded state migration runs at startup and through Doctor to preserve a
166+
validated `restart-sentinel.json` left by the older process after an update.
167+
The migration verifies the typed row and removes the source file before normal
168+
restart handling continues.
169+
162170
## Safety valves and observability
163171

164172
- **Crash-loop breaker:** 3 unclean boots within 5 minutes trip a breaker that

docs/refactor/database-first.md

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1256,8 +1256,12 @@ sessionId})`; create, branch, continue, list, and fork flows live in their
12561256
- Gateway restart sentinel state now uses typed shared SQLite
12571257
`gateway_restart_sentinel` rows instead of `restart-sentinel.json`; runtime
12581258
reads sentinel kind, status, routing, message, continuation, and stats from
1259-
typed columns. `payload_json` is only a replay/debug copy. Runtime code clears
1260-
the SQLite row directly and no longer carries file cleanup plumbing.
1259+
typed columns. Those columns are authoritative; `payload_json` is only a
1260+
replay/debug shadow. Runtime read, write, and clear paths are SQLite-only.
1261+
One bounded state-migration module runs during startup and Doctor to import a
1262+
validated older post-update sentinel before normal restart recovery, verify
1263+
the typed row, and remove the source file. No steady-state runtime module
1264+
reads, writes, or cleans up the legacy file.
12611265
- Gateway restart intent and supervisor handoff state now use typed shared
12621266
SQLite `gateway_restart_intent` and `gateway_restart_handoff` rows instead of
12631267
`gateway-restart-intent.json` and

scripts/check-database-first-legacy-stores.mjs

Lines changed: 113 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,26 @@ const fsSafePackageModulePattern = /^@openclaw\/fs-safe(?:\/(?:root|store))?$/u;
8484

8585
const bridgeMarkerPattern = /\btranscriptLocator\b|sqlite-transcript:\/\//u;
8686

87+
// The restart handoff must survive its one cutover migration without leaving
88+
// filesystem fallback imports in the steady-state runtime owner.
89+
const legacyRestartSentinelMigrationPath = "src/infra/state-migrations.restart-sentinel.ts";
90+
const legacyRestartSentinelPreflightPath = "src/cli/program/config-guard.ts";
91+
const legacyRestartSentinelRuntimePath = "src/infra/restart-sentinel.ts";
92+
const legacyRestartSentinelPreflightFilenames = new Set([
93+
"restart-sentinel.json",
94+
"restart-sentinel.json.doctor-importing",
95+
]);
96+
const legacyRestartSentinelFilenamePattern =
97+
/(?:^|[/\\])restart-sentinel\.json(?:\.doctor-importing)?$/u;
98+
const legacyRestartSentinelRuntimeImportSpecifiers = new Set([
99+
"fs",
100+
"fs/promises",
101+
"node:fs",
102+
"node:fs/promises",
103+
"node:path",
104+
"path",
105+
]);
106+
87107
const legacyStorePatterns = [
88108
/\bsessions\.json\b/u,
89109
/\.trajectory\.jsonl\b/u,
@@ -129,6 +149,7 @@ const allowedRuntimeMigrationPaths = [
129149
"src/infra/state-migrations.managed-outgoing-images.ts",
130150
"src/infra/state-migrations.apns.ts",
131151
"src/infra/state-migrations.mcp-oauth.ts",
152+
legacyRestartSentinelMigrationPath,
132153
"src/infra/state-migrations.workspace-setup.ts",
133154
"src/infra/state-migrations.web-push.ts",
134155
"src/infra/state-migrations.node-host.ts",
@@ -327,6 +348,86 @@ function importSource(node) {
327348
return ts.isStringLiteral(moduleSpecifier) ? moduleSpecifier.text : "";
328349
}
329350

351+
function isLegacyRestartSentinelPreflightDetection(node, relativePath) {
352+
if (
353+
relativePath !== legacyRestartSentinelPreflightPath ||
354+
!legacyRestartSentinelPreflightFilenames.has(node.text)
355+
) {
356+
return false;
357+
}
358+
const joinCall = node.parent;
359+
if (
360+
!ts.isCallExpression(joinCall) ||
361+
joinCall.arguments.length !== 2 ||
362+
joinCall.arguments[1] !== node ||
363+
!ts.isPropertyAccessExpression(joinCall.expression) ||
364+
!ts.isIdentifier(joinCall.expression.expression) ||
365+
joinCall.expression.expression.text !== "path" ||
366+
joinCall.expression.name.text !== "join" ||
367+
!ts.isIdentifier(joinCall.arguments[0]) ||
368+
joinCall.arguments[0].text !== "stateDir"
369+
) {
370+
return false;
371+
}
372+
const paths = joinCall.parent;
373+
if (!ts.isArrayLiteralExpression(paths)) {
374+
return false;
375+
}
376+
const someAccess = paths.parent;
377+
if (
378+
!ts.isPropertyAccessExpression(someAccess) ||
379+
someAccess.expression !== paths ||
380+
someAccess.name.text !== "some"
381+
) {
382+
return false;
383+
}
384+
const someCall = someAccess.parent;
385+
return (
386+
ts.isCallExpression(someCall) &&
387+
someCall.arguments.length === 1 &&
388+
ts.isIdentifier(someCall.arguments[0]) &&
389+
someCall.arguments[0].text === "fileOrDirExists"
390+
);
391+
}
392+
393+
function collectLegacyRestartSentinelBoundaryViolations(sourceFile, relativePath) {
394+
if (relativePath === legacyRestartSentinelMigrationPath) {
395+
return [];
396+
}
397+
398+
const violations = [];
399+
const seen = new Set();
400+
function add(node, kind) {
401+
const line = toLine(sourceFile, node);
402+
const key = `${line}:${kind}`;
403+
if (seen.has(key)) {
404+
return;
405+
}
406+
seen.add(key);
407+
violations.push({ kind, line });
408+
}
409+
410+
function visit(node) {
411+
if (
412+
ts.isStringLiteralLike(node) &&
413+
legacyRestartSentinelFilenamePattern.test(node.text) &&
414+
!isLegacyRestartSentinelPreflightDetection(node, relativePath)
415+
) {
416+
add(node, "legacy restart sentinel reference");
417+
}
418+
if (
419+
relativePath === legacyRestartSentinelRuntimePath &&
420+
ts.isImportDeclaration(node) &&
421+
legacyRestartSentinelRuntimeImportSpecifiers.has(importSource(node))
422+
) {
423+
add(node, "legacy restart sentinel filesystem import");
424+
}
425+
ts.forEachChild(node, visit);
426+
}
427+
visit(sourceFile);
428+
return violations;
429+
}
430+
330431
function isHelperWriteModuleSource(source) {
331432
return (
332433
source === "openclaw/plugin-sdk/file-access-runtime" ||
@@ -526,21 +627,28 @@ function legacyCandidateTexts(sourceFile, node) {
526627
*/
527628
export function collectDatabaseFirstLegacyStoreViolations(
528629
content,
529-
relativePath = "source.ts",
630+
inputRelativePath = "source.ts",
530631
scanOptions = {},
531632
) {
633+
const relativePath = inputRelativePath.replaceAll("\\", "/");
634+
const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true);
635+
const boundaryViolations = collectLegacyRestartSentinelBoundaryViolations(
636+
sourceFile,
637+
relativePath,
638+
);
532639
if (isAllowedLegacyOwnerPath(relativePath)) {
533-
return [];
640+
return boundaryViolations;
534641
}
535642

536-
const sourceFile = ts.createSourceFile(relativePath, content, ts.ScriptTarget.Latest, true);
537643
const currentLegacyWriteAllowances =
538644
scanOptions.currentLegacyWriteAllowances ?? currentLegacyWriteViolationAllowances(relativePath);
539645
const createRequireBindings = collectCreateRequireBindings(sourceFile);
540646
const { fsModuleBindings, fsWriteAliases, fsSafeStoreFactoryAliases } =
541647
collectFsBindings(sourceFile);
542-
const violations = [];
543-
const seenViolations = new Set();
648+
const violations = [...boundaryViolations];
649+
const seenViolations = new Set(
650+
boundaryViolations.map((violation) => `${violation.line}:${violation.kind}`),
651+
);
544652
const fsModuleBindingScopes = [new Map([...fsModuleBindings].map((name) => [name, true]))];
545653
const fsModulePropertyScopes = [new Map()];
546654
const fsWriteAliasScopes = [fsWriteAliases];

src/cli/program/config-guard.test.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,23 @@ describe("ensureConfigReady", () => {
207207
});
208208
});
209209

210+
it.each(["restart-sentinel.json", "restart-sentinel.json.doctor-importing"])(
211+
"runs doctor flow when lightweight startup detection finds %s",
212+
async (relativePath) => {
213+
const root = useTempOpenClawHome();
214+
writeStateMarker(root, relativePath);
215+
216+
await runEnsureConfigReady(["status"]);
217+
218+
expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledWith({
219+
migrateState: true,
220+
migrateLegacyConfig: false,
221+
invalidConfigNote: false,
222+
observe: false,
223+
});
224+
},
225+
);
226+
210227
it("runs doctor flow when lightweight startup detection finds a pending SQLite archive", async () => {
211228
const root = useTempOpenClawHome();
212229
writePendingTaskSidecarArchiveMarker(root);

src/cli/program/config-guard.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,8 @@ function hasLegacyStateMigrationInputs(): boolean {
124124
path.join(stateDir, "agent"),
125125
path.join(stateDir, "agents"),
126126
path.join(stateDir, "plugins", "installs.json"),
127+
path.join(stateDir, "restart-sentinel.json"),
128+
path.join(stateDir, "restart-sentinel.json.doctor-importing"),
127129
path.join(stateDir, "sessions"),
128130
path.join(stateDir, "state", "openclaw.sqlite"),
129131
].some(fileOrDirExists) ||
Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
// Exercises restart-notice retries against the real SQLite outbound queue.
2+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
3+
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
4+
import { getDeliveryQueueEntryStatus } from "../infra/delivery-queue-sqlite.js";
5+
import { PlatformMessageNotDispatchedError } from "../infra/outbound/deliver-types.js";
6+
import { loadPendingDelivery } from "../infra/outbound/delivery-queue-storage.js";
7+
import { markDeliveryPlatformSendAttemptStarted } from "../infra/outbound/delivery-queue.js";
8+
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
9+
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
10+
11+
const mocks = vi.hoisted(() => ({
12+
sendDurableMessageBatch: vi.fn(),
13+
recoveryDeliver: vi.fn(),
14+
resolveOutboundChannelMessageAdapter: vi.fn(() => undefined),
15+
sleep: vi.fn(async () => {}),
16+
}));
17+
18+
vi.mock("../channels/message/runtime.js", () => ({
19+
sendDurableMessageBatch: mocks.sendDurableMessageBatch,
20+
}));
21+
22+
vi.mock("../infra/outbound/deliver.js", () => ({
23+
deliverOutboundPayloadsInternal: mocks.recoveryDeliver,
24+
}));
25+
26+
vi.mock("../infra/outbound/channel-resolution.js", () => ({
27+
resolveOutboundChannelMessageAdapter: mocks.resolveOutboundChannelMessageAdapter,
28+
}));
29+
30+
vi.mock("../utils/sleep.js", () => ({ sleep: mocks.sleep }));
31+
32+
const { deliverRestartSentinelNotice, enqueueRestartSentinelNotice } =
33+
await import("./server-restart-sentinel-notice.js");
34+
35+
type DeliveryRequest = { deliveryQueueId?: string; deliveryQueueStateDir?: string };
36+
37+
describe("restart sentinel notice recovery", () => {
38+
let envSnapshot: ReturnType<typeof captureEnv> | undefined;
39+
let stateDir = "";
40+
const tempDirs = useAutoCleanupTempDirTracker((cleanup) => {
41+
afterEach(() => {
42+
closeOpenClawStateDatabaseForTest();
43+
envSnapshot?.restore();
44+
envSnapshot = undefined;
45+
cleanup();
46+
});
47+
});
48+
49+
beforeEach(() => {
50+
closeOpenClawStateDatabaseForTest();
51+
stateDir = tempDirs.make("openclaw-restart-notice-");
52+
envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
53+
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
54+
mocks.sendDurableMessageBatch.mockReset();
55+
mocks.recoveryDeliver.mockReset();
56+
mocks.resolveOutboundChannelMessageAdapter.mockClear();
57+
mocks.sleep.mockClear();
58+
});
59+
60+
async function enqueueNotice(): Promise<string> {
61+
const queued = await enqueueRestartSentinelNotice({
62+
channel: "whatsapp",
63+
to: "+15550002",
64+
message: "restart complete",
65+
sessionKey: "agent:main:main",
66+
revision: 123,
67+
});
68+
return queued.id;
69+
}
70+
71+
async function deliverNotice(queueId: string): Promise<void> {
72+
await deliverRestartSentinelNotice({
73+
deps: {} as never,
74+
cfg: {},
75+
channel: "whatsapp",
76+
to: "+15550002",
77+
message: "restart complete",
78+
sessionKey: "agent:main:main",
79+
summary: "restart summary",
80+
queueId,
81+
});
82+
}
83+
84+
async function markAttempt(request: unknown): Promise<void> {
85+
const { deliveryQueueId, deliveryQueueStateDir } = request as DeliveryRequest;
86+
if (!deliveryQueueId) {
87+
throw new Error("expected durable delivery queue id");
88+
}
89+
await markDeliveryPlatformSendAttemptStarted(deliveryQueueId, deliveryQueueStateDir);
90+
}
91+
92+
function queueStatus(queueId: string): string | undefined {
93+
return getDeliveryQueueEntryStatus("outbound", queueId, stateDir);
94+
}
95+
96+
it("replays a retryable provider-not-dispatched failure after the startup scan", async () => {
97+
const queueId = await enqueueNotice();
98+
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
99+
await markAttempt(request);
100+
return {
101+
status: "failed",
102+
error: new PlatformMessageNotDispatchedError("connect failed before dispatch", {
103+
cause: new Error("connect failed"),
104+
}),
105+
};
106+
});
107+
mocks.recoveryDeliver.mockResolvedValueOnce([
108+
{ channel: "whatsapp", messageId: "recovered-1" },
109+
]);
110+
111+
await deliverNotice(queueId);
112+
113+
expect(mocks.recoveryDeliver).toHaveBeenCalledOnce();
114+
expect(await loadPendingDelivery(queueId)).toBeNull();
115+
expect(queueStatus(queueId)).toBe("completed");
116+
});
117+
118+
it("does not blindly resend an ambiguous platform attempt", async () => {
119+
const queueId = await enqueueNotice();
120+
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
121+
await markAttempt(request);
122+
return { status: "failed", error: new Error("platform outcome unknown") };
123+
});
124+
125+
await deliverNotice(queueId);
126+
127+
expect(mocks.recoveryDeliver).not.toHaveBeenCalled();
128+
expect(await loadPendingDelivery(queueId)).toBeNull();
129+
expect(queueStatus(queueId)).toBe("failed");
130+
});
131+
132+
it("dead-letters a permanent provider rejection without replay", async () => {
133+
const queueId = await enqueueNotice();
134+
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
135+
await markAttempt(request);
136+
return {
137+
status: "failed",
138+
error: new PlatformMessageNotDispatchedError("payload rejected", {
139+
cause: new Error("invalid payload"),
140+
retryable: false,
141+
}),
142+
};
143+
});
144+
145+
await deliverNotice(queueId);
146+
147+
expect(mocks.recoveryDeliver).not.toHaveBeenCalled();
148+
expect(await loadPendingDelivery(queueId)).toBeNull();
149+
expect(queueStatus(queueId)).toBe("failed");
150+
});
151+
152+
it("preserves the shipped 45-attempt budget before dead-lettering", async () => {
153+
const queueId = await enqueueNotice();
154+
const retryableFailure = () =>
155+
new PlatformMessageNotDispatchedError("transport unavailable before dispatch", {
156+
cause: new Error("transport unavailable"),
157+
});
158+
mocks.sendDurableMessageBatch.mockImplementationOnce(async (request) => {
159+
await markAttempt(request);
160+
return { status: "failed", error: retryableFailure() };
161+
});
162+
mocks.recoveryDeliver.mockImplementation(async (request) => {
163+
await markAttempt(request);
164+
throw retryableFailure();
165+
});
166+
167+
await deliverNotice(queueId);
168+
169+
expect(mocks.sendDurableMessageBatch).toHaveBeenCalledOnce();
170+
expect(mocks.recoveryDeliver).toHaveBeenCalledTimes(44);
171+
expect(queueStatus(queueId)).toBe("failed");
172+
});
173+
});

0 commit comments

Comments
 (0)