Skip to content

Commit 97548db

Browse files
fix(server): retry provider event writes after brief SQLite locks (pingdotgg#24571)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent ce8d644 commit 97548db

2 files changed

Lines changed: 171 additions & 24 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts‎

Lines changed: 158 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,23 @@ import {
1717
type OrchestrationV2TurnItem,
1818
ProviderDriverKind,
1919
ProviderInstanceId,
20-
type ProviderThreadId,
20+
ProviderThreadId,
2121
ProviderTurnId,
2222
RunAttemptId,
2323
RunId,
2424
RuntimeRequestId,
2525
TurnItemId,
2626
} from "@t3tools/contracts";
27+
import * as Cause from "effect/Cause";
2728
import * as DateTime from "effect/DateTime";
2829
import * as Deferred from "effect/Deferred";
2930
import * as Fiber from "effect/Fiber";
3031
import * as Effect from "effect/Effect";
32+
import * as Exit from "effect/Exit";
3133
import * as FileSystem from "effect/FileSystem";
3234
import * as Layer from "effect/Layer";
3335
import * as SqlClient from "effect/sql/SqlClient";
36+
import * as SqlError from "effect/sql/SqlError";
3437
import * as Stream from "effect/Stream";
3538

3639
import { toolOutputImages } from "@t3tools/shared/toolOutput";
@@ -65,25 +68,63 @@ const layerGit = GitVcsDriver.layer.pipe(
6568
Layer.provideMerge(NodeServices.layer),
6669
);
6770

68-
const layerTest = Layer.mergeAll(
69-
layerTestDatabase,
70-
layerTestStores,
71-
layerTestEventSink,
72-
IdAllocator.layer,
73-
ThreadCommandExecutor.layer,
74-
layerGit,
75-
ProviderEventIngestor.layer.pipe(
76-
Layer.provide(
77-
Layer.mergeAll(
78-
layerTestStores,
79-
layerTestEventSink,
80-
IdAllocator.layer,
81-
ThreadCommandExecutor.layer,
82-
layerGit,
71+
const makeLayerTest = <E>(ingestorEventSink: Layer.Layer<EventSink.EventSinkV2, E>) =>
72+
Layer.mergeAll(
73+
layerTestDatabase,
74+
layerTestStores,
75+
layerTestEventSink,
76+
IdAllocator.layer,
77+
ThreadCommandExecutor.layer,
78+
layerGit,
79+
ProviderEventIngestor.layer.pipe(
80+
Layer.provide(
81+
Layer.mergeAll(
82+
layerTestStores,
83+
ingestorEventSink,
84+
IdAllocator.layer,
85+
ThreadCommandExecutor.layer,
86+
layerGit,
87+
),
8388
),
8489
),
85-
),
86-
);
90+
);
91+
const layerTest = makeLayerTest(layerTestEventSink);
92+
93+
// EventSink reports a lock that outlasted busy_timeout at BEGIN IMMEDIATE this way.
94+
const lockTimeout = new EventSink.EventSinkWriteError({
95+
eventCount: 1,
96+
cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }),
97+
});
98+
const constraintFailure = new EventSink.EventSinkWriteError({
99+
eventCount: 1,
100+
cause: new SqlError.SqlError({ reason: new SqlError.ConstraintError({ cause: null }) }),
101+
});
102+
103+
/** The ingestor's next sink writes of `events` fail with `errors`, in order; the rest go through. */
104+
const layerFailingWrites = (
105+
errors: ReadonlyArray<EventSink.EventSinkWriteError>,
106+
matches: (events: ReadonlyArray<OrchestrationV2DomainEvent>) => boolean = () => true,
107+
) =>
108+
makeLayerTest(
109+
Layer.effect(
110+
EventSink.EventSinkV2,
111+
Effect.gen(function* () {
112+
const eventSink = yield* EventSink.EventSinkV2;
113+
const pending = [...errors];
114+
const failNext = <A, R>(
115+
events: ReadonlyArray<OrchestrationV2DomainEvent>,
116+
write: Effect.Effect<A, EventSink.EventSinkV2Error, R>,
117+
) => Effect.suspend(() => (matches(events) ? (pending.shift() ?? write) : write));
118+
return EventSink.EventSinkV2.of({
119+
...eventSink,
120+
write: (input) => failNext(input.events, eventSink.write(input)),
121+
writeIfRunCurrent: (input) => failNext(input.events, eventSink.writeIfRunCurrent(input)),
122+
writeIfProviderThreadOwner: (input) =>
123+
failNext(input.events, eventSink.writeIfProviderThreadOwner(input)),
124+
});
125+
}),
126+
).pipe(Layer.provide(layerTestEventSink)),
127+
);
87128

88129
const makeSharedWorktree = Effect.gen(function* () {
89130
const fs = yield* FileSystem.FileSystem;
@@ -2005,7 +2046,7 @@ layer("ProviderEventIngestorV2", (it) => {
20052046
}),
20062047
);
20072048

2008-
it.effect("moves a native subagent's thread to the model its provider reports later", () =>
2049+
it.effect("moves a native subagent's thread to its later model through a database lock", () =>
20092050
Effect.gen(function* () {
20102051
const now = yield* DateTime.now;
20112052
const eventSink = yield* EventSink.EventSinkV2;
@@ -2124,6 +2165,103 @@ layer("ProviderEventIngestorV2", (it) => {
21242165
(yield* projectionStore.getThread(childThreadId)).modelSelection,
21252166
reportedSelection,
21262167
);
2127-
}),
2168+
}).pipe(
2169+
// Fresh: the shared test layer has already built the ingestor this replaces.
2170+
Effect.provide(
2171+
Layer.fresh(
2172+
layerFailingWrites([lockTimeout, lockTimeout], (events) =>
2173+
events.some((event) => event.type === "thread.model-selection-updated"),
2174+
),
2175+
),
2176+
),
2177+
),
21282178
);
21292179
});
2180+
2181+
const ingestCompletedProviderTurn = (
2182+
guard: Pick<
2183+
Parameters<ProviderEventIngestor.ProviderEventIngestorV2Shape["ingestNormalized"]>[0],
2184+
"writeIfRunCurrent" | "writeIfProviderThreadOwner"
2185+
>,
2186+
) =>
2187+
Effect.gen(function* () {
2188+
const now = yield* DateTime.now;
2189+
const eventSink = yield* EventSink.EventSinkV2;
2190+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2191+
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2192+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
2193+
const threadEvent = yield* threadCreatedEvent(now);
2194+
yield* eventSink.write({ events: [threadEvent] });
2195+
const providerTurnId = ProviderTurnId.make("provider-turn:locked");
2196+
const exit = yield* ingestor
2197+
.ingestNormalized({
2198+
providerSessionId: yield* idAllocator.allocate.providerSession({
2199+
providerInstanceId: modelSelection.instanceId,
2200+
threadId: threadEvent.threadId,
2201+
}),
2202+
providerInstanceId: modelSelection.instanceId,
2203+
threadId: threadEvent.threadId,
2204+
event: {
2205+
type: "provider_turn.updated",
2206+
driver: CODEX_DRIVER,
2207+
providerTurn: {
2208+
id: providerTurnId,
2209+
providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2210+
nodeId: NodeId.make("node:locked-turn"),
2211+
runAttemptId: null,
2212+
nativeTurnRef: null,
2213+
ordinal: 1,
2214+
status: "completed",
2215+
startedAt: now,
2216+
completedAt: now,
2217+
},
2218+
},
2219+
...guard,
2220+
})
2221+
.pipe(Effect.exit);
2222+
const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
2223+
return { exit, providerTurnId, persisted: projection.providerTurns.map((turn) => turn.id) };
2224+
});
2225+
2226+
// Neither guard matches a stored run, so a guarded write correctly commits nothing.
2227+
const runId = RunId.make("run:locked");
2228+
const activeAttemptId = RunAttemptId.make("attempt:locked");
2229+
it.effect.each([
2230+
{ route: "write", guard: {} },
2231+
{
2232+
route: "writeIfRunCurrent",
2233+
guard: { writeIfRunCurrent: { runId, activeAttemptId, expectedStatus: "running" } },
2234+
},
2235+
{
2236+
route: "writeIfProviderThreadOwner",
2237+
guard: {
2238+
writeIfProviderThreadOwner: {
2239+
providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2240+
runId,
2241+
activeAttemptId,
2242+
expectedLastRunOrdinal: 1,
2243+
},
2244+
},
2245+
},
2246+
] as const)("a provider event $route outlasts a brief database lock", ({ route, guard }) =>
2247+
Effect.gen(function* () {
2248+
const { exit, providerTurnId, persisted } = yield* ingestCompletedProviderTurn(guard);
2249+
assert.isTrue(Exit.isSuccess(exit));
2250+
assert.deepEqual(persisted, route === "write" ? [providerTurnId] : []);
2251+
}).pipe(Effect.provide(layerFailingWrites([lockTimeout, lockTimeout]))),
2252+
);
2253+
2254+
// The write after the queued errors would succeed, so failing proves the retries stopped.
2255+
it.effect.each([
2256+
{ failure: "stays locked", errors: [lockTimeout, lockTimeout, lockTimeout] },
2257+
{ failure: "fails for another reason", errors: [constraintFailure] },
2258+
])("fails a provider event write that $failure", ({ errors }) =>
2259+
Effect.gen(function* () {
2260+
const { exit, persisted } = yield* ingestCompletedProviderTurn({});
2261+
assert.instanceOf(
2262+
Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined,
2263+
ProviderEventIngestor.ProviderEventPublishError,
2264+
);
2265+
assert.isEmpty(persisted);
2266+
}).pipe(Effect.provide(layerFailingWrites(errors))),
2267+
);

‎apps/server/src/orchestration-v2/ProviderEventIngestor.ts‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import * as DateTime from "effect/DateTime";
2727
import * as Effect from "effect/Effect";
2828
import * as Layer from "effect/Layer";
2929
import * as Schema from "effect/Schema";
30+
import * as SqlError from "effect/sql/SqlError";
3031

3132
import { AgentScope } from "@t3tools/shared/AgentScope";
3233
import { isTemporaryWorktreeBranch } from "@t3tools/shared/git";
@@ -69,6 +70,14 @@ export class ProviderEventPublishError extends Schema.TaggedError<ProviderEventP
6970
}
7071
}
7172

73+
// A write that outlasts busy_timeout at BEGIN IMMEDIATE fails before anything
74+
// commits, and the sink publishes only after a commit, so repeating the whole
75+
// write is safe. Failing it would fail the run over a brief lock.
76+
const isLockTimeout = (error: EventSink.EventSinkV2Error) =>
77+
SqlError.isSqlError(error.cause) && error.cause.reason._tag === "LockTimeoutError";
78+
const retryLockTimeout = <A, R>(write: Effect.Effect<A, EventSink.EventSinkV2Error, R>) =>
79+
Effect.retry(write, { while: isLockTimeout, times: 2 });
80+
7281
export const ProviderEventIngestorV2Error = Schema.Union([
7382
ProviderEventNormalizeError,
7483
ProviderEventPublishError,
@@ -450,7 +459,7 @@ export const layer: Layer.Layer<
450459
},
451460
occurredAt: now,
452461
});
453-
return yield* eventSink.write({ events: [event] });
462+
return yield* eventSink.write({ events: [event] }).pipe(retryLockTimeout);
454463
}),
455464
);
456465
},
@@ -767,7 +776,7 @@ export const layer: Layer.Layer<
767776
...input.writeIfProviderThreadOwner,
768777
events,
769778
})
770-
.pipe(Effect.mapError(mapWriteError));
779+
.pipe(retryLockTimeout, Effect.mapError(mapWriteError));
771780
return ownerResult.storedEvents;
772781
}
773782
if (input.writeIfRunCurrent === undefined) {
@@ -777,7 +786,7 @@ export const layer: Layer.Layer<
777786
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
778787
events,
779788
})
780-
.pipe(Effect.mapError(mapWriteError));
789+
.pipe(retryLockTimeout, Effect.mapError(mapWriteError));
781790
}
782791
const result = yield* eventSink
783792
.writeIfRunCurrent({
@@ -787,7 +796,7 @@ export const layer: Layer.Layer<
787796
...input.writeIfRunCurrent,
788797
events,
789798
})
790-
.pipe(Effect.mapError(mapWriteError));
799+
.pipe(retryLockTimeout, Effect.mapError(mapWriteError));
791800
return result.storedEvents;
792801
},
793802
(effect, input) =>

0 commit comments

Comments
 (0)