Skip to content

Commit ce73ea8

Browse files
AnobleSCMclaude
andcommitted
fix(server): a provider event write retries a brief database lock
When a provider event write times out on the database lock, the ingestion stream fails and the whole run is marked "Provider turn failed", although the provider is fine and the lock clears moments later. A write that failed with LockTimeoutError was rolled back, so the ingestor now repeats it up to two more times. busy_timeout already waits before each attempt, so the retry adds no delay of its own. Other write errors fail as before. Refs #15542 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 16918b5 commit ce73ea8

2 files changed

Lines changed: 139 additions & 3 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import * as Effect from "effect/Effect";
3333
import * as FileSystem from "effect/FileSystem";
3434
import * as Layer from "effect/Layer";
3535
import * as SqlClient from "effect/sql/SqlClient";
36+
import * as SqlError from "effect/sql/SqlError";
3637
import * as Stream from "effect/Stream";
3738
import * as TestClock from "effect/testing/TestClock";
3839

@@ -2242,3 +2243,123 @@ layer("ProviderEventIngestorV2", (it) => {
22422243
}),
22432244
);
22442245
});
2246+
2247+
const lockTimeoutWriteError = new EventSink.EventSinkWriteError({
2248+
eventCount: 1,
2249+
cause: new SqlError.SqlError({
2250+
reason: new SqlError.LockTimeoutError({
2251+
cause: new Error("database is locked"),
2252+
message: "database is locked",
2253+
}),
2254+
}),
2255+
});
2256+
2257+
/** The ingestor sees an event sink whose next writes fail with `errors`, in order. */
2258+
function layerFailingNextWrites(errors: ReadonlyArray<EventSink.EventSinkV2Error>) {
2259+
const attempts = { count: 0 };
2260+
const layerFailingEventSink = Layer.effect(
2261+
EventSink.EventSinkV2,
2262+
Effect.gen(function* () {
2263+
const eventSink = yield* EventSink.EventSinkV2;
2264+
return EventSink.EventSinkV2.of({
2265+
...eventSink,
2266+
write: (input) =>
2267+
Effect.suspend(() => {
2268+
const error = errors[attempts.count];
2269+
attempts.count += 1;
2270+
return error === undefined ? eventSink.write(input) : Effect.fail(error);
2271+
}),
2272+
});
2273+
}),
2274+
).pipe(Layer.provide(layerTestEventSink));
2275+
const layer = Layer.mergeAll(
2276+
layerTestDatabase,
2277+
layerTestStores,
2278+
layerTestEventSink,
2279+
IdAllocator.layer,
2280+
ProviderEventIngestor.layer.pipe(
2281+
Layer.provide(
2282+
Layer.mergeAll(
2283+
layerTestStores,
2284+
layerFailingEventSink,
2285+
IdAllocator.layer,
2286+
ThreadCommandExecutor.layer,
2287+
layerGit,
2288+
),
2289+
),
2290+
),
2291+
);
2292+
return { attempts, layer };
2293+
}
2294+
2295+
const ingestCompletedProviderTurn = Effect.gen(function* () {
2296+
const now = yield* DateTime.now;
2297+
const eventSink = yield* EventSink.EventSinkV2;
2298+
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2299+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
2300+
const threadEvent = yield* threadCreatedEvent(now);
2301+
yield* eventSink.write({ events: [threadEvent] });
2302+
const providerSessionId = yield* idAllocator.allocate.providerSession({
2303+
providerInstanceId: modelSelection.instanceId,
2304+
threadId: threadEvent.threadId,
2305+
});
2306+
return yield* ingestor.ingestNormalized({
2307+
providerSessionId,
2308+
providerInstanceId: modelSelection.instanceId,
2309+
threadId: threadEvent.threadId,
2310+
event: {
2311+
type: "provider_turn.updated",
2312+
driver: CODEX_DRIVER,
2313+
providerTurn: {
2314+
id: idAllocator.derive.providerTurn({ driver: CODEX_DRIVER, nativeTurnId: "locked-turn" }),
2315+
providerThreadId: idAllocator.derive.providerThread({
2316+
driver: CODEX_DRIVER,
2317+
nativeThreadId: "locked-thread",
2318+
}),
2319+
nodeId: NodeId.make("node:locked-turn"),
2320+
runAttemptId: null,
2321+
nativeTurnRef: null,
2322+
ordinal: 1,
2323+
status: "completed",
2324+
startedAt: now,
2325+
completedAt: now,
2326+
},
2327+
},
2328+
});
2329+
});
2330+
2331+
it.effect("retries a provider event write that timed out on the database lock", () => {
2332+
const { attempts, layer } = layerFailingNextWrites([
2333+
lockTimeoutWriteError,
2334+
lockTimeoutWriteError,
2335+
]);
2336+
return Effect.gen(function* () {
2337+
const stored = yield* ingestCompletedProviderTurn;
2338+
assert.equal(attempts.count, 3);
2339+
assert.isNotEmpty(stored);
2340+
}).pipe(Effect.provide(layer));
2341+
});
2342+
2343+
it.effect("fails a provider event write that stays locked after its retries", () => {
2344+
const { attempts, layer } = layerFailingNextWrites([
2345+
lockTimeoutWriteError,
2346+
lockTimeoutWriteError,
2347+
lockTimeoutWriteError,
2348+
]);
2349+
return Effect.gen(function* () {
2350+
const error = yield* Effect.flip(ingestCompletedProviderTurn);
2351+
assert.equal(attempts.count, 3);
2352+
assert.equal(error._tag, "ProviderEventPublishError");
2353+
}).pipe(Effect.provide(layer));
2354+
});
2355+
2356+
it.effect("does not retry a provider event write that failed for another reason", () => {
2357+
const { attempts, layer } = layerFailingNextWrites([
2358+
new EventSink.EventSinkWriteError({ eventCount: 1, cause: new Error("constraint failed") }),
2359+
]);
2360+
return Effect.gen(function* () {
2361+
const error = yield* Effect.flip(ingestCompletedProviderTurn);
2362+
assert.equal(attempts.count, 1);
2363+
assert.equal(error._tag, "ProviderEventPublishError");
2364+
}).pipe(Effect.provide(layer));
2365+
});

‎apps/server/src/orchestration-v2/ProviderEventIngestor.ts‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ import * as Context from "effect/Context";
2424
import * as DateTime from "effect/DateTime";
2525
import * as Effect from "effect/Effect";
2626
import * as Layer from "effect/Layer";
27+
import * as Predicate from "effect/Predicate";
2728
import * as Schema from "effect/Schema";
29+
import { isSqlError } from "effect/sql/SqlError";
2830

2931
import { AgentScope } from "@t3tools/shared/AgentScope";
3032
import { isTemporaryWorktreeBranch } from "@t3tools/shared/git";
@@ -67,6 +69,19 @@ export class ProviderEventPublishError extends Schema.TaggedError<ProviderEventP
6769
}
6870
}
6971

72+
// A write that timed out on the database lock was rolled back, so it is safe
73+
// to repeat; busy_timeout already waited before each attempt. Failing it would
74+
// fail the whole run over a brief lock.
75+
const isLockTimeout = (cause: unknown): boolean => {
76+
let value = cause;
77+
for (let depth = 0; depth < 6 && Predicate.isObject(value); depth += 1) {
78+
if (isSqlError(value)) return value.reason._tag === "LockTimeoutError";
79+
value = "cause" in value ? value.cause : undefined;
80+
}
81+
return false;
82+
};
83+
const retryLockTimeout = Effect.retry({ while: isLockTimeout, times: 2 });
84+
7085
export const ProviderEventIngestorV2Error = Schema.Union([
7186
ProviderEventNormalizeError,
7287
ProviderEventPublishError,
@@ -742,7 +757,7 @@ export const layer: Layer.Layer<
742757
...input.writeIfProviderThreadOwner,
743758
events,
744759
})
745-
.pipe(Effect.mapError(mapWriteError));
760+
.pipe(retryLockTimeout, Effect.mapError(mapWriteError));
746761
return ownerResult.storedEvents;
747762
}
748763
if (input.writeIfRunCurrent === undefined) {
@@ -752,7 +767,7 @@ export const layer: Layer.Layer<
752767
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
753768
events,
754769
})
755-
.pipe(Effect.mapError(mapWriteError));
770+
.pipe(retryLockTimeout, Effect.mapError(mapWriteError));
756771
}
757772
const result = yield* eventSink
758773
.writeIfRunCurrent({
@@ -762,7 +777,7 @@ export const layer: Layer.Layer<
762777
...input.writeIfRunCurrent,
763778
events,
764779
})
765-
.pipe(Effect.mapError(mapWriteError));
780+
.pipe(retryLockTimeout, Effect.mapError(mapWriteError));
766781
return result.storedEvents;
767782
}).pipe(
768783
Effect.flatMap((storedEvents) =>

0 commit comments

Comments
 (0)