Skip to content

Commit 08e5e1b

Browse files
fix(server): a provider event write retries a brief database lock
A provider event write that waits out busy_timeout on BEGIN IMMEDIATE fails with a LockTimeoutError, which fails the run although the provider is fine and the lock clears moments later. The ingestor now repeats such a write up to two more times. Nothing has committed when BEGIN IMMEDIATE times out and the sink publishes only after a commit, so the repeat is safe and the guarded routes re-run their guards. The native-subagent model write gets the same policy, as its failure also fails the run after the primary event committed. Other write errors fail on the first attempt as before. Takeover of #21092 by AnobleSCM. The retry is a typed predicate on the sink error instead of a nested-cause walk; the tests check persisted behavior on all three write routes, the retry bound, and a non-lock SQL error. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 136687a commit 08e5e1b

2 files changed

Lines changed: 139 additions & 143 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts‎

Lines changed: 130 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,19 @@ import {
1919
ProviderDriverKind,
2020
ProviderInstanceId,
2121
PlanId,
22-
type ProviderThreadId,
22+
ProviderThreadId,
2323
ProviderTurnId,
2424
RunAttemptId,
2525
RunId,
2626
RuntimeRequestId,
2727
TurnItemId,
2828
} from "@t3tools/contracts";
29+
import * as Cause from "effect/Cause";
2930
import * as DateTime from "effect/DateTime";
3031
import * as Deferred from "effect/Deferred";
3132
import * as Fiber from "effect/Fiber";
3233
import * as Effect from "effect/Effect";
34+
import * as Exit from "effect/Exit";
3335
import * as FileSystem from "effect/FileSystem";
3436
import * as Layer from "effect/Layer";
3537
import * as SqlClient from "effect/sql/SqlClient";
@@ -69,25 +71,27 @@ const layerGit = GitVcsDriver.layer.pipe(
6971
Layer.provideMerge(NodeServices.layer),
7072
);
7173

72-
const layerTest = Layer.mergeAll(
73-
layerTestDatabase,
74-
layerTestStores,
75-
layerTestEventSink,
76-
IdAllocator.layer,
77-
ThreadCommandExecutor.layer,
78-
layerGit,
79-
ProviderEventIngestor.layer.pipe(
80-
Layer.provide(
81-
Layer.mergeAll(
82-
layerTestStores,
83-
layerTestEventSink,
84-
IdAllocator.layer,
85-
ThreadCommandExecutor.layer,
86-
layerGit,
74+
const makeLayerTest = <E>(ingestorEventSink: Layer.Layer<EventSink.EventSinkV2, E>) =>
75+
Layer.mergeAll(
76+
layerTestDatabase,
77+
layerTestStores,
78+
layerTestEventSink,
79+
IdAllocator.layer,
80+
ThreadCommandExecutor.layer,
81+
layerGit,
82+
ProviderEventIngestor.layer.pipe(
83+
Layer.provide(
84+
Layer.mergeAll(
85+
layerTestStores,
86+
ingestorEventSink,
87+
IdAllocator.layer,
88+
ThreadCommandExecutor.layer,
89+
layerGit,
90+
),
8791
),
8892
),
89-
),
90-
);
93+
);
94+
const layerTest = makeLayerTest(layerTestEventSink);
9195

9296
const makeSharedWorktree = Effect.gen(function* () {
9397
const fs = yield* FileSystem.FileSystem;
@@ -2244,122 +2248,120 @@ layer("ProviderEventIngestorV2", (it) => {
22442248
);
22452249
});
22462250

2247-
const lockTimeoutWriteError = new EventSink.EventSinkWriteError({
2251+
// EventSink reports a lock that outlasted busy_timeout at BEGIN IMMEDIATE this way.
2252+
const lockTimeout = new EventSink.EventSinkWriteError({
22482253
eventCount: 1,
2249-
cause: new SqlError.SqlError({
2250-
reason: new SqlError.LockTimeoutError({
2251-
cause: new Error("database is locked"),
2252-
message: "database is locked",
2253-
}),
2254-
}),
2254+
cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }),
2255+
});
2256+
const constraintFailure = new EventSink.EventSinkWriteError({
2257+
eventCount: 1,
2258+
cause: new SqlError.SqlError({ reason: new SqlError.ConstraintError({ cause: null }) }),
22552259
});
22562260

2257-
/** The ingestor sees an event sink whose next writes fail with `errors`, in order. */
2258-
function layerFailingNextWrites(errors: ReadonlyArray<EventSink.EventSinkV2Error>) {
2259-
const attempts = { count: 0 };
2260-
const layerFailingEventSink = Layer.effect(
2261-
EventSink.EventSinkV2,
2262-
Effect.gen(function* () {
2263-
const eventSink = yield* EventSink.EventSinkV2;
2264-
return EventSink.EventSinkV2.of({
2265-
...eventSink,
2266-
write: (input) =>
2267-
Effect.suspend(() => {
2268-
const error = errors[attempts.count];
2269-
attempts.count += 1;
2270-
return error === undefined ? eventSink.write(input) : Effect.fail(error);
2271-
}),
2272-
});
2273-
}),
2274-
).pipe(Layer.provide(layerTestEventSink));
2275-
const layer = Layer.mergeAll(
2276-
layerTestDatabase,
2277-
layerTestStores,
2278-
layerTestEventSink,
2279-
IdAllocator.layer,
2280-
ProviderEventIngestor.layer.pipe(
2281-
Layer.provide(
2282-
Layer.mergeAll(
2283-
layerTestStores,
2284-
layerFailingEventSink,
2285-
IdAllocator.layer,
2286-
ThreadCommandExecutor.layer,
2287-
layerGit,
2288-
),
2289-
),
2290-
),
2261+
/** The ingestor's next sink writes fail with `errors`, in order; later writes go through. */
2262+
const layerFailingNextWrites = (errors: ReadonlyArray<EventSink.EventSinkWriteError>) =>
2263+
makeLayerTest(
2264+
Layer.effect(
2265+
EventSink.EventSinkV2,
2266+
Effect.gen(function* () {
2267+
const eventSink = yield* EventSink.EventSinkV2;
2268+
const pending = [...errors];
2269+
const failNext = <A, R>(write: Effect.Effect<A, EventSink.EventSinkV2Error, R>) =>
2270+
Effect.suspend(() => pending.shift() ?? write);
2271+
return EventSink.EventSinkV2.of({
2272+
...eventSink,
2273+
write: (input) => failNext(eventSink.write(input)),
2274+
writeIfRunCurrent: (input) => failNext(eventSink.writeIfRunCurrent(input)),
2275+
writeIfProviderThreadOwner: (input) =>
2276+
failNext(eventSink.writeIfProviderThreadOwner(input)),
2277+
});
2278+
}),
2279+
).pipe(Layer.provide(layerTestEventSink)),
22912280
);
2292-
return { attempts, layer };
2293-
}
22942281

2295-
const ingestCompletedProviderTurn = Effect.gen(function* () {
2296-
const now = yield* DateTime.now;
2297-
const eventSink = yield* EventSink.EventSinkV2;
2298-
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2299-
const idAllocator = yield* IdAllocator.IdAllocatorV2;
2300-
const threadEvent = yield* threadCreatedEvent(now);
2301-
yield* eventSink.write({ events: [threadEvent] });
2302-
const providerSessionId = yield* idAllocator.allocate.providerSession({
2303-
providerInstanceId: modelSelection.instanceId,
2304-
threadId: threadEvent.threadId,
2305-
});
2306-
return yield* ingestor.ingestNormalized({
2307-
providerSessionId,
2308-
providerInstanceId: modelSelection.instanceId,
2309-
threadId: threadEvent.threadId,
2310-
event: {
2311-
type: "provider_turn.updated",
2312-
driver: CODEX_DRIVER,
2313-
providerTurn: {
2314-
id: idAllocator.derive.providerTurn({ driver: CODEX_DRIVER, nativeTurnId: "locked-turn" }),
2315-
providerThreadId: idAllocator.derive.providerThread({
2316-
driver: CODEX_DRIVER,
2317-
nativeThreadId: "locked-thread",
2282+
const ingestCompletedProviderTurn = (
2283+
guard: Pick<
2284+
Parameters<ProviderEventIngestor.ProviderEventIngestorV2Shape["ingestNormalized"]>[0],
2285+
"writeIfRunCurrent" | "writeIfProviderThreadOwner"
2286+
>,
2287+
) =>
2288+
Effect.gen(function* () {
2289+
const now = yield* DateTime.now;
2290+
const eventSink = yield* EventSink.EventSinkV2;
2291+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2292+
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2293+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
2294+
const threadEvent = yield* threadCreatedEvent(now);
2295+
yield* eventSink.write({ events: [threadEvent] });
2296+
const providerTurnId = ProviderTurnId.make("provider-turn:locked");
2297+
const exit = yield* ingestor
2298+
.ingestNormalized({
2299+
providerSessionId: yield* idAllocator.allocate.providerSession({
2300+
providerInstanceId: modelSelection.instanceId,
2301+
threadId: threadEvent.threadId,
23182302
}),
2319-
nodeId: NodeId.make("node:locked-turn"),
2320-
runAttemptId: null,
2321-
nativeTurnRef: null,
2322-
ordinal: 1,
2323-
status: "completed",
2324-
startedAt: now,
2325-
completedAt: now,
2326-
},
2327-
},
2303+
providerInstanceId: modelSelection.instanceId,
2304+
threadId: threadEvent.threadId,
2305+
event: {
2306+
type: "provider_turn.updated",
2307+
driver: CODEX_DRIVER,
2308+
providerTurn: {
2309+
id: providerTurnId,
2310+
providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2311+
nodeId: NodeId.make("node:locked-turn"),
2312+
runAttemptId: null,
2313+
nativeTurnRef: null,
2314+
ordinal: 1,
2315+
status: "completed",
2316+
startedAt: now,
2317+
completedAt: now,
2318+
},
2319+
},
2320+
...guard,
2321+
})
2322+
.pipe(Effect.exit);
2323+
const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
2324+
return { exit, providerTurnId, persisted: projection.providerTurns.map((turn) => turn.id) };
23282325
});
2329-
});
2330-
2331-
it.effect("retries a provider event write that timed out on the database lock", () => {
2332-
const { attempts, layer } = layerFailingNextWrites([
2333-
lockTimeoutWriteError,
2334-
lockTimeoutWriteError,
2335-
]);
2336-
return Effect.gen(function* () {
2337-
const stored = yield* ingestCompletedProviderTurn;
2338-
assert.equal(attempts.count, 3);
2339-
assert.isNotEmpty(stored);
2340-
}).pipe(Effect.provide(layer));
2341-
});
23422326

2343-
it.effect("fails a provider event write that stays locked after its retries", () => {
2344-
const { attempts, layer } = layerFailingNextWrites([
2345-
lockTimeoutWriteError,
2346-
lockTimeoutWriteError,
2347-
lockTimeoutWriteError,
2348-
]);
2349-
return Effect.gen(function* () {
2350-
const error = yield* Effect.flip(ingestCompletedProviderTurn);
2351-
assert.equal(attempts.count, 3);
2352-
assert.equal(error._tag, "ProviderEventPublishError");
2353-
}).pipe(Effect.provide(layer));
2354-
});
2327+
// Neither guard matches a stored run, so a guarded write commits nothing.
2328+
const runId = RunId.make("run:locked");
2329+
const activeAttemptId = RunAttemptId.make("attempt:locked");
2330+
it.effect.each([
2331+
{ route: "write", guard: {} },
2332+
{
2333+
route: "writeIfRunCurrent",
2334+
guard: { writeIfRunCurrent: { runId, activeAttemptId, expectedStatus: "running" } },
2335+
},
2336+
{
2337+
route: "writeIfProviderThreadOwner",
2338+
guard: {
2339+
writeIfProviderThreadOwner: {
2340+
providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2341+
runId,
2342+
activeAttemptId,
2343+
expectedLastRunOrdinal: 1,
2344+
},
2345+
},
2346+
},
2347+
] as const)("a provider event $route outlasts a brief database lock", ({ route, guard }) =>
2348+
Effect.gen(function* () {
2349+
const { exit, providerTurnId, persisted } = yield* ingestCompletedProviderTurn(guard);
2350+
assert.isTrue(Exit.isSuccess(exit));
2351+
assert.deepEqual(persisted, route === "write" ? [providerTurnId] : []);
2352+
}).pipe(Effect.provide(layerFailingNextWrites([lockTimeout, lockTimeout]))),
2353+
);
23552354

2356-
it.effect("does not retry a provider event write that failed for another reason", () => {
2357-
const { attempts, layer } = layerFailingNextWrites([
2358-
new EventSink.EventSinkWriteError({ eventCount: 1, cause: new Error("constraint failed") }),
2359-
]);
2360-
return Effect.gen(function* () {
2361-
const error = yield* Effect.flip(ingestCompletedProviderTurn);
2362-
assert.equal(attempts.count, 1);
2363-
assert.equal(error._tag, "ProviderEventPublishError");
2364-
}).pipe(Effect.provide(layer));
2365-
});
2355+
it.effect.each([
2356+
{ failure: "stays locked", errors: [lockTimeout, lockTimeout, lockTimeout] },
2357+
{ failure: "fails for another reason", errors: [constraintFailure] },
2358+
])("fails a provider event write that $failure", ({ errors }) =>
2359+
Effect.gen(function* () {
2360+
const { exit, persisted } = yield* ingestCompletedProviderTurn({});
2361+
assert.instanceOf(
2362+
Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined,
2363+
ProviderEventIngestor.ProviderEventPublishError,
2364+
);
2365+
assert.isEmpty(persisted);
2366+
}).pipe(Effect.provide(layerFailingNextWrites(errors))),
2367+
);

‎apps/server/src/orchestration-v2/ProviderEventIngestor.ts‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,8 @@ import * as Context from "effect/Context";
2424
import * as DateTime from "effect/DateTime";
2525
import * as Effect from "effect/Effect";
2626
import * as Layer from "effect/Layer";
27-
import * as Predicate from "effect/Predicate";
2827
import * as Schema from "effect/Schema";
29-
import { isSqlError } from "effect/sql/SqlError";
28+
import * as SqlError from "effect/sql/SqlError";
3029

3130
import { AgentScope } from "@t3tools/shared/AgentScope";
3231
import { isTemporaryWorktreeBranch } from "@t3tools/shared/git";
@@ -69,18 +68,13 @@ export class ProviderEventPublishError extends Schema.TaggedError<ProviderEventP
6968
}
7069
}
7170

72-
// A write that timed out on the database lock was rolled back, so it is safe
73-
// to repeat; busy_timeout already waited before each attempt. Failing it would
74-
// fail the whole run over a brief lock.
75-
const isLockTimeout = (cause: unknown): boolean => {
76-
let value = cause;
77-
for (let depth = 0; depth < 6 && Predicate.isObject(value); depth += 1) {
78-
if (isSqlError(value)) return value.reason._tag === "LockTimeoutError";
79-
value = "cause" in value ? value.cause : undefined;
80-
}
81-
return false;
82-
};
83-
const retryLockTimeout = Effect.retry({ while: isLockTimeout, times: 2 });
71+
// A write that outlasts busy_timeout at BEGIN IMMEDIATE fails before anything
72+
// commits, and the sink publishes only after a commit, so repeating the whole
73+
// write is safe. Failing it would fail the run over a brief lock.
74+
const isLockTimeout = (error: EventSink.EventSinkV2Error) =>
75+
SqlError.isSqlError(error.cause) && error.cause.reason._tag === "LockTimeoutError";
76+
const retryLockTimeout = <A, R>(write: Effect.Effect<A, EventSink.EventSinkV2Error, R>) =>
77+
Effect.retry(write, { while: isLockTimeout, times: 2 });
8478

8579
export const ProviderEventIngestorV2Error = Schema.Union([
8680
ProviderEventNormalizeError,
@@ -448,7 +442,7 @@ export const layer: Layer.Layer<
448442
},
449443
occurredAt: now,
450444
});
451-
return yield* eventSink.write({ events: [event] });
445+
return yield* eventSink.write({ events: [event] }).pipe(retryLockTimeout);
452446
}),
453447
);
454448
},

0 commit comments

Comments
 (0)