Skip to content

Commit 30fd973

Browse files
authored
fix(server): publish committed events before writer cancellation (pingdotgg#24836)
1 parent 056b445 commit 30fd973

3 files changed

Lines changed: 175 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2335,6 +2335,72 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
23352335
}),
23362336
);
23372337

2338+
it.effect(
2339+
"recovers an unpublished child failure after its completion delivery was disposed",
2340+
() =>
2341+
Effect.gen(function* () {
2342+
const orchestrator = yield* Orchestrator.OrchestratorV2;
2343+
const eventSink = yield* EventSink.EventSinkV2;
2344+
const now = yield* DateTime.now;
2345+
const threadId = ThreadId.make("thread:unpublished-failure-parent");
2346+
const projectId = ProjectId.make("project:unpublished-failure-parent");
2347+
const runId = RunId.make("run:unpublished-failure-parent");
2348+
const rootNodeId = NodeId.make("node:unpublished-failure-parent-root");
2349+
yield* seedParentWithTerminalTask({
2350+
threadId,
2351+
projectId,
2352+
runId,
2353+
rootNodeId,
2354+
taskId: NodeId.make("node:unpublished-failure-parent-settled"),
2355+
deliveryState: "delivered",
2356+
now,
2357+
});
2358+
const child = yield* seedRestartCancelledChild({
2359+
parentThreadId: threadId,
2360+
projectId,
2361+
parentRunId: runId,
2362+
rootNodeId,
2363+
name: "unpublished-failure-child",
2364+
completionWake: "always",
2365+
continuationPending: false,
2366+
now,
2367+
});
2368+
// Skip the live terminal listener to model a committed failure whose
2369+
// publication was interrupted. task_cancel then disposed its delivery.
2370+
yield* eventSink.write({
2371+
commandId: reconcileCommandId("unpublished-failure"),
2372+
events: [
2373+
runEvent({
2374+
threadId: child.childThreadId,
2375+
runId: child.childRunId,
2376+
ordinal: 1,
2377+
status: "failed",
2378+
now,
2379+
}),
2380+
],
2381+
});
2382+
yield* orchestrator.dispatch({
2383+
type: "delegated_task.completion-delivery.dispose",
2384+
commandId: CommandId.make("command:unpublished-failure-dispose"),
2385+
parentThreadId: threadId,
2386+
taskId: child.taskId,
2387+
});
2388+
yield* orchestrator.recoverDelegatedTasks;
2389+
const recovered = yield* orchestrator.getThreadProjection(threadId);
2390+
const task = recovered.subagents.find((row) => row.id === child.taskId);
2391+
assert.equal(task?.status, "failed");
2392+
assert.equal(task?.result, "Child task ended with status failed.");
2393+
assert.equal(task?.completionDelivery?.state, "disposed");
2394+
assert.lengthOf(recovered.runs, 1);
2395+
assert.lengthOf(
2396+
recovered.contextTransfers.filter((transfer) => transfer.type === "subagent_result"),
2397+
1,
2398+
);
2399+
yield* orchestrator.recoverDelegatedTasks;
2400+
assert.deepEqual(yield* orchestrator.getThreadProjection(threadId), recovered);
2401+
}),
2402+
);
2403+
23382404
it.effect("wakes the parent for a settled_only task once its spawning turn ended", () =>
23392405
Effect.gen(function* () {
23402406
const orchestrator = yield* Orchestrator.OrchestratorV2;

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ const layerBase: Layer.Layer<
258258
transaction: Effect.Effect<A, E, R>,
259259
publish: (committed: A) => Effect.Effect<void>,
260260
) =>
261-
Effect.suspend(() => {
261+
Effect.uninterruptibleMask((restore) => {
262262
let holdsLane = false;
263263
const takeLane = publishLane.take(1).pipe(
264264
Effect.andThen(
@@ -268,8 +268,11 @@ const layerBase: Layer.Layer<
268268
),
269269
Effect.uninterruptible,
270270
);
271+
// Cancellation may roll back the transaction's work. Once it commits,
272+
// every subscriber must hear about it before this writer can exit.
273+
// Otherwise a failed child run can leave its parent's task running.
271274
return sql
272-
.withTransaction(Effect.tap(transaction, () => takeLane))
275+
.withTransaction(Effect.tap(restore(transaction), () => takeLane))
273276
.pipe(
274277
Effect.tap(publish),
275278
Effect.ensuring(

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3295,3 +3295,107 @@ it.effect("loads a shell snapshot while the writer connection is occupied", () =
32953295
}).pipe(Effect.provide(projectionLayer));
32963296
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
32973297
);
3298+
3299+
it.effect.each(["cancel transaction", "fail transaction", "cancel publication"] as const)(
3300+
"preserves event consistency and allows later writes after %s",
3301+
(scenario) =>
3302+
Effect.gen(function* () {
3303+
const paused = yield* Deferred.make<void>();
3304+
const release = yield* Deferred.make<void>();
3305+
const committed = scenario === "cancel publication";
3306+
const pause = Deferred.succeed(paused, undefined).pipe(
3307+
Effect.andThen(Deferred.await(release)),
3308+
);
3309+
const layerPausingOutbox = Layer.effect(
3310+
EffectOutbox.EffectOutboxV2,
3311+
Effect.gen(function* () {
3312+
const delegate = yield* EffectOutbox.EffectOutboxV2;
3313+
return EffectOutbox.EffectOutboxV2.of({
3314+
...delegate,
3315+
enqueue: (effects) =>
3316+
delegate
3317+
.enqueue(effects)
3318+
.pipe(
3319+
Effect.andThen(!committed && effects.length > 0 ? pause : Effect.void),
3320+
Effect.andThen(
3321+
scenario === "fail transaction" && effects.length > 0
3322+
? Effect.die("Injected transaction failure")
3323+
: Effect.void,
3324+
),
3325+
),
3326+
notifyAvailable: (count) =>
3327+
(committed ? pause : Effect.void).pipe(
3328+
Effect.andThen(delegate.notifyAvailable(count)),
3329+
),
3330+
});
3331+
}),
3332+
).pipe(Layer.provide(layerEffectOutboxProvided));
3333+
const layerEventSink = EventSink.layerFromStores.pipe(
3334+
Layer.provide(
3335+
Layer.mergeAll(
3336+
layerStoresProvided,
3337+
layerPausingOutbox,
3338+
layerCommandReceiptStoreProvided,
3339+
ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
3340+
TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
3341+
),
3342+
),
3343+
);
3344+
3345+
yield* Effect.gen(function* () {
3346+
const eventSink = yield* EventSink.EventSinkV2;
3347+
const projections = yield* ProjectionStore.ProjectionStoreV2;
3348+
const outbox = yield* EffectOutbox.EffectOutboxV2;
3349+
const now = yield* DateTime.now;
3350+
const first = makeThread(ThreadId.make("thread:interrupted-publish:first"), now);
3351+
const marker = makeThread(ThreadId.make("thread:interrupted-publish:marker"), now);
3352+
const published = yield* eventSink
3353+
.stream({ afterSequence: yield* eventSink.latestSequence() })
3354+
.pipe(
3355+
Stream.takeUntil((stored) => stored.event.threadId === marker.id),
3356+
Stream.runCollect,
3357+
Effect.forkScoped({ startImmediately: true }),
3358+
);
3359+
const writer = yield* eventSink
3360+
.writeWithEffects({
3361+
events: [
3362+
threadCreatedEvent({ id: "event:interrupted-publish:first", thread: first, now }),
3363+
],
3364+
effects: [
3365+
{
3366+
id: "effect:interrupted-publish:first",
3367+
commandId: CommandId.make("command:interrupted-publish:first"),
3368+
threadId: first.id,
3369+
request: { type: "terminal.cleanup" },
3370+
},
3371+
],
3372+
})
3373+
.pipe(Effect.forkScoped);
3374+
yield* Deferred.await(paused);
3375+
if (scenario !== "fail transaction") writer.interruptUnsafe();
3376+
if (scenario !== "cancel transaction") yield* Deferred.succeed(release, undefined);
3377+
assert.equal((yield* Fiber.await(writer))._tag, "Failure");
3378+
assert.equal(
3379+
(yield* projections.getThreadShell(first.id))?.id ?? null,
3380+
committed ? first.id : null,
3381+
);
3382+
assert.equal(
3383+
Option.isSome(yield* outbox.get("effect:interrupted-publish:first")),
3384+
committed,
3385+
);
3386+
yield* eventSink.write({
3387+
events: [
3388+
threadCreatedEvent({ id: "event:interrupted-publish:marker", thread: marker, now }),
3389+
],
3390+
});
3391+
assert.deepEqual(
3392+
Array.from(yield* Fiber.join(published), (stored) => stored.event.threadId),
3393+
committed ? [first.id, marker.id] : [marker.id],
3394+
);
3395+
}).pipe(
3396+
Effect.provide(
3397+
Layer.mergeAll(layerEventSink, layerStoresProvided, layerEffectOutboxProvided),
3398+
),
3399+
);
3400+
}).pipe(Effect.provide(layerDatabase)),
3401+
);

0 commit comments

Comments
 (0)