Skip to content

Commit 6be8a58

Browse files
fix(server): correct delegated stop receipt bounds
1 parent 6c0e5cf commit 6be8a58

2 files changed

Lines changed: 96 additions & 35 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9724,35 +9724,24 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
97249724
);
97259725
const latestRun = projection.runs.findLast((run) => run.status !== "queued");
97269726
if (command.throughSequence !== undefined && latestRun !== undefined) {
9727-
// A child Stop may have failed before committing its receipt. Retrying
9728-
// it must not reach a run that started after the parent's original Stop.
9729-
const hadStarted = yield* eventSink
9727+
// A queued run already belonged to Stop even if it started before the delegated walk.
9728+
const existed = yield* eventSink
97309729
.read({
97319730
threadId: command.threadId,
97329731
throughSequence: command.throughSequence,
97339732
eventType: "run.created",
97349733
})
97359734
.pipe(
9736-
Stream.concat(
9737-
eventSink.read({
9738-
threadId: command.threadId,
9739-
throughSequence: command.throughSequence,
9740-
eventType: "run.updated",
9741-
}),
9742-
),
97439735
Stream.runFold(
97449736
() => false,
9745-
(found, stored) =>
9746-
found ||
9747-
((stored.event.type === "run.created" || stored.event.type === "run.updated") &&
9748-
stored.event.runId === latestRun.id &&
9749-
stored.event.payload.status !== "queued"),
9737+
(found, stored) => found || stored.event.runId === latestRun.id,
97509738
),
97519739
Effect.mapError(
97529740
(cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
97539741
),
97549742
);
9755-
if (!hadStarted) return undefined;
9743+
if (!existed) return undefined;
9744+
// Partial retries keep queue/watch changes thread-wide while the original run is current.
97569745
}
97579746
const target =
97589747
projection.runs.findLast(
@@ -11420,7 +11409,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1142011409
// A settle that ended nothing still records its receipt: a replayed Stop
1142111410
// effect then finds it instead of settling work that appeared since.
1142211411
const resultSequence = yield* Effect.gen(function* () {
11423-
const sequence = yield* eventSink.latestSequence({ threadId: commandThreadId(command) });
11412+
// Internal Stop receipts also bound other threads; client commands keep their thread cursor.
11413+
const sequence = yield* eventSink.latestSequence(
11414+
command.type === "thread.stop" ? undefined : { threadId: commandThreadId(command) },
11415+
);
1142411416
yield* commandReceipts.insertIfAbsent({
1142511417
commandId: command.commandId,
1142611418
threadId: commandThreadId(command),

‎apps/server/src/orchestration-v2/ThreadStop.test.ts‎

Lines changed: 87 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import * as Schema from "effect/Schema";
2929
import * as SqlClient from "effect/sql/SqlClient";
3030
import * as Stream from "effect/Stream";
3131
import * as EventStore from "./EventStore.ts";
32+
import * as EventSink from "./EventSink.ts";
3233
import * as TestClock from "effect/testing/TestClock";
3334

3435
import * as SqlitePersistence from "../persistence/Sqlite.ts";
@@ -456,7 +457,7 @@ it.effect.each([
456457
it.effect("Stop ends watches, holds queues, and stops the delegated tasks under the thread", () =>
457458
Effect.gen(function* () {
458459
const orchestrator = yield* Orchestrator.OrchestratorV2;
459-
const projections = yield* ProjectionStore.ProjectionStoreV2;
460+
const sink = yield* EventSink.EventSinkV2;
460461
const threads = yield* ThreadManagementService.ThreadManagementService;
461462
const sql = yield* SqlClient.SqlClient;
462463
const parentThreadId = ThreadId.make("thread:stop-parent");
@@ -466,16 +467,21 @@ it.effect("Stop ends watches, holds queues, and stops the delegated tasks under
466467
assert.isTrue(Exit.isFailure(yield* Effect.exit(watch(childThreadId, 2))));
467468
const grandchildThreadId = yield* delegate(childThreadId, "grandchild task");
468469
yield* send(childThreadId, "child follow-up", "queue_after_active");
470+
yield* send(childThreadId, "child queued after follow-up", "queue_after_active");
469471

470472
const now = yield* DateTime.now;
471473
const firstRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!;
472-
yield* projections.apply({
473-
id: EventId.make("event:stop-parent:first-run-completed"),
474-
type: "run.updated",
475-
threadId: parentThreadId,
476-
runId: firstRun.id,
477-
occurredAt: now,
478-
payload: { ...firstRun, status: "completed", completedAt: now },
474+
yield* sink.write({
475+
events: [
476+
{
477+
id: EventId.make("event:stop-parent:first-run-completed"),
478+
type: "run.updated",
479+
threadId: parentThreadId,
480+
runId: firstRun.id,
481+
occurredAt: now,
482+
payload: { ...firstRun, status: "completed", completedAt: now },
483+
},
484+
],
479485
});
480486
yield* send(parentThreadId, "second", "start_immediately");
481487
const secondRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.at(-1)!;
@@ -503,15 +509,39 @@ it.effect("Stop ends watches, holds queues, and stops the delegated tasks under
503509
"delegated-tasks.stop",
504510
);
505511

512+
const childRun = (yield* orchestrator.getThreadProjection(childThreadId)).runs[0]!;
513+
yield* sink.write({
514+
events: [
515+
{
516+
id: EventId.make("event:stop-child:first-run-completed"),
517+
type: "run.updated",
518+
threadId: childThreadId,
519+
runId: childRun.id,
520+
occurredAt: now,
521+
payload: { ...childRun, status: "completed", completedAt: now },
522+
},
523+
],
524+
});
525+
yield* orchestrator.dispatch({
526+
type: "queue.resume",
527+
commandId: CommandId.make("resume-child-before-stop-walk"),
528+
threadId: childThreadId,
529+
});
530+
assert.deepEqual((yield* threadState(childThreadId)).runs, ["completed", "starting", "queued"]);
531+
506532
yield* threads.stopDelegatedTasks({ threadId: parentThreadId, commandId: stopCommandId });
507533
assert.deepEqual(yield* threadState(childThreadId), {
508-
runs: ["interrupted", "queued:held"],
534+
runs: ["completed", "interrupted", "queued:held"],
509535
watched: [],
510536
});
511537
assert.deepEqual((yield* threadState(grandchildThreadId)).runs, ["interrupted"]);
512538

513539
yield* threads.stopDelegatedTasks({ threadId: parentThreadId, commandId: stopCommandId });
514-
assert.deepEqual((yield* threadState(childThreadId)).runs, ["interrupted", "queued:held"]);
540+
assert.deepEqual((yield* threadState(childThreadId)).runs, [
541+
"completed",
542+
"interrupted",
543+
"queued:held",
544+
]);
515545

516546
yield* send(parentThreadId, "new work", "start_immediately");
517547
const newChild = yield* delegate(parentThreadId, "new task after Stop");
@@ -1408,13 +1438,25 @@ it.effect("an MCP Stop replays its original result after the database and servic
14081438
const idleProjection = yield* (yield* Orchestrator.OrchestratorV2).getThreadProjection(idle);
14091439
const idleRun = idleProjection.runs[0]!;
14101440
const now = yield* DateTime.now;
1411-
yield* (yield* ProjectionStore.ProjectionStoreV2).apply({
1412-
id: EventId.make("event:mcp-stop-idle:completed"),
1413-
type: "run.updated",
1414-
threadId: idle,
1415-
runId: idleRun.id,
1416-
occurredAt: now,
1417-
payload: { ...idleRun, status: "completed", completedAt: now },
1441+
yield* (yield* EventSink.EventSinkV2).write({
1442+
events: [
1443+
{
1444+
id: EventId.make("event:mcp-stop-idle:completed"),
1445+
type: "run.updated",
1446+
threadId: idle,
1447+
runId: idleRun.id,
1448+
occurredAt: now,
1449+
payload: { ...idleRun, status: "completed", completedAt: now },
1450+
},
1451+
{
1452+
id: EventId.make("event:mcp-stop-idle:held"),
1453+
type: "run.updated",
1454+
threadId: idle,
1455+
runId: idleProjection.runs[1]!.id,
1456+
occurredAt: now,
1457+
payload: { ...idleProjection.runs[1]!, queueHeld: true },
1458+
},
1459+
],
14181460
});
14191461
assert.deepEqual(
14201462
yield* mcp.interruptThread(scope, {
@@ -1438,7 +1480,27 @@ it.effect("an MCP Stop replays its original result after the database and servic
14381480
yield* delegate(childThreadId, "original grandchild");
14391481
const stopped = yield* mcp.interruptThread(scope, stop);
14401482
assert.equal(stopped.runId, sent.runId);
1441-
return { sent, stopped };
1483+
yield* send(childThreadId, "child work after parent stopped", "start_immediately");
1484+
const store = yield* EventStore.EventStoreV2;
1485+
const childSequence = yield* store.latestSequence({ threadId: childThreadId });
1486+
const noopStop = { ...stop, clientRequestId: "already-stopped-parent" };
1487+
const noopStopped = yield* mcp.interruptThread(scope, noopStop);
1488+
assert.deepEqual(noopStopped, { threadId, runId: null, status: "no_active_run" });
1489+
assert.deepEqual((yield* threadState(childThreadId)).runs, ["interrupted", "interrupted"]);
1490+
const sql = yield* SqlClient.SqlClient;
1491+
const [noopReceipt] = yield* sql<{ command_id: string; result_sequence: number }>`
1492+
SELECT command_id, result_sequence FROM orchestration_command_receipts
1493+
WHERE aggregate_id = ${threadId} AND command_type = 'thread.stop'
1494+
ORDER BY result_sequence DESC LIMIT 1
1495+
`;
1496+
assert.isAtLeast(noopReceipt!.result_sequence, childSequence);
1497+
assert.deepEqual(
1498+
yield* store
1499+
.readByCommandId({ commandId: CommandId.make(noopReceipt!.command_id) })
1500+
.pipe(Stream.runCollect),
1501+
[],
1502+
);
1503+
return { sent, stopped, childThreadId, noopStop, noopStopped };
14421504
}).pipe(Effect.provide(layerMcp(database).pipe(Layer.fresh)));
14431505

14441506
yield* Effect.gen(function* () {
@@ -1453,6 +1515,13 @@ it.effect("an MCP Stop replays its original result after the database and servic
14531515
assert.equal((yield* orchestrator.getThreadProjection(threadId)).runs.length, 1);
14541516
yield* send(threadId, "after restart", "start_immediately");
14551517
const newChild = yield* delegate(threadId, "child after restart");
1518+
yield* send(first.childThreadId, "child work after restart", "start_immediately");
1519+
assert.deepEqual(yield* mcp.interruptThread(scope, first.noopStop), first.noopStopped);
1520+
assert.deepEqual((yield* threadState(first.childThreadId)).runs, [
1521+
"interrupted",
1522+
"interrupted",
1523+
"starting",
1524+
]);
14561525
assert.deepEqual(yield* mcp.interruptThread(scope, stop), first.stopped);
14571526
assert.deepEqual((yield* threadState(threadId)).runs, ["interrupted", "starting"]);
14581527
assert.deepEqual((yield* threadState(newChild)).runs, ["starting"]);

0 commit comments

Comments
 (0)