Skip to content

Commit 6ff0395

Browse files
fix(orchestration): protect deferred organization lifecycle
1 parent 04bbc33 commit 6ff0395

5 files changed

Lines changed: 246 additions & 13 deletions

File tree

‎apps/server/src/orchestration-v2/DeferredOrganizationRecovery.integration.test.ts‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
22
import { assert, it } from "@effect/vitest";
33
import {
44
CommandId,
5+
EventId,
56
MessageId,
67
type ModelSelection,
78
ProjectId,
@@ -10,19 +11,23 @@ import {
1011
ThreadId,
1112
} from "@t3tools/contracts";
1213
import * as Cause from "effect/Cause";
14+
import * as DateTime from "effect/DateTime";
1315
import * as Effect from "effect/Effect";
1416
import * as Exit from "effect/Exit";
1517
import * as FileSystem from "effect/FileSystem";
18+
import * as Fiber from "effect/Fiber";
1619
import * as Layer from "effect/Layer";
1720
import * as Option from "effect/Option";
1821
import * as Path from "effect/Path";
1922
import * as Ref from "effect/Ref";
2023
import * as SqlClient from "effect/unstable/sql/SqlClient";
24+
import * as Stream from "effect/Stream";
2125

2226
import { makeSqlitePersistenceLive } from "../persistence/Layers/Sqlite.ts";
2327
import { runV2RecoveryPhase } from "../serverRuntimeStartup.ts";
2428
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
2529
import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
30+
import { EventSinkV2 } from "./EventSink.ts";
2631
import * as Orchestrator from "./Orchestrator.ts";
2732
import { ProviderRuntimeRecoveryService } from "./ProviderRuntimeRecoveryService.ts";
2833
import type { ProviderAdapterV2Shape } from "./ProviderAdapter.ts";
@@ -211,6 +216,188 @@ it.effect("retries a transient deferred apply read before recording its receipt"
211216
),
212217
);
213218

219+
it.effect("promotes queued work when terminal deferred apply fails", () =>
220+
Effect.scoped(
221+
Effect.gen(function* () {
222+
const fs = yield* FileSystem.FileSystem;
223+
const path = yield* Path.Path;
224+
const tempDir = yield* Effect.acquireRelease(
225+
fs.makeTempDirectory({ prefix: "t3-deferred-organization-terminal-" }),
226+
(directory) => fs.remove(directory, { recursive: true, force: true }).pipe(Effect.orDie),
227+
);
228+
const databaseLayer = makeSqlitePersistenceLive(path.join(tempDir, "state.sqlite")).pipe(
229+
Layer.provide(NodeServices.layer),
230+
);
231+
const registryLayer = makeProviderAdapterRegistryLayer([adapter]);
232+
const threadId = ThreadId.make("thread:deferred-organization-terminal");
233+
234+
const seeded = yield* Effect.scoped(
235+
Effect.gen(function* () {
236+
const orchestrator = yield* Orchestrator.OrchestratorV2;
237+
yield* orchestrator.dispatch({
238+
type: "thread.create",
239+
createdBy: "user",
240+
creationSource: "web",
241+
commandId: CommandId.make("command:deferred-organization-terminal:create"),
242+
threadId,
243+
projectId: ProjectId.make("project:deferred-organization-terminal"),
244+
title: "Deferred organization terminal failure",
245+
modelSelection,
246+
runtimeMode: "full-access",
247+
interactionMode: "default",
248+
branch: null,
249+
worktreePath: tempDir,
250+
});
251+
yield* orchestrator.dispatch({
252+
type: "message.dispatch",
253+
createdBy: "user",
254+
creationSource: "web",
255+
commandId: CommandId.make("command:deferred-organization-terminal:active"),
256+
threadId,
257+
messageId: MessageId.make("message:deferred-organization-terminal:active"),
258+
text: "Complete this run before applying organization.",
259+
attachments: [],
260+
modelSelection,
261+
dispatchMode: { type: "start_immediately" },
262+
});
263+
const activeRun = (yield* orchestrator.getThreadProjection(threadId)).runs[0];
264+
assert.isDefined(activeRun);
265+
yield* orchestrator.dispatch({
266+
type: "thread.organization.defer",
267+
commandId: CommandId.make("command:deferred-organization-terminal:schedule"),
268+
threadId,
269+
runId: activeRun.id,
270+
action: "settle",
271+
});
272+
yield* orchestrator.dispatch({
273+
type: "message.dispatch",
274+
createdBy: "user",
275+
creationSource: "web",
276+
commandId: CommandId.make("command:deferred-organization-terminal:queued"),
277+
threadId,
278+
messageId: MessageId.make("message:deferred-organization-terminal:queued"),
279+
text: "Promote this run after the active run ends.",
280+
attachments: [],
281+
modelSelection,
282+
dispatchMode: { type: "queue_after_active" },
283+
});
284+
const projection = yield* orchestrator.getThreadProjection(threadId);
285+
const queuedRun = projection.runs.find((run) => run.status === "queued");
286+
assert.isDefined(queuedRun);
287+
return { activeRun, queuedRun };
288+
}).pipe(
289+
Effect.provide(
290+
makeOrchestratorV2ReplayLayerWithRegistry(
291+
{
292+
name: "deferred-organization-terminal:first-runtime",
293+
runtimePolicyOverride: { cwd: tempDir },
294+
},
295+
registryLayer,
296+
{ databaseLayer, runEffectWorker: false },
297+
),
298+
),
299+
),
300+
);
301+
302+
const armed = yield* Ref.make(false);
303+
const projectionReads = yield* Ref.make(0);
304+
const decorateProjectionStore = (store: ProjectionStore.ProjectionStoreV2["Service"]) =>
305+
ProjectionStore.ProjectionStoreV2.of({
306+
...store,
307+
getThreadProjection: (requestedThreadId) =>
308+
Ref.get(armed).pipe(
309+
Effect.flatMap((isArmed) =>
310+
isArmed
311+
? Ref.updateAndGet(projectionReads, (count) => count + 1).pipe(
312+
Effect.flatMap((count) =>
313+
count === 3
314+
? Effect.fail(
315+
new ProjectionStore.ProjectionStoreReadError({
316+
threadId: requestedThreadId,
317+
cause: "simulated terminal deferred apply failure",
318+
}),
319+
)
320+
: store.getThreadProjection(requestedThreadId),
321+
),
322+
)
323+
: store.getThreadProjection(requestedThreadId),
324+
),
325+
),
326+
});
327+
328+
yield* Effect.scoped(
329+
Effect.gen(function* () {
330+
const orchestrator = yield* Orchestrator.OrchestratorV2;
331+
const eventSink = yield* EventSinkV2;
332+
const receipts = yield* CommandReceiptStoreV2;
333+
const afterSequence = yield* eventSink.latestSequence({ threadId });
334+
const promoted = yield* eventSink.stream({ threadId, afterSequence }).pipe(
335+
Stream.filter(
336+
(stored) =>
337+
stored.event.type === "run.updated" &&
338+
stored.event.payload.id === seeded.queuedRun.id &&
339+
stored.event.payload.status === "starting",
340+
),
341+
Stream.runHead,
342+
Effect.forkChild,
343+
);
344+
yield* Ref.set(armed, true);
345+
const completedAt = yield* DateTime.now;
346+
yield* eventSink.write({
347+
events: [
348+
{
349+
id: EventId.make("event:deferred-organization-terminal:completed"),
350+
type: "run.updated",
351+
threadId,
352+
runId: seeded.activeRun.id,
353+
...(seeded.activeRun.rootNodeId === null
354+
? {}
355+
: { nodeId: seeded.activeRun.rootNodeId }),
356+
providerInstanceId: seeded.activeRun.providerInstanceId,
357+
occurredAt: completedAt,
358+
payload: {
359+
...seeded.activeRun,
360+
status: "completed",
361+
completedAt,
362+
},
363+
},
364+
],
365+
});
366+
assert.isTrue(Option.isSome(yield* Fiber.join(promoted)));
367+
yield* Ref.set(armed, false);
368+
369+
const projection = yield* orchestrator.getThreadProjection(threadId);
370+
assert.equal(
371+
projection.runs.find((run) => run.id === seeded.queuedRun.id)?.status,
372+
"starting",
373+
);
374+
assert.equal(yield* Ref.get(projectionReads), 4);
375+
assert.isTrue(
376+
Option.isNone(
377+
yield* receipts.getByCommandId(
378+
CommandId.make(
379+
`command:system:thread-organization-defer:${threadId}:${seeded.activeRun.id}`,
380+
),
381+
),
382+
),
383+
);
384+
}).pipe(
385+
Effect.provide(
386+
makeOrchestratorV2ReplayLayerWithRegistry(
387+
{
388+
name: "deferred-organization-terminal:second-runtime",
389+
runtimePolicyOverride: { cwd: tempDir },
390+
},
391+
registryLayer,
392+
{ databaseLayer, runEffectWorker: false, decorateProjectionStore },
393+
),
394+
),
395+
),
396+
);
397+
}).pipe(Effect.provide(NodeServices.layer)),
398+
),
399+
);
400+
214401
it.effect("discards a stale deferred organization intent after runtime restart", () =>
215402
Effect.scoped(
216403
Effect.gen(function* () {

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7415,23 +7415,31 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
74157415
yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(threadId));
74167416
}
74177417
if (stored.event.type === "run.updated") {
7418+
const runId = stored.event.payload.id;
74187419
yield* threadDispatch.withLock(
74197420
threadId,
7420-
finalizeDelegatedCompletionDelivery(threadId, stored.event.payload.id),
7421+
finalizeDelegatedCompletionDelivery(threadId, runId),
74217422
);
7422-
yield* threadDispatch.withLock(
7423-
threadId,
7424-
applyDeferredOrganization(threadId, stored.event.payload.id),
7423+
yield* threadDispatch.withLock(threadId, applyDeferredOrganization(threadId, runId)).pipe(
7424+
Effect.catch((cause) =>
7425+
Effect.logWarning("Failed to apply deferred thread organization", {
7426+
threadId,
7427+
runId,
7428+
cause,
7429+
}),
7430+
),
74257431
);
74267432
}
74277433
yield* threadDispatch.withLock(threadId, startNextQueuedRun(threadId));
74287434
}).pipe(
74297435
Effect.catchCause((cause) =>
7430-
Effect.logWarning("Failed to react to terminal V2 run", {
7431-
threadId: stored.event.threadId,
7432-
sequence: stored.sequence,
7433-
cause,
7434-
}),
7436+
Cause.hasInterruptsOnly(cause)
7437+
? Effect.failCause(cause)
7438+
: Effect.logWarning("Failed to react to terminal V2 run", {
7439+
threadId: stored.event.threadId,
7440+
sequence: stored.sequence,
7441+
cause,
7442+
}),
74357443
),
74367444
);
74377445

‎apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import {
3232
layer as effectWorkerLayer,
3333
runDaemon as runEffectWorkerDaemon,
3434
} from "../EffectWorker.ts";
35-
import { layerFromStores as eventSinkLayer } from "../EventSink.ts";
35+
import { EventSinkV2, layerFromStores as eventSinkLayer } from "../EventSink.ts";
3636
import { layer as eventStoreLayer } from "../EventStore.ts";
3737
import { layer as idAllocatorLayer } from "../IdAllocator.ts";
3838
import { layer as orchestratorLayer } from "../Orchestrator.ts";
@@ -232,7 +232,7 @@ export function makeOrchestratorV2ProviderReplayLayer<
232232
readonly replayGate?: ProviderReplayGate;
233233
} = {},
234234
): Layer.Layer<
235-
OrchestratorV2 | ProviderRuntimeRecoveryService | CommandReceiptStoreV2,
235+
OrchestratorV2 | ProviderRuntimeRecoveryService | CommandReceiptStoreV2 | EventSinkV2,
236236
Error | MigrationError | PlatformError.PlatformError | SqlError
237237
> {
238238
const registryLayer = harness.makeProviderAdapterRegistryLayer(
@@ -257,7 +257,7 @@ export function makeOrchestratorV2ReplayLayerWithRegistry<Error>(
257257
) => ProjectionStoreV2["Service"];
258258
} = {},
259259
): Layer.Layer<
260-
OrchestratorV2 | ProviderRuntimeRecoveryService | CommandReceiptStoreV2,
260+
OrchestratorV2 | ProviderRuntimeRecoveryService | CommandReceiptStoreV2 | EventSinkV2,
261261
Error | MigrationError | PlatformError.PlatformError | SqlError
262262
> {
263263
const serverConfigLayer = Layer.effect(
@@ -443,6 +443,7 @@ export function makeOrchestratorV2ReplayLayerWithRegistry<Error>(
443443
orchestratorProvided,
444444
providerRuntimeRecoveryProvided,
445445
commandReceiptStoreProvided,
446+
eventSinkProvided,
446447
).pipe(Layer.provide(worktreeRepairDependenciesTestLayer), Layer.provide(NodeServices.layer));
447448
}
448449
const orchestratorWithWorker = Layer.effect(
@@ -457,5 +458,6 @@ export function makeOrchestratorV2ReplayLayerWithRegistry<Error>(
457458
orchestratorWithWorker,
458459
providerRuntimeRecoveryProvided,
459460
commandReceiptStoreProvided,
461+
eventSinkProvided,
460462
).pipe(Layer.provide(worktreeRepairDependenciesTestLayer), Layer.provide(NodeServices.layer));
461463
}

‎packages/contracts/src/orchestrationV2.test.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
OrchestrationV2DomainEvent,
2727
OrchestrationV2ProviderThread,
2828
OrchestrationV2ProviderThreadJson,
29+
OrchestrationV2RpcSchemas,
2930
OrchestrationV2ShellSnapshot,
3031
OrchestrationV2Subagent,
3132
OrchestrationV2ThreadProjection,
@@ -43,6 +44,9 @@ const LegacyShellStreamItem = Schema.Union([
4344
]);
4445
const decodeLegacyShellStreamItem = Schema.decodeUnknownSync(LegacyShellStreamItem);
4546
const decodeOrchestrationV2Command = Schema.decodeUnknownSync(OrchestrationV2Command);
47+
const decodeOrchestrationV2ClientCommand = Schema.decodeUnknownSync(
48+
OrchestrationV2RpcSchemas.dispatchCommand.input,
49+
);
4650
const decodeOrchestrationV2TurnItem = Schema.decodeUnknownSync(OrchestrationV2TurnItem);
4751
const decodeOrchestrationV2CheckpointScope = Schema.decodeUnknownSync(
4852
OrchestrationV2CheckpointScope,
@@ -61,6 +65,25 @@ const decodeOrchestrationV2ProviderThread = Schema.decodeUnknownSync(Orchestrati
6165
const decodeOrchestrationV2ThreadShell = Schema.decodeUnknownSync(OrchestrationV2ThreadShell);
6266

6367
describe("orchestration V2 contracts", () => {
68+
it("keeps deferred organization apply internal to the server", () => {
69+
const applyCommand = {
70+
type: "thread.organization.defer.apply",
71+
commandId: "command:deferred-organization-apply",
72+
threadId: "thread:deferred-organization-apply",
73+
runId: "run:deferred-organization-apply",
74+
};
75+
76+
expect(decodeOrchestrationV2Command(applyCommand).type).toBe("thread.organization.defer.apply");
77+
expect(() => decodeOrchestrationV2ClientCommand(applyCommand)).toThrow();
78+
expect(
79+
decodeOrchestrationV2ClientCommand({
80+
type: "thread.organization.defer.cancel",
81+
commandId: "command:deferred-organization-cancel",
82+
threadId: "thread:deferred-organization-apply",
83+
}).type,
84+
).toBe("thread.organization.defer.cancel");
85+
});
86+
6487
it("lets legacy snapshot decoders ignore enrichment metadata", () => {
6588
const decoded = decodeLegacyShellStreamItem({
6689
kind: "snapshot",

‎packages/contracts/src/orchestrationV2.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2393,6 +2393,19 @@ export const OrchestrationV2Command = Schema.Union([
23932393
]);
23942394
export type OrchestrationV2Command = typeof OrchestrationV2Command.Type;
23952395

2396+
export type OrchestrationV2ClientCommand = Exclude<
2397+
OrchestrationV2Command,
2398+
{ readonly type: "thread.organization.defer.apply" }
2399+
>;
2400+
2401+
export const OrchestrationV2ClientCommand = OrchestrationV2Command.pipe(
2402+
Schema.refine(
2403+
(command): command is OrchestrationV2ClientCommand =>
2404+
command.type !== "thread.organization.defer.apply",
2405+
{ expected: "a client-dispatchable orchestration command" },
2406+
),
2407+
);
2408+
23962409
export const ORCHESTRATION_V2_WS_METHODS = {
23972410
dispatchCommand: "orchestration.dispatchCommand",
23982411
getTurnDiff: "orchestration.getTurnDiff",
@@ -2687,7 +2700,7 @@ export class OrchestrationGetWorkflowScriptError extends Schema.TaggedErrorClass
26872700

26882701
export const OrchestrationV2RpcSchemas = {
26892702
dispatchCommand: {
2690-
input: OrchestrationV2Command,
2703+
input: OrchestrationV2ClientCommand,
26912704
output: OrchestrationV2DispatchCommandResult,
26922705
},
26932706
getTurnDiff: {

0 commit comments

Comments
 (0)