Skip to content

Commit 4544929

Browse files
authored
Merge branch 'main' into fix/desktop-narrow-window-tiles
2 parents 9ced079 + c222511 commit 4544929

5 files changed

Lines changed: 322 additions & 106 deletions

File tree

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -563,6 +563,159 @@ it.effect.each(
563563
},
564564
);
565565

566+
it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {
567+
let commits = 0;
568+
const tracer = Tracer.make({
569+
span(options) {
570+
const span = new Tracer.NativeSpan(options);
571+
const end = span.end.bind(span);
572+
span.end = (time, exit) => {
573+
end(time, exit);
574+
if (span.name === "COMMIT") commits++;
575+
};
576+
return span;
577+
},
578+
});
579+
return Effect.gen(function* () {
580+
const sink = yield* EventSink.EventSinkV2;
581+
const events = yield* EventStore.EventStoreV2;
582+
const projections = yield* ProjectionStore.ProjectionStoreV2;
583+
const sql = yield* SqlClient.SqlClient;
584+
const outbox = yield* EffectOutbox.EffectOutboxV2;
585+
const now = yield* DateTime.now;
586+
const threads = Array.from({ length: 65 }, (_, i) =>
587+
ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),
588+
);
589+
const sharedSession = {
590+
id: ProviderSessionId.make("shutdown-shared"),
591+
driver: providerDriver,
592+
providerInstanceId,
593+
status: "ready" as const,
594+
cwd: process.cwd(),
595+
model: modelSelection.model,
596+
capabilities: CodexProviderCapabilitiesV2,
597+
createdAt: now,
598+
updatedAt: now,
599+
lastError: null,
600+
};
601+
yield* sink.write({
602+
events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [
603+
threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),
604+
...(i < 2
605+
? [
606+
{
607+
id: EventId.make(`shutdown-session:${i}`),
608+
type: "provider-session.attached" as const,
609+
threadId,
610+
occurredAt: now,
611+
payload: sharedSession,
612+
},
613+
]
614+
: []),
615+
{
616+
id: EventId.make(`shutdown-provider:${i}`),
617+
type: "provider-thread.updated",
618+
threadId,
619+
occurredAt: now,
620+
payload: {
621+
id: ProviderThreadId.make(`shutdown-provider:${i}`),
622+
driver: providerDriver,
623+
providerInstanceId,
624+
providerSessionId: i < 2 ? sharedSession.id : null,
625+
appThreadId: threadId,
626+
ownerNodeId: null,
627+
nativeThreadRef: null,
628+
nativeConversationHeadRef: null,
629+
status: i === 1 || i === 32 ? "idle" : "active",
630+
firstRunOrdinal: null,
631+
lastRunOrdinal: null,
632+
handoffIds: [],
633+
forkedFrom: null,
634+
createdAt: now,
635+
updatedAt: now,
636+
},
637+
},
638+
]),
639+
});
640+
yield* outbox.enqueue(
641+
threads.map((threadId, i) => ({
642+
id: `shutdown-effect:${i}`,
643+
commandId: CommandId.make(`shutdown-command:${i}`),
644+
threadId,
645+
request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },
646+
})),
647+
);
648+
let eventlessNotifications = 0;
649+
let armed = fail;
650+
const recovery = yield* ProviderRuntimeRecovery.make.pipe(
651+
Effect.provide(ServerSettings.layerTest()),
652+
Effect.provideService(EffectOutbox.EffectOutboxV2, {
653+
...outbox,
654+
signalCancellations: (ids) =>
655+
Effect.gen(function* () {
656+
assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));
657+
eventlessNotifications += ids.length;
658+
yield* outbox.signalCancellations(ids);
659+
}),
660+
}),
661+
Effect.provideService(
662+
EffectWorker.OrchestrationEffectWorkerV2,
663+
EffectWorker.OrchestrationEffectWorkerV2.of({
664+
awaitWork: Effect.void,
665+
runRecoveryOnce: Effect.succeed(false),
666+
runOnce: Effect.succeed(false),
667+
nextClaimableAt: Effect.succeedNone,
668+
drain: () => Effect.succeed(0),
669+
}),
670+
),
671+
Effect.provideService(EventSink.EventSinkV2, {
672+
...sink,
673+
commitCommand: (input) =>
674+
armed && input.threadId === threads[33]
675+
? Effect.die("injected shutdown failure")
676+
: sink.commitCommand(input),
677+
}),
678+
);
679+
commits = 0;
680+
const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);
681+
assert.equal(Exit.isFailure(result), fail);
682+
assert.equal(commits, fail ? 1 : 3);
683+
const receipts =
684+
yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;
685+
assert.deepEqual(
686+
receipts.map((r) => r.aggregate_id),
687+
threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),
688+
);
689+
assert.deepEqual(
690+
yield* projections.getRecoveryThreadIds("runtime"),
691+
fail ? threads.slice(32) : [],
692+
);
693+
assert.equal(eventlessNotifications, fail ? 1 : 2);
694+
for (const [i] of threads.entries()) {
695+
const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));
696+
assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");
697+
}
698+
assert.equal(
699+
(yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(
700+
(e) =>
701+
e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,
702+
).length,
703+
1,
704+
);
705+
armed = false;
706+
yield* recovery.reconcile("shutdown");
707+
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
708+
const highWater = yield* sink.latestSequence();
709+
yield* recovery.reconcile("shutdown");
710+
assert.equal(yield* sink.latestSequence(), highWater);
711+
}).pipe(
712+
Effect.provide(Layer.fresh(layerTest)),
713+
Effect.withTracer(tracer),
714+
Effect.withTracerEnabled(true),
715+
Effect.provideService(Statement.SpanPropagationEnabled, true),
716+
);
717+
});
718+
566719
it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
567720
it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
568721
Effect.scoped(

‎apps/server/src/orchestration-v2/ProjectionRecovery.test.ts‎

Lines changed: 82 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -213,55 +213,87 @@ it.effect.each(["startup", "shutdown"] as const)(
213213
}).pipe(Effect.provide(layerRecoveryTest)),
214214
);
215215

216-
it.effect("stops a session its thread detached, which kept its live status", () =>
217-
Effect.gen(function* () {
218-
const projections = yield* ProjectionStore.ProjectionStoreV2;
219-
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
220-
const now = yield* DateTime.now;
221-
const threadId = yield* createThread("detached-session");
222-
const providerSessionId = ProviderSessionId.make("session:recovery:detached");
223-
yield* projections.apply({
224-
id: EventId.make("event:recovery:detached-session:attached"),
225-
type: "provider-session.attached",
226-
threadId,
227-
driver,
228-
providerInstanceId,
229-
occurredAt: now,
230-
payload: {
231-
id: providerSessionId,
216+
it.effect.each(["startup", "shutdown"] as const)(
217+
"stops a detached session and its orphaned active provider thread on %s",
218+
(trigger) =>
219+
Effect.gen(function* () {
220+
const projections = yield* ProjectionStore.ProjectionStoreV2;
221+
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
222+
const now = yield* DateTime.now;
223+
const threadId = yield* createThread("detached-session");
224+
const providerSessionId = ProviderSessionId.make("session:recovery:detached");
225+
yield* projections.apply({
226+
id: EventId.make("event:recovery:detached-session:attached"),
227+
type: "provider-session.attached",
228+
threadId,
232229
driver,
233230
providerInstanceId,
234-
status: "ready",
235-
cwd: "/workspace",
236-
model: modelSelection.model,
237-
capabilities: CodexProviderCapabilitiesV2,
238-
createdAt: now,
239-
updatedAt: now,
240-
lastError: null,
241-
},
242-
});
243-
yield* projections.apply({
244-
id: EventId.make("event:recovery:detached-session:detached"),
245-
type: "provider-session.detached",
246-
threadId,
247-
driver,
248-
providerInstanceId,
249-
occurredAt: now,
250-
payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },
251-
});
252-
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), [threadId]);
231+
occurredAt: now,
232+
payload: {
233+
id: providerSessionId,
234+
driver,
235+
providerInstanceId,
236+
status: "ready",
237+
cwd: "/workspace",
238+
model: modelSelection.model,
239+
capabilities: CodexProviderCapabilitiesV2,
240+
createdAt: now,
241+
updatedAt: now,
242+
lastError: null,
243+
},
244+
});
245+
yield* projections.apply({
246+
id: EventId.make("event:recovery:detached-session:detached"),
247+
type: "provider-session.detached",
248+
threadId,
249+
driver,
250+
providerInstanceId,
251+
occurredAt: now,
252+
payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },
253+
});
254+
const providerThreadId = ProviderThreadId.make("provider-thread:recovery:detached");
255+
yield* projections.apply({
256+
id: EventId.make("event:recovery:detached-provider-thread"),
257+
type: "provider-thread.updated",
258+
threadId,
259+
occurredAt: now,
260+
payload: {
261+
id: providerThreadId,
262+
appThreadId: threadId,
263+
ownerNodeId: null,
264+
driver,
265+
providerInstanceId,
266+
providerSessionId: null,
267+
nativeThreadRef: null,
268+
nativeConversationHeadRef: null,
269+
status: "active",
270+
firstRunOrdinal: null,
271+
lastRunOrdinal: null,
272+
handoffIds: [],
273+
forkedFrom: null,
274+
createdAt: now,
275+
updatedAt: now,
276+
},
277+
});
278+
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), [threadId]);
253279

254-
const summary = yield* recovery.reconcile("startup");
280+
const summary = yield* recovery.reconcile(trigger);
255281

256-
assert.equal(summary.stoppedSessions, 1);
257-
const sql = yield* SqlClient.SqlClient;
258-
const rows = yield* sql<{ readonly status: string }>`
282+
assert.equal(summary.stoppedSessions, 1);
283+
const sql = yield* SqlClient.SqlClient;
284+
const rows = yield* sql<{ readonly status: string }>`
259285
SELECT status FROM orchestration_v2_projection_provider_sessions
260286
WHERE provider_session_id = ${providerSessionId}
261287
`;
262-
assert.deepEqual(rows, [{ status: "stopped" }]);
263-
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
264-
}).pipe(Effect.provide(layerRecoveryTest)),
288+
assert.deepEqual(rows, [{ status: "stopped" }]);
289+
const projection = yield* projections.getThreadProjection(threadId);
290+
assert.equal(projection.providerThreads[0]?.status, "idle");
291+
const eventStore = yield* EventStore.EventStoreV2;
292+
const sequence = yield* eventStore.latestSequence();
293+
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
294+
yield* recovery.reconcile(trigger);
295+
assert.equal(yield* eventStore.latestSequence(), sequence);
296+
}).pipe(Effect.provide(layerRecoveryTest)),
265297
);
266298

267299
it.effect.each(["attached", "detached", "replaced"] as const)(
@@ -964,11 +996,18 @@ it.effect("bounds streaming recovery reads by candidate runs and preserves messa
964996
assert.equal(statements.length, 2);
965997
for (const [query, params] of statements) {
966998
const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
967-
assert.isTrue(plan[0]?.detail.includes("SEARCH run "), JSON.stringify(plan));
999+
assert.isTrue(
1000+
plan.some((row) => row.detail.includes("SEARCH message_run ")),
1001+
JSON.stringify(plan),
1002+
);
9681003
assert.isTrue(
9691004
plan.some((row) => row.detail.includes("messages_run_idx")),
9701005
JSON.stringify(plan),
9711006
);
1007+
assert.isFalse(
1008+
plan.some((row) => /\bSCAN message\b/.test(row.detail)),
1009+
JSON.stringify(plan),
1010+
);
9721011
assert.isFalse(
9731012
plan.some((row) => row.detail.includes("messages_thread_created_idx")),
9741013
JSON.stringify(plan),

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8439,6 +8439,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
84398439
.pipe(Effect.withTracer(tracer))).runs.map((run) => run.id),
84408440
[runId, newerRunId],
84418441
);
8442+
assert.lengthOf(statements, 2);
84428443
// Live-item filters must hit the active partial indexes, not walk the thread's history.
84438444
const sql = yield* SqlClient.SqlClient;
84448445
for (const statement of statements.filter((query) => query.includes("turn_items"))) {

0 commit comments

Comments
 (0)