Skip to content

Commit 7a2b1c7

Browse files
StiensWoutclaude
andauthored
fix(server): reconcile Pi native session rewinds (pingdotgg#13839)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent c908cea commit 7a2b1c7

5 files changed

Lines changed: 861 additions & 58 deletions

File tree

‎apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts‎

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { assert, it } from "@effect/vitest";
22
import {
3+
CheckpointId,
4+
CheckpointRef,
5+
CheckpointScopeId,
36
MessageId,
47
type ModelSelection,
58
NodeId,
@@ -14,6 +17,8 @@ import {
1417
ProviderDriverKind,
1518
ProviderInstanceId,
1619
PlanId,
20+
type ProviderThreadId,
21+
ProviderTurnId,
1722
RunAttemptId,
1823
RunId,
1924
RuntimeRequestId,
@@ -318,6 +323,254 @@ layer("ProviderEventIngestorV2", (it) => {
318323
}),
319324
);
320325

326+
it.effect("rolls back runs a provider rewound off its active branch", () =>
327+
Effect.gen(function* () {
328+
const now = yield* DateTime.now;
329+
const eventSink = yield* EventSink.EventSinkV2;
330+
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
331+
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
332+
const idAllocator = yield* IdAllocator.IdAllocatorV2;
333+
const threadEvent = yield* threadCreatedEvent(now);
334+
const threadId = threadEvent.threadId;
335+
const providerSessionId = yield* idAllocator.allocate.providerSession({
336+
providerInstanceId: modelSelection.instanceId,
337+
threadId,
338+
});
339+
const rewoundThreadId = idAllocator.derive.providerThread({
340+
driver: CODEX_DRIVER,
341+
nativeThreadId: "rewound-thread",
342+
});
343+
const otherThreadId = idAllocator.derive.providerThread({
344+
driver: CODEX_DRIVER,
345+
nativeThreadId: "other-thread",
346+
});
347+
const scopeId = CheckpointScopeId.make("checkpoint-scope:rewound");
348+
// A run as execution leaves it: attempt, provider turn, root node, and a
349+
// checkpoint once completed. A null native turn id is a weak ref.
350+
const seedRun = Effect.fnUntraced(function* (input: {
351+
readonly ordinal: number;
352+
readonly providerThreadId: ProviderThreadId;
353+
readonly status: "completed" | "interrupted" | "running";
354+
readonly nativeTurnId: string | null;
355+
}) {
356+
const runId = RunId.make(`run:rewound:${input.ordinal}`);
357+
const attemptId = RunAttemptId.make(`attempt:rewound:${input.ordinal}`);
358+
const nodeId = NodeId.make(`node:rewound:${input.ordinal}`);
359+
const providerTurnId = ProviderTurnId.make(`provider-turn:rewound:${input.ordinal}`);
360+
const completedAt = input.status === "running" ? null : now;
361+
const eventId = () => idAllocator.allocate.event({ threadId });
362+
const events: Array<OrchestrationV2DomainEvent> = [
363+
{
364+
id: yield* eventId(),
365+
type: "run.created",
366+
threadId,
367+
occurredAt: now,
368+
payload: {
369+
id: runId,
370+
threadId,
371+
ordinal: input.ordinal,
372+
providerInstanceId: modelSelection.instanceId,
373+
modelSelection,
374+
providerThreadId: input.providerThreadId,
375+
userMessageId: MessageId.make(`message:rewound:${input.ordinal}`),
376+
rootNodeId: nodeId,
377+
activeAttemptId: attemptId,
378+
status: input.status,
379+
requestedAt: now,
380+
startedAt: now,
381+
completedAt,
382+
checkpointId: null,
383+
contextHandoffId: null,
384+
},
385+
},
386+
{
387+
id: yield* eventId(),
388+
type: "run-attempt.created",
389+
threadId,
390+
occurredAt: now,
391+
payload: {
392+
id: attemptId,
393+
runId,
394+
attemptOrdinal: 1,
395+
rootNodeId: nodeId,
396+
providerInstanceId: modelSelection.instanceId,
397+
providerThreadId: input.providerThreadId,
398+
providerTurnId,
399+
reason: "initial",
400+
status: input.status,
401+
startedAt: now,
402+
completedAt,
403+
},
404+
},
405+
{
406+
id: yield* eventId(),
407+
type: "provider-turn.updated",
408+
threadId,
409+
occurredAt: now,
410+
payload: {
411+
id: providerTurnId,
412+
providerThreadId: input.providerThreadId,
413+
nodeId,
414+
runAttemptId: attemptId,
415+
nativeTurnRef:
416+
input.nativeTurnId === null
417+
? {
418+
driver: CODEX_DRIVER,
419+
nativeId: `synthetic:${input.ordinal}`,
420+
strength: "weak",
421+
}
422+
: { driver: CODEX_DRIVER, nativeId: input.nativeTurnId, strength: "strong" },
423+
ordinal: input.ordinal,
424+
status: input.status,
425+
startedAt: now,
426+
completedAt,
427+
},
428+
},
429+
{
430+
id: yield* eventId(),
431+
type: "node.updated",
432+
threadId,
433+
occurredAt: now,
434+
payload: {
435+
id: nodeId,
436+
threadId,
437+
runId,
438+
parentNodeId: null,
439+
rootNodeId: nodeId,
440+
kind: "root_turn",
441+
status: input.status,
442+
countsForRun: true,
443+
providerThreadId: input.providerThreadId,
444+
providerTurnId,
445+
nativeItemRef: null,
446+
runtimeRequestId: null,
447+
checkpointScopeId: scopeId,
448+
startedAt: now,
449+
completedAt,
450+
},
451+
},
452+
...(input.status === "completed"
453+
? [
454+
{
455+
id: yield* eventId(),
456+
type: "checkpoint.captured" as const,
457+
threadId,
458+
occurredAt: now,
459+
payload: {
460+
id: CheckpointId.make(`checkpoint:rewound:${input.ordinal}`),
461+
threadId,
462+
scopeId,
463+
runId,
464+
nodeId,
465+
parentCheckpointId: null,
466+
ordinalWithinScope: input.ordinal,
467+
appRunOrdinal: input.ordinal,
468+
ref: CheckpointRef.make(`refs/t3/checkpoints/rewound/${input.ordinal}`),
469+
status: "ready" as const,
470+
files: [],
471+
capturedAt: now,
472+
},
473+
},
474+
]
475+
: []),
476+
];
477+
yield* eventSink.write({ events });
478+
return runId;
479+
});
480+
481+
yield* eventSink.write({ events: [threadEvent] });
482+
const otherProviderRun = yield* seedRun({
483+
ordinal: 1,
484+
providerThreadId: otherThreadId,
485+
status: "completed",
486+
nativeTurnId: "other-turn",
487+
});
488+
const keptRun = yield* seedRun({
489+
ordinal: 2,
490+
providerThreadId: rewoundThreadId,
491+
status: "completed",
492+
nativeTurnId: "kept-turn",
493+
});
494+
const abandonedRun = yield* seedRun({
495+
ordinal: 3,
496+
providerThreadId: rewoundThreadId,
497+
status: "completed",
498+
nativeTurnId: "abandoned-turn",
499+
});
500+
const unlocatedRun = yield* seedRun({
501+
ordinal: 4,
502+
providerThreadId: rewoundThreadId,
503+
status: "interrupted",
504+
nativeTurnId: null,
505+
});
506+
const rewindingRun = yield* seedRun({
507+
ordinal: 5,
508+
providerThreadId: rewoundThreadId,
509+
status: "running",
510+
nativeTurnId: null,
511+
});
512+
513+
yield* ingestor.ingestNormalized({
514+
providerSessionId,
515+
providerInstanceId: modelSelection.instanceId,
516+
threadId,
517+
runId: rewindingRun,
518+
event: {
519+
type: "provider_thread.updated",
520+
driver: CODEX_DRIVER,
521+
providerThread: {
522+
id: rewoundThreadId,
523+
driver: CODEX_DRIVER,
524+
providerInstanceId: modelSelection.instanceId,
525+
providerSessionId,
526+
appThreadId: threadId,
527+
ownerNodeId: null,
528+
nativeThreadRef: {
529+
driver: CODEX_DRIVER,
530+
nativeId: "rewound-thread",
531+
strength: "strong",
532+
},
533+
nativeConversationHeadRef: {
534+
driver: CODEX_DRIVER,
535+
nativeId: "kept-turn-reply",
536+
strength: "strong",
537+
},
538+
status: "idle",
539+
firstRunOrdinal: 2,
540+
lastRunOrdinal: 5,
541+
handoffIds: [],
542+
forkedFrom: null,
543+
createdAt: now,
544+
updatedAt: now,
545+
},
546+
retainedNativeTurnIds: ["turn-started-outside-t3", "kept-turn"],
547+
},
548+
});
549+
550+
const projection = yield* projectionStore.getThreadProjection(threadId);
551+
const runIds = [otherProviderRun, keptRun, abandonedRun, unlocatedRun, rewindingRun];
552+
assert.deepEqual(
553+
runIds.map((runId) => projection.runs.find((run) => run.id === runId)?.status),
554+
["completed", "completed", "rolled_back", "rolled_back", "running"],
555+
);
556+
assert.deepEqual(
557+
runIds.map((runId) => projection.nodes.find((node) => node.runId === runId)?.status),
558+
["completed", "completed", "rolled_back", "rolled_back", "running"],
559+
);
560+
assert.deepEqual(
561+
[otherProviderRun, keptRun, abandonedRun].map(
562+
(runId) =>
563+
projection.checkpoints.find((checkpoint) => checkpoint.runId === runId)?.status,
564+
),
565+
["ready", "ready", "stale"],
566+
);
567+
assert.equal(
568+
projection.providerThreads.find((thread) => thread.id === rewoundThreadId)?.lastRunOrdinal,
569+
5,
570+
);
571+
}),
572+
);
573+
321574
it.effect("carries plan-step durations through consecutive and restarted ingestion", () =>
322575
Effect.gen(function* () {
323576
yield* TestClock.setTime(Date.parse("2026-09-07T00:00:00.000Z"));

0 commit comments

Comments
 (0)