Skip to content

Commit 5d4c49e

Browse files
committed
fix(pi): stop marking recovered turns as failed
1 parent a96cf08 commit 5d4c49e

3 files changed

Lines changed: 267 additions & 12 deletions

File tree

‎apps/server/src/orchestration-v2/testkit/PiReplayFixtures.integration.test.ts‎

Lines changed: 72 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,79 @@
1-
import { describe } from "@effect/vitest";
1+
import { assert, describe, it } from "@effect/vitest";
2+
import * as Effect from "effect/Effect";
3+
import type { ProviderReplayTranscript } from "@t3tools/contracts";
4+
import * as HostProcess from "@t3tools/shared/HostProcess";
25

36
import { PiOrchestratorReplayHarness } from "../Adapters/PiAdapterV2.testkit.ts";
4-
import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
7+
import {
8+
findReplayFixture,
9+
itRunsReplayFixtures,
10+
runReplayFixture,
11+
} from "./ProviderReplayFixtures.ts";
512

613
describe("Pi replay fixtures", () => {
14+
it.effect("does not persist a recovered Pi message error as a failed run", () =>
15+
Effect.gen(function* () {
16+
const fixture = findReplayFixture("simple");
17+
assert.isDefined(fixture);
18+
if (fixture === undefined) return;
19+
const driver = fixture.providers.find((provider) => provider.driver === "pi");
20+
assert.isDefined(driver);
21+
if (driver === undefined) return;
22+
const result = yield* runReplayFixture({
23+
fixtureName: fixture.name,
24+
buildInput: fixture.buildInput,
25+
driver,
26+
harness: PiOrchestratorReplayHarness,
27+
transformTranscript: (transcript) => ({
28+
...transcript,
29+
entries: transcript.entries.flatMap<ProviderReplayTranscript["entries"][number]>(
30+
(entry) =>
31+
entry.type === "emit_inbound" && entry.label === "agent_start"
32+
? [
33+
entry,
34+
{
35+
type: "emit_inbound",
36+
frame: {
37+
type: "message_end",
38+
message: {
39+
role: "assistant",
40+
content: [],
41+
stopReason: "error",
42+
errorMessage: "This operation was aborted",
43+
},
44+
},
45+
},
46+
{
47+
type: "emit_inbound",
48+
frame: { type: "agent_end", messages: [], willRetry: false },
49+
},
50+
{ type: "emit_inbound", frame: { type: "agent_start" } },
51+
]
52+
: [entry],
53+
),
54+
}),
55+
});
56+
for (const projection of result.projections.values()) {
57+
assert.deepEqual(
58+
projection.attempts.map((attempt) => attempt.status),
59+
["completed"],
60+
);
61+
assert.deepEqual(
62+
projection.providerSessions.map((session) => ({
63+
status: session.status,
64+
lastError: session.lastError,
65+
})),
66+
[{ status: "ready", lastError: null }],
67+
);
68+
assert.isTrue(projection.providerTurns.every((turn) => turn.status === "completed"));
69+
assert.isFalse(projection.turnItems.some((item) => item.type === "error"));
70+
}
71+
}).pipe(
72+
// The recording uses POSIX argv, without Windows npm .cmd wrapping.
73+
Effect.provideService(HostProcess.Platform, "linux"),
74+
),
75+
);
76+
777
itRunsReplayFixtures("pi", (input) =>
878
runReplayFixture({ ...input, harness: PiOrchestratorReplayHarness }),
979
);

‎packages/provider-pi/src/server/adapter.test.ts‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3737,6 +3737,182 @@ describe("PiAdapterV2", () => {
37373737
}).pipe(Effect.scoped, Effect.provide(layerTest)),
37383738
);
37393739

3740+
it.effect.each([false, true])(
3741+
"completes recovered message errors after native continuation, compaction=%s",
3742+
(compact) =>
3743+
Effect.gen(function* () {
3744+
const fake = yield* makeFakePi;
3745+
const { runtime, takeEvent } = yield* openRuntime(fake);
3746+
const providerThread = yield* runtime.ensureThread({
3747+
threadId: THREAD_ID,
3748+
modelSelection: modelSelection("default"),
3749+
runtimePolicy,
3750+
});
3751+
yield* startTurn(runtime, providerThread);
3752+
yield* fake.takeRequest("prompt");
3753+
yield* fake.emit({ type: "agent_start" });
3754+
yield* fake.emit({
3755+
type: "message_end",
3756+
message: {
3757+
role: "assistant",
3758+
content: [],
3759+
stopReason: "error",
3760+
errorMessage: "This operation was aborted",
3761+
},
3762+
});
3763+
yield* fake.emit({ type: "agent_end", messages: [], willRetry: false });
3764+
if (compact) {
3765+
yield* fake.emit({ type: "compaction_start", reason: "manual" });
3766+
yield* fake.emit({
3767+
type: "compaction_end",
3768+
reason: "manual",
3769+
result: { summary: "Recovered context" },
3770+
aborted: false,
3771+
willRetry: false,
3772+
});
3773+
}
3774+
yield* fake.emit({ type: "agent_start" });
3775+
yield* fake.emit({ type: "message_start", message: { role: "assistant" } });
3776+
yield* fake.emit({
3777+
type: "message_update",
3778+
assistantMessageEvent: { type: "text_end", contentIndex: 0, content: "Paused." },
3779+
});
3780+
yield* fake.emit({
3781+
type: "message_end",
3782+
message: {
3783+
role: "assistant",
3784+
content: [{ type: "text", text: "Paused." }],
3785+
stopReason: "stop",
3786+
},
3787+
});
3788+
yield* fake.emit({ type: "agent_settled", aborted: false });
3789+
3790+
const completedTurn = yield* takeEvent(
3791+
(event) =>
3792+
event.type === "provider_turn.updated" && event.providerTurn.completedAt !== null,
3793+
);
3794+
assert.isTrue(
3795+
completedTurn.type === "provider_turn.updated" &&
3796+
completedTurn.providerTurn.status === "completed",
3797+
);
3798+
const session = yield* takeEvent((event) => event.type === "provider_session.updated");
3799+
assert.isTrue(
3800+
session.type === "provider_session.updated" &&
3801+
session.providerSession.status === "ready" &&
3802+
session.providerSession.lastError === null,
3803+
);
3804+
const terminal = yield* takeEvent((event) => event.type === "turn.terminal");
3805+
assert.isTrue(
3806+
terminal.type === "turn.terminal" &&
3807+
terminal.status === "completed" &&
3808+
terminal.failure === null,
3809+
);
3810+
}).pipe(Effect.scoped, Effect.provide(layerTest)),
3811+
);
3812+
3813+
it.effect.each([
3814+
{
3815+
name: "unrecovered error",
3816+
messages: [],
3817+
exhausted: false,
3818+
aborted: false,
3819+
error: "original error",
3820+
},
3821+
{
3822+
name: "unfinished continuation",
3823+
messages: [
3824+
{ type: "agent_start" },
3825+
{ type: "message_start", message: { role: "assistant" } },
3826+
],
3827+
exhausted: false,
3828+
aborted: false,
3829+
error: "original error",
3830+
},
3831+
{
3832+
name: "later request error",
3833+
messages: [
3834+
{
3835+
type: "message_end",
3836+
message: { role: "assistant", stopReason: "error", errorMessage: "latest error" },
3837+
},
3838+
],
3839+
exhausted: false,
3840+
aborted: false,
3841+
error: "latest error",
3842+
},
3843+
{
3844+
name: "failed request after a successful tool response",
3845+
messages: [
3846+
{ type: "message_end", message: { role: "assistant", stopReason: "toolUse" } },
3847+
{
3848+
type: "message_end",
3849+
message: { role: "assistant", stopReason: "error", errorMessage: "latest error" },
3850+
},
3851+
],
3852+
exhausted: false,
3853+
aborted: false,
3854+
error: "latest error",
3855+
},
3856+
{
3857+
name: "compaction without successful continuation",
3858+
messages: [
3859+
{ type: "compaction_start", reason: "manual" },
3860+
{
3861+
type: "compaction_end",
3862+
reason: "manual",
3863+
result: { summary: "Smaller context" },
3864+
aborted: false,
3865+
willRetry: false,
3866+
},
3867+
],
3868+
exhausted: false,
3869+
aborted: false,
3870+
error: "original error",
3871+
},
3872+
{
3873+
name: "exhausted retry",
3874+
messages: [{ type: "message_end", message: { role: "assistant", stopReason: "stop" } }],
3875+
exhausted: true,
3876+
aborted: false,
3877+
error: "retry exhausted",
3878+
},
3879+
{ name: "native cancellation", messages: [], exhausted: false, aborted: true, error: null },
3880+
])(
3881+
"preserves terminal semantics after a message error: $name",
3882+
({ messages, exhausted, aborted, error }) =>
3883+
Effect.gen(function* () {
3884+
const fake = yield* makeFakePi;
3885+
const { runtime, takeEvent } = yield* openRuntime(fake);
3886+
const providerThread = yield* runtime.ensureThread({
3887+
threadId: THREAD_ID,
3888+
modelSelection: modelSelection("default"),
3889+
runtimePolicy,
3890+
});
3891+
yield* startTurn(runtime, providerThread);
3892+
yield* fake.takeRequest("prompt");
3893+
yield* fake.emit({ type: "agent_start" });
3894+
yield* fake.emit({
3895+
type: "message_end",
3896+
message: { role: "assistant", stopReason: "error", errorMessage: "original error" },
3897+
});
3898+
if (exhausted) {
3899+
yield* fake.emit({
3900+
type: "auto_retry_end",
3901+
success: false,
3902+
attempt: 3,
3903+
finalError: error,
3904+
});
3905+
}
3906+
for (const message of messages) yield* fake.emit(message);
3907+
yield* fake.emit({ type: "agent_settled", aborted });
3908+
const terminal = yield* takeEvent((event) => event.type === "turn.terminal");
3909+
assert.isTrue(terminal.type === "turn.terminal");
3910+
if (terminal.type !== "turn.terminal") return;
3911+
assert.equal(terminal.status, aborted ? "interrupted" : "failed");
3912+
assert.equal(terminal.failure?.message ?? null, error);
3913+
}).pipe(Effect.scoped, Effect.provide(layerTest)),
3914+
);
3915+
37403916
it.effect("marks retry progress recovered when Pi succeeds", () =>
37413917
Effect.gen(function* () {
37423918
const fake = yield* makeFakePi;

‎packages/provider-pi/src/server/adapter.ts‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,7 @@ interface ActivePiTurn {
369369
activeCompaction: PiCompactionState | null;
370370
activeProviderRetry: PiProviderRetryState | null;
371371
failure: ReturnType<typeof makeProviderFailure> | null;
372+
messageFailure: ReturnType<typeof makeProviderFailure> | null;
372373
rejectedPromptFailure: ReturnType<typeof makeProviderFailure> | null;
373374
stopTreeRefs?: PiTurnTreeRefs | null;
374375
}
@@ -1462,18 +1463,21 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
14621463
if (turn === null) return;
14631464
state.activeTurn = null;
14641465
const interrupted = turn.interrupted || turn.nativeAborted;
1466+
const failure = interrupted
1467+
? null
1468+
: (turn.rejectedPromptFailure ?? turn.failure ?? turn.messageFailure);
14651469
const completedAt = yield* DateTime.now;
14661470
yield* completeOpenStreamItems(turn);
14671471
if (turn.activeCompaction !== null) {
1468-
const status = interrupted ? "cancelled" : turn.failure === null ? "completed" : "failed";
1472+
const status = interrupted ? "cancelled" : failure === null ? "completed" : "failed";
14691473
yield* emitCompaction(turn, turn.activeCompaction, status);
14701474
turn.activeCompaction = null;
14711475
}
14721476
if (turn.activeProviderRetry !== null) {
14731477
if (interrupted) {
14741478
yield* emitProviderRetry(turn, turn.activeProviderRetry, "interrupted", completedAt);
14751479
turn.activeProviderRetry = null;
1476-
} else if (turn.failure === null) {
1480+
} else if (failure === null) {
14771481
yield* emitProviderRetry(turn, turn.activeProviderRetry, "completed", completedAt);
14781482
turn.activeProviderRetry = null;
14791483
}
@@ -1484,7 +1488,6 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
14841488
const tokenUsage = readUsage
14851489
? yield* readTokenUsage(turn.latestCompactionAfterTokens, completedAt)
14861490
: undefined;
1487-
const failure = interrupted ? null : (turn.rejectedPromptFailure ?? turn.failure);
14881491
yield* emit({
14891492
type: "provider_turn.updated",
14901493
driver: PI_PROVIDER,
@@ -1725,12 +1728,13 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
17251728
const message = event["message"];
17261729
if (recordString(message, "role") !== "assistant") return;
17271730
yield* completeOpenStreamItems(turn);
1728-
if (recordString(message, "stopReason") === "error" && turn.failure === null) {
1729-
turn.failure = makeProviderFailure({
1730-
message: recordString(message, "errorMessage") ?? "Pi reported a model error.",
1731-
class: "provider_error",
1732-
});
1733-
}
1731+
turn.messageFailure =
1732+
recordString(message, "stopReason") === "error"
1733+
? makeProviderFailure({
1734+
message: recordString(message, "errorMessage") ?? "Pi reported a model error.",
1735+
class: "provider_error",
1736+
})
1737+
: null;
17341738
return;
17351739
}
17361740
case "tool_execution_start":
@@ -1789,7 +1793,10 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
17891793
return;
17901794
}
17911795
// Clear overflow failures only when compaction will retry; ordinary compaction must retain exhausted retries.
1792-
if (event["willRetry"] === true) turn.failure = null;
1796+
if (event["willRetry"] === true) {
1797+
turn.failure = null;
1798+
turn.messageFailure = null;
1799+
}
17931800
turn.latestCompactionAfterTokens =
17941801
nonNegativeInteger(result, "estimatedTokensAfter") ?? null;
17951802
const summary = recordString(result, "summary");
@@ -1852,6 +1859,7 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
18521859
turn.activeProviderRetry = null;
18531860
}
18541861
turn.failure = null;
1862+
turn.messageFailure = null;
18551863
return;
18561864
}
18571865
const failure = makeProviderFailure({
@@ -2464,6 +2472,7 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
24642472
activeCompaction: null,
24652473
activeProviderRetry: null,
24662474
failure: null,
2475+
messageFailure: null,
24672476
rejectedPromptFailure: null,
24682477
};
24692478
// Lock only install/send/start events; correlated setup must leave the pump free for dialogs.

0 commit comments

Comments
 (0)