Skip to content

Commit 6738b5a

Browse files
committed
fix(server): retain reset fallback when an early retry omits it
1 parent 4b8f398 commit 6738b5a

4 files changed

Lines changed: 66 additions & 29 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3174,7 +3174,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
31743174
thread.settledOverride === "settled" ||
31753175
run?.id !== command.limitRecovery.runId ||
31763176
failure?.class !== "usage_limit" ||
3177-
failure.resetAt !== command.limitRecovery.resetAt ||
3177+
(failure.resetAt ??
3178+
(thread.limitRecovery?.runId === run.id && thread.limitRecovery.waitForReset === true
3179+
? thread.limitRecovery.resetAt
3180+
: null)) !== command.limitRecovery.resetAt ||
31783181
resetMs <= DateTime.toEpochMillis(run.completedAt ?? run.requestedAt) ||
31793182
projection.runtimeRequests.some((request) => request.status === "pending")
31803183
) {
@@ -5040,7 +5043,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
50405043
!recovery?.autoResume ||
50415044
recovery.requestId !== command.usageLimitRecoveryRequestId ||
50425045
recovery.runId !== run.id ||
5043-
recovery.resetAt !== failure.resetAt ||
5046+
recovery.resetAt !==
5047+
(failure.resetAt ?? (recovery.waitForReset === true ? recovery.resetAt : null)) ||
50445048
(Date.parse(recovery.resetAt) > DateTime.toEpochMillis(now) &&
50455049
(recovery.waitForReset === true ||
50465050
command.usageLimitClearedAt === undefined ||

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -983,6 +983,20 @@ it.effect.each([
983983
const retry = { ...recovery, resetAt: "2099-01-02T00:00:00.000Z", waitForReset: true };
984984
yield* updateThread("retry", { limitRecovery: retry });
985985
assert.deepEqual((yield* candidate)?.limitRecovery, retry);
986+
const failure = (yield* store.getThreadProjection(threadId)).turnItems.find(
987+
(item) => item.id === TurnItemId.make("armed-before-reset:error"),
988+
);
989+
assert(failure?.type === "error");
990+
yield* store.apply({
991+
id: EventId.make("event:armed-before-reset:no-reset"),
992+
type: "turn-item.updated",
993+
threadId,
994+
occurredAt: now,
995+
payload: { ...failure, failure: { ...failure.failure!, resetAt: null } },
996+
});
997+
assert.equal((yield* candidate)?.usageLimitResetAt, retry.resetAt);
998+
yield* updateThread("no-reset-unarmed", {});
999+
assert.isUndefined(yield* candidate);
9861000
yield* updateThread("retry-cancelled", { limitRecovery: { ...retry, autoResume: false } });
9871001
assert.isUndefined(yield* candidate);
9881002
yield* updateThread("snoozed", { snoozedUntil: DateTime.makeUnsafe(resetAt) });

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3863,9 +3863,16 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
38633863
readonly completed_at: string | null;
38643864
readonly failure_payload_json: string;
38653865
readonly last_error: string | null;
3866+
readonly recovery_reset_at: string | null;
38663867
}>`
38673868
SELECT t.payload_json, r.run_id, r.completed_at,
38683869
item.payload_json AS failure_payload_json,
3870+
COALESCE(json_extract(item.payload_json, '$.failure.resetAt'), CASE
3871+
WHEN json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id
3872+
AND json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1
3873+
AND json_extract(t.payload_json, '$.limitRecovery.waitForReset') = 1
3874+
THEN json_extract(t.payload_json, '$.limitRecovery.resetAt')
3875+
END) AS recovery_reset_at,
38693876
(
38703877
SELECT json_extract(session.payload_json, '$.lastError')
38713878
-- Keep bindings outermost: one provider instance can hold every session.
@@ -3911,13 +3918,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
39113918
AND json_extract(t.payload_json, '$.archivedAt') IS NULL
39123919
AND json_extract(t.payload_json, '$.settledOverride') IS NOT 'settled'
39133920
AND json_extract(item.payload_json, '$.failure.class') = 'usage_limit'
3914-
AND json_extract(item.payload_json, '$.failure.resetAt') IS NOT NULL
3915-
AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt')))
3921+
AND recovery_reset_at IS NOT NULL
3922+
AND julianday(recovery_reset_at) > julianday(COALESCE(r.completed_at, json_extract(t.payload_json, '$.updatedAt')))
39163923
AND (
39173924
(
39183925
json_extract(t.payload_json, '$.limitRecovery.runId') IS r.run_id
39193926
AND (
3920-
json_extract(t.payload_json, '$.limitRecovery.resetAt') IS json_extract(item.payload_json, '$.failure.resetAt')
3927+
json_extract(t.payload_json, '$.limitRecovery.resetAt') IS recovery_reset_at
39213928
OR json_extract(t.payload_json, '$.limitRecovery.waitForReset') = 1
39223929
)
39233930
AND json_extract(t.payload_json, '$.limitRecovery.autoResume') = 1
@@ -3929,11 +3936,11 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
39293936
OR (
39303937
(
39313938
json_extract(t.payload_json, '$.limitRecovery.runId') IS NOT r.run_id
3932-
OR json_extract(t.payload_json, '$.limitRecovery.resetAt') IS NOT json_extract(item.payload_json, '$.failure.resetAt')
3939+
OR json_extract(t.payload_json, '$.limitRecovery.resetAt') IS NOT recovery_reset_at
39333940
)
39343941
AND (
39353942
${booleanInt(options.autoResume)}
3936-
OR (${booleanInt(options.snooze)} AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)}))
3943+
OR (${booleanInt(options.snooze)} AND julianday(recovery_reset_at) > julianday(${DateTime.formatIso(options.now)}))
39373944
)
39383945
)
39393946
)
@@ -3957,7 +3964,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
39573964
providerInstanceId: thread.providerInstanceId,
39583965
status: "failed",
39593966
lastErrorClass: summary.lastErrorClass,
3960-
usageLimitResetAt: summary.usageLimitResetAt,
3967+
usageLimitResetAt: row.recovery_reset_at,
39613968
latestRunId: RunId.make(row.run_id),
39623969
latestRunCompletedAt:
39633970
row.completed_at === null ? null : DateTime.makeUnsafe(row.completed_at),
@@ -7075,6 +7082,16 @@ export const layerMemory: Layer.Layer<ProjectionStoreV2> = Layer.effect(
70757082
thread.settledOverride !== "settled",
70767083
)
70777084
.map(threadShellFromProjection)
7085+
.map((thread) => ({
7086+
...thread,
7087+
usageLimitResetAt:
7088+
thread.usageLimitResetAt ??
7089+
(thread.limitRecovery?.runId === thread.latestRunId &&
7090+
thread.limitRecovery.autoResume &&
7091+
thread.limitRecovery.waitForReset === true
7092+
? thread.limitRecovery.resetAt
7093+
: null),
7094+
}))
70787095
.filter(
70797096
(thread) =>
70807097
thread.status === "failed" &&

‎apps/server/src/orchestration-v2/runtimeLayer.test.ts‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6183,15 +6183,15 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
61836183
}),
61846184
);
61856185

6186-
it.effect(
6187-
"resumes an armed continuation early and preserves its reset fallback with global auto-resume off",
6188-
() =>
6186+
it.effect.each([true, false])(
6187+
"preserves early recovery with global auto-resume off when retry reports reset: %s",
6188+
(retryReportsReset) =>
61896189
Effect.gen(function* () {
61906190
const orchestrator = yield* Orchestrator.OrchestratorV2;
61916191
const events = yield* EventSink.EventSinkV2;
61926192
const projections = yield* ProjectionStore.ProjectionStoreV2;
6193-
const threadId = ThreadId.make("recovery:early-reset");
6194-
const projectId = ProjectId.make("recovery:project:early-reset");
6193+
const threadId = ThreadId.make(`recovery:early-reset:${retryReportsReset}`);
6194+
const projectId = ProjectId.make(`recovery:project:early-reset:${retryReportsReset}`);
61956195
const now = yield* DateTime.now;
61966196
yield* seedProject({
61976197
projectId,
@@ -6202,7 +6202,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
62026202
});
62036203
yield* orchestrator.dispatch({
62046204
type: "thread.create",
6205-
commandId: CommandId.make("recovery:create:early-reset"),
6205+
commandId: CommandId.make(`recovery:create:early-reset:${retryReportsReset}`),
62066206
threadId,
62076207
projectId,
62086208
title: "Limited thread",
@@ -6216,9 +6216,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
62166216
});
62176217
yield* orchestrator.dispatch({
62186218
type: "message.dispatch",
6219-
commandId: CommandId.make("recovery:message:early-reset"),
6219+
commandId: CommandId.make(`recovery:message:early-reset:${retryReportsReset}`),
62206220
threadId,
6221-
messageId: MessageId.make("recovery:message:early-reset"),
6221+
messageId: MessageId.make(`recovery:message:early-reset:${retryReportsReset}`),
62226222
text: "Work on this.",
62236223
attachments: [],
62246224
dispatchMode: { type: "defer_start" },
@@ -6230,19 +6230,19 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
62306230
yield* events.write({
62316231
events: [
62326232
{
6233-
id: EventId.make("recovery:run:early-reset"),
6233+
id: EventId.make(`recovery:run:early-reset:${retryReportsReset}`),
62346234
type: "run.updated",
62356235
threadId,
62366236
occurredAt: now,
62376237
payload: { ...run, status: "failed", completedAt: now },
62386238
},
62396239
{
6240-
id: EventId.make("recovery:error:early-reset"),
6240+
id: EventId.make(`recovery:error:early-reset:${retryReportsReset}`),
62416241
type: "turn-item.updated",
62426242
threadId,
62436243
occurredAt: now,
62446244
payload: {
6245-
id: TurnItemId.make("recovery:error:early-reset"),
6245+
id: TurnItemId.make(`recovery:error:early-reset:${retryReportsReset}`),
62466246
type: "error",
62476247
threadId,
62486248
runId: run.id,
@@ -6347,7 +6347,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63476347
assert(staleResume?.type === "message.dispatch");
63486348
yield* orchestrator.dispatch({
63496349
type: "thread.metadata.update",
6350-
commandId: CommandId.make("recovery:cancel:early-reset"),
6350+
commandId: CommandId.make(`recovery:cancel:early-reset:${retryReportsReset}`),
63516351
threadId,
63526352
limitRecovery: { runId: run.id, resetAt, autoResume: false },
63536353
});
@@ -6356,7 +6356,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63566356
assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 1);
63576357
yield* orchestrator.dispatch({
63586358
type: "thread.metadata.update",
6359-
commandId: CommandId.make("recovery:rearm:early-reset"),
6359+
commandId: CommandId.make(`recovery:rearm:early-reset:${retryReportsReset}`),
63606360
threadId,
63616361
limitRecovery: { runId: run.id, resetAt, autoResume: true },
63626362
});
@@ -6370,7 +6370,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63706370
const nodeId = run.rootNodeId;
63716371
assert(nodeId !== null);
63726372
const request = {
6373-
id: RuntimeRequestId.make("recovery:pending-race"),
6373+
id: RuntimeRequestId.make(`recovery:pending-race:${retryReportsReset}`),
63746374
nodeId,
63756375
providerTurnId: null,
63766376
nativeRequestRef: null,
@@ -6383,7 +6383,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63836383
yield* events.write({
63846384
events: [
63856385
{
6386-
id: EventId.make("recovery:pending-race"),
6386+
id: EventId.make(`recovery:pending-race:${retryReportsReset}`),
63876387
type: "runtime-request.updated",
63886388
threadId,
63896389
nodeId,
@@ -6397,7 +6397,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
63976397
yield* events.write({
63986398
events: [
63996399
{
6400-
id: EventId.make("recovery:pending-race:resolved"),
6400+
id: EventId.make(`recovery:pending-race:resolved:${retryReportsReset}`),
64016401
type: "runtime-request.updated",
64026402
threadId,
64036403
nodeId,
@@ -6414,23 +6414,25 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
64146414
// The reading can omit the limit that stopped the run, such as a
64156415
// per-model weekly limit. The early run then fails on the same limit.
64166416
const earlyRun = resumed.runs[1]!;
6417-
const retryResetAt = DateTime.formatIso(DateTime.add(now, { hours: 4 }));
6417+
const retryResetAt = retryReportsReset
6418+
? DateTime.formatIso(DateTime.add(now, { hours: 4 }))
6419+
: resetAt;
64186420
yield* events.write({
64196421
events: [
64206422
{
6421-
id: EventId.make("recovery:run:early-reset:again"),
6423+
id: EventId.make(`recovery:run:early-reset:again:${retryReportsReset}`),
64226424
type: "run.updated",
64236425
threadId,
64246426
occurredAt: checkedNow,
64256427
payload: { ...earlyRun, status: "failed", completedAt: checkedNow },
64266428
},
64276429
{
6428-
id: EventId.make("recovery:error:early-reset:again"),
6430+
id: EventId.make(`recovery:error:early-reset:again:${retryReportsReset}`),
64296431
type: "turn-item.updated",
64306432
threadId,
64316433
occurredAt: checkedNow,
64326434
payload: {
6433-
id: TurnItemId.make("recovery:error:early-reset:again"),
6435+
id: TurnItemId.make(`recovery:error:early-reset:again:${retryReportsReset}`),
64346436
type: "error",
64356437
threadId,
64366438
runId: earlyRun.id,
@@ -6450,7 +6452,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
64506452
message: "Plan limit reached.",
64516453
code: "usageLimitExceeded",
64526454
retryable: null,
6453-
resetAt: retryResetAt,
6455+
resetAt: retryReportsReset ? retryResetAt : null,
64546456
},
64556457
},
64566458
},

0 commit comments

Comments
 (0)