Skip to content

Commit 0ef475e

Browse files
committed
fix(server): memory projection store skips stopped limit recovery runs
1 parent 1c2122b commit 0ef475e

2 files changed

Lines changed: 102 additions & 0 deletions

File tree

‎apps/server/src/orchestration-v2/ProjectionStore.test.ts‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -675,6 +675,98 @@ it.effect("memory recovery selection includes unfinished items from missing runs
675675
}).pipe(Effect.provide(ProjectionStore.layerMemory)),
676676
);
677677

678+
it.effect("memory limit recovery drops a due candidate once a stop request reached its run", () =>
679+
Effect.gen(function* () {
680+
const store = yield* ProjectionStore.ProjectionStoreV2;
681+
const threadId = yield* addRolledBackRecoveryCandidate("memory-limit-interrupted");
682+
const projection = yield* store.getThreadProjection(threadId);
683+
const run = projection.runs[0]!;
684+
const now = yield* DateTime.now;
685+
const resetAt = "2099-01-01T00:00:00.000Z";
686+
const turnItemBase = {
687+
threadId,
688+
runId: run.id,
689+
nodeId: run.rootNodeId,
690+
providerThreadId: null,
691+
providerTurnId: null,
692+
nativeItemRef: null,
693+
parentItemId: null,
694+
startedAt: now,
695+
completedAt: now,
696+
updatedAt: now,
697+
};
698+
yield* store.apply({
699+
id: EventId.make("event:memory-limit-interrupted:error"),
700+
type: "turn-item.updated",
701+
threadId,
702+
occurredAt: now,
703+
payload: {
704+
...turnItemBase,
705+
id: TurnItemId.make("memory-limit-interrupted:error"),
706+
ordinal: 2,
707+
status: "failed",
708+
title: "Usage limit reached",
709+
type: "error",
710+
failure: {
711+
class: "usage_limit",
712+
message: "Plan limit reached.",
713+
resetAt,
714+
code: "usageLimitExceeded",
715+
retryable: null,
716+
},
717+
},
718+
});
719+
yield* store.apply({
720+
id: EventId.make("event:memory-limit-interrupted:run-failed"),
721+
type: "run.updated",
722+
threadId,
723+
occurredAt: now,
724+
payload: { ...run, status: "failed", completedAt: now },
725+
});
726+
yield* store.apply({
727+
id: EventId.make("event:memory-limit-interrupted:armed"),
728+
type: "thread.metadata-updated",
729+
threadId,
730+
occurredAt: now,
731+
payload: {
732+
...projection.thread,
733+
limitRecovery: {
734+
runId: run.id,
735+
resetAt,
736+
autoResume: true,
737+
requestId: CommandId.make("recovery:memory-limit-interrupted"),
738+
},
739+
},
740+
});
741+
const options = {
742+
now: DateTime.makeUnsafe(resetAt),
743+
autoResume: false,
744+
snooze: false,
745+
};
746+
assert.isDefined(
747+
(yield* store.getLimitRecoveryCandidates(options)).find((row) => row.id === threadId),
748+
);
749+
yield* store.apply({
750+
id: EventId.make("event:memory-limit-interrupted:interrupt-request"),
751+
type: "turn-item.updated",
752+
threadId,
753+
occurredAt: now,
754+
payload: {
755+
...turnItemBase,
756+
id: TurnItemId.make("memory-limit-interrupted:interrupt-request"),
757+
ordinal: 3,
758+
status: "completed",
759+
title: "Interrupt requested",
760+
type: "run_interrupt_request",
761+
message: "Interrupt requested",
762+
},
763+
});
764+
assert.isUndefined(
765+
(yield* store.getLimitRecoveryCandidates(options)).find((row) => row.id === threadId),
766+
);
767+
}).pipe(Effect.provide(ProjectionStore.layerMemory)),
768+
);
769+
678770
const runlessNativeSubagentNeedsRecovery = Effect.gen(function* () {
679771
const store = yield* ProjectionStore.ProjectionStoreV2;
680772
const threadId = yield* addRolledBackRecoveryCandidate("runless-native-recovery");

‎apps/server/src/orchestration-v2/ProjectionStore.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6283,6 +6283,16 @@ export const layerMemory: Layer.Layer<ProjectionStoreV2> = Layer.effect(
62836283
thread.usageLimitResetAt !== null &&
62846284
thread.pendingRuntimeRequest === null,
62856285
)
6286+
// Stop reached this run: only a fresh user action may resume it.
6287+
.filter(
6288+
(thread) =>
6289+
!state.projections
6290+
.get(thread.id)
6291+
?.turnItems.some(
6292+
(item) =>
6293+
item.type === "run_interrupt_request" && item.runId === thread.latestRunId,
6294+
),
6295+
)
62866296
.filter((thread) => {
62876297
const resetMs = Date.parse(thread.usageLimitResetAt!);
62886298
const nowMs = DateTime.toEpochMillis(options.now);

0 commit comments

Comments
 (0)