Skip to content

Commit 8bcf676

Browse files
committed
fix(server): preserve account-specific reset recovery evidence
Reset recovery trusted a post-consume allowance without checking its account, overwrote fresh result metadata with cached values, and only revisited one instance after a confirmed reset. Require matching identity and preserve fresh driver evidence while probing each linked account instance independently. Bound client acquisition and consume together for both redemption paths. Keep ambiguous automatic keys for reconciliation by either caller, expiring them at the same window boundary as the durable ledger; unknown manual keys remain guarded until settled. Add regressions for mismatched accounts, fresh report metadata, independent instance continuation, ambiguous key reconciliation and expiry, and an app-server that never initialises.
1 parent d168903 commit 8bcf676

11 files changed

Lines changed: 596 additions & 118 deletions

File tree

‎apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts‎

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
ProviderDriverKind,
55
RunId,
66
ThreadId,
7+
type OrchestrationV2LimitRecovery,
78
type ServerProvider,
89
type OrchestrationV2DomainEvent,
910
} from "@t3tools/contracts";
@@ -105,6 +106,256 @@ it.effect("backs off a pending account attempt across repeated sweep ticks", ()
105106
}),
106107
);
107108

109+
it.effect("reports the driver's fresh redemption timestamp and credit count", () =>
110+
Effect.gen(function* () {
111+
yield* TestClock.setTime(nowMs);
112+
const base = failedThread(0, -HOUR_MS);
113+
const fingerprint = "codex-account:fresh-result";
114+
const checkedAt = DateTime.formatIso(DateTime.makeUnsafe(nowMs + 1_000));
115+
const thread = {
116+
...base,
117+
limitRecovery: {
118+
runId: base.latestRunId!,
119+
resetAt: base.usageLimitResetAt!,
120+
autoResume: true,
121+
},
122+
};
123+
const provider = {
124+
instanceId: base.providerInstanceId,
125+
driver: ProviderDriverKind.make("codex"),
126+
usageLimits: {
127+
credentialFingerprint: fingerprint,
128+
checkedAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - 60_000)),
129+
windows: [],
130+
resetCredits: { availableCount: 1 },
131+
},
132+
} as unknown as ServerProvider;
133+
const ledger = yield* Ref.make<ProjectionStore.CodexResetAttempt | null>(null);
134+
const reports = yield* Ref.make<
135+
ReadonlyArray<{
136+
readonly threadId: ThreadId;
137+
readonly report: NonNullable<OrchestrationV2LimitRecovery["codexResetRedemption"]>;
138+
}>
139+
>([]);
140+
const dispatches = yield* Ref.make<ReadonlyArray<ThreadId>>([]);
141+
const layer = Layer.mergeAll(
142+
Layer.succeed(ProjectionStore.ProjectionStoreV2, {
143+
getLimitRecoveryCandidates: () => Effect.succeed([thread]),
144+
getLimitRecoveryWakeAt: () => Effect.succeed(nowMs + HOUR_MS),
145+
getCodexResetAttempt: () => Ref.get(ledger),
146+
claimCodexResetAttempt: (attempt: ProjectionStore.CodexResetAttempt) =>
147+
Ref.set(ledger, attempt).pipe(Effect.as(true)),
148+
updateCodexResetAttempt: (attempt: ProjectionStore.CodexResetAttempt) =>
149+
Ref.set(ledger, attempt),
150+
releaseCodexResetAttempt: () => Ref.set(ledger, null),
151+
} as never),
152+
Layer.mock(ThreadManagement.ThreadManagementService)({
153+
dispatch: (command) =>
154+
command.type === "message.dispatch"
155+
? Ref.update(dispatches, (ids) => [...ids, command.threadId]).pipe(
156+
Effect.as({ sequence: 1, storedEvents: [] }),
157+
)
158+
: Effect.die("expected a limit continuation"),
159+
}),
160+
Layer.mock(Orchestrator.OrchestratorV2)({
161+
streamDomainEvents: Stream.never,
162+
recordCodexResetRedemption: (input) =>
163+
Ref.update(reports, (entries) => [
164+
...entries,
165+
{ threadId: input.threadId, report: input.codexResetRedemption },
166+
]).pipe(Effect.as(true)),
167+
}),
168+
Layer.mock(ServerSettings.ServerSettingsService)({
169+
getSettings: Effect.succeed({
170+
autoResumeLimitedThreads: true,
171+
snoozeLimitedThreads: false,
172+
autoRedeemCodexResetCredits: { [base.providerInstanceId]: true },
173+
} as never),
174+
streamChanges: Stream.never,
175+
}),
176+
Layer.mock(ProviderRegistry.ProviderRegistry)({
177+
getProviders: Effect.succeed([provider]),
178+
refreshInstance: () => Effect.succeed([provider]),
179+
supportsAutoRedeemResetCredit: () => Effect.succeed(true),
180+
autoRedeemResetCredit: (input) =>
181+
Effect.succeed({
182+
status: "sent",
183+
accountFingerprint: fingerprint,
184+
creditsBefore: 4,
185+
outcome: "reset",
186+
checkedAt,
187+
allowanceCleared: true,
188+
idempotencyKey: input.idempotencyKey,
189+
} as const),
190+
}),
191+
);
192+
const sweep = yield* UsageLimitRecoveryWorker.makeSweep.pipe(Effect.provide(layer));
193+
yield* sweep();
194+
195+
const driverReport = (yield* Ref.get(reports)).find(
196+
(entry) => entry.threadId === base.id && entry.report.outcome === "reset",
197+
);
198+
assert.isDefined(driverReport);
199+
assert.equal(driverReport.report.checkedAt, checkedAt);
200+
assert.equal(driverReport.report.creditsBefore, 4);
201+
assert.isTrue(driverReport.report.allowanceCleared);
202+
assert.deepEqual(yield* Ref.get(dispatches), [base.id]);
203+
}),
204+
);
205+
206+
it.effect("resumes a cleared account instance after a reset while the driver remains limited", () =>
207+
Effect.gen(function* () {
208+
yield* TestClock.setTime(nowMs);
209+
const base = failedThread(0, -HOUR_MS);
210+
const driverInstanceId = base.providerInstanceId;
211+
const secondInstanceId = ProviderInstanceId.make("provider-codex-second-instance");
212+
const fingerprint = "codex-account:follow-up";
213+
const attemptedAt = DateTime.formatIso(DateTime.makeUnsafe(nowMs - 120_000));
214+
const checkedAt = DateTime.formatIso(DateTime.makeUnsafe(nowMs - 60_000));
215+
const threadFor = (input: { readonly id: string; readonly instanceId: ProviderInstanceId }) => {
216+
const threadId = ThreadId.make(input.id);
217+
const runId = RunId.make(`run:${input.id}`);
218+
return {
219+
...base,
220+
id: threadId,
221+
providerInstanceId: input.instanceId,
222+
modelSelection: { ...base.modelSelection, instanceId: input.instanceId },
223+
latestRunId: runId,
224+
limitRecovery: {
225+
runId,
226+
resetAt: base.usageLimitResetAt!,
227+
autoResume: true,
228+
codexResetRedemption: {
229+
providerInstanceId: input.instanceId,
230+
redeemedByProviderInstanceId: driverInstanceId,
231+
accountFingerprint: fingerprint,
232+
idempotencyKey: "codex-auto-reset:follow-up",
233+
attemptedAt,
234+
creditsBefore: 1,
235+
outcome: "reset" as const,
236+
checkedAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - 180_000)),
237+
allowanceCleared: false,
238+
},
239+
},
240+
};
241+
};
242+
const driverThread = threadFor({ id: "thread:follow-up-driver", instanceId: driverInstanceId });
243+
const secondThread = threadFor({ id: "thread:follow-up-second", instanceId: secondInstanceId });
244+
const candidates = [driverThread, secondThread];
245+
const previousAttempt: ProjectionStore.CodexResetAttempt = {
246+
accountFingerprint: fingerprint,
247+
resetAt: base.usageLimitResetAt!,
248+
idempotencyKey: "codex-auto-reset:follow-up",
249+
attemptedAt,
250+
outcome: "reset",
251+
allowanceCleared: false,
252+
checkedAt,
253+
nextCheckAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - 1_000)),
254+
};
255+
const ledger = yield* Ref.make(previousAttempt);
256+
const cachedProvider = (instanceId: ProviderInstanceId) =>
257+
({
258+
instanceId,
259+
driver: ProviderDriverKind.make("codex"),
260+
usageLimits: {
261+
credentialFingerprint: fingerprint,
262+
checkedAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - 180_000)),
263+
windows: [],
264+
resetCredits: { availableCount: 1 },
265+
},
266+
}) as unknown as ServerProvider;
267+
const refreshedDriver = {
268+
...cachedProvider(driverInstanceId),
269+
usageLimits: {
270+
credentialFingerprint: fingerprint,
271+
checkedAt,
272+
windows: [
273+
{
274+
id: "session",
275+
label: "Session",
276+
usedPercent: 100,
277+
resetsAt: base.usageLimitResetAt,
278+
},
279+
],
280+
resetCredits: { availableCount: 1 },
281+
},
282+
} as unknown as ServerProvider;
283+
const refreshedSecond = {
284+
...cachedProvider(secondInstanceId),
285+
usageLimits: {
286+
credentialFingerprint: fingerprint,
287+
checkedAt,
288+
windows: [],
289+
resetCredits: { availableCount: 1 },
290+
},
291+
} as unknown as ServerProvider;
292+
const reports = yield* Ref.make<
293+
ReadonlyArray<{
294+
readonly threadId: ThreadId;
295+
readonly report: NonNullable<OrchestrationV2LimitRecovery["codexResetRedemption"]>;
296+
}>
297+
>([]);
298+
const dispatches = yield* Ref.make<ReadonlyArray<ThreadId>>([]);
299+
const layer = Layer.mergeAll(
300+
Layer.succeed(ProjectionStore.ProjectionStoreV2, {
301+
getLimitRecoveryCandidates: () => Effect.succeed(candidates),
302+
getLimitRecoveryWakeAt: () => Effect.succeed(nowMs + HOUR_MS),
303+
getCodexResetAttempt: () => Ref.get(ledger),
304+
updateCodexResetAttempt: (attempt: ProjectionStore.CodexResetAttempt) =>
305+
Ref.set(ledger, attempt),
306+
} as never),
307+
Layer.mock(ThreadManagement.ThreadManagementService)({
308+
dispatch: (command) =>
309+
command.type === "message.dispatch"
310+
? Ref.update(dispatches, (ids) => [...ids, command.threadId]).pipe(
311+
Effect.as({ sequence: 1, storedEvents: [] }),
312+
)
313+
: Effect.die("expected a limit continuation"),
314+
}),
315+
Layer.mock(Orchestrator.OrchestratorV2)({
316+
streamDomainEvents: Stream.never,
317+
recordCodexResetRedemption: (input) =>
318+
Ref.update(reports, (entries) => [
319+
...entries,
320+
{ threadId: input.threadId, report: input.codexResetRedemption },
321+
]).pipe(Effect.as(true)),
322+
}),
323+
Layer.mock(ServerSettings.ServerSettingsService)({
324+
getSettings: Effect.succeed({
325+
autoResumeLimitedThreads: true,
326+
snoozeLimitedThreads: false,
327+
autoRedeemCodexResetCredits: {
328+
[driverInstanceId]: true,
329+
[secondInstanceId]: true,
330+
},
331+
} as never),
332+
streamChanges: Stream.never,
333+
}),
334+
Layer.mock(ProviderRegistry.ProviderRegistry)({
335+
getProviders: Effect.succeed([
336+
cachedProvider(driverInstanceId),
337+
cachedProvider(secondInstanceId),
338+
]),
339+
refreshInstance: (instanceId) =>
340+
Effect.succeed([instanceId === driverInstanceId ? refreshedDriver : refreshedSecond]),
341+
supportsAutoRedeemResetCredit: () => Effect.succeed(true),
342+
}),
343+
);
344+
const sweep = yield* UsageLimitRecoveryWorker.makeSweep.pipe(Effect.provide(layer));
345+
yield* sweep();
346+
347+
const results = yield* Ref.get(reports);
348+
const driverReport = results.find((entry) => entry.threadId === driverThread.id);
349+
const secondReport = results.find((entry) => entry.threadId === secondThread.id);
350+
assert.isDefined(driverReport);
351+
assert.isDefined(secondReport);
352+
assert.isFalse(driverReport.report.allowanceCleared);
353+
assert.isTrue(secondReport.report.allowanceCleared);
354+
assert.equal(secondReport.report.checkedAt, checkedAt);
355+
assert.deepEqual(yield* Ref.get(dispatches), [secondThread.id]);
356+
}),
357+
);
358+
108359
it.effect(
109360
"releases a first claim after a pre-send probe failure and throttles the next probe",
110361
() =>

0 commit comments

Comments
 (0)