|
4 | 4 | ProviderDriverKind, |
5 | 5 | RunId, |
6 | 6 | ThreadId, |
| 7 | + type OrchestrationV2LimitRecovery, |
7 | 8 | type ServerProvider, |
8 | 9 | type OrchestrationV2DomainEvent, |
9 | 10 | } from "@t3tools/contracts"; |
@@ -105,6 +106,256 @@ it.effect("backs off a pending account attempt across repeated sweep ticks", () |
105 | 106 | }), |
106 | 107 | ); |
107 | 108 |
|
| 109 | +it.effect("reports the driver's fresh redemption timestamp and credit count", () => |
| 110 | + Effect.gen(function* () { |
| 111 | + yield* TestClock.setTime(nowMs); |
| 112 | + const base = failedThread(0, -HOUR_MS); |
| 113 | + const fingerprint = "codex-account:fresh-result"; |
| 114 | + const checkedAt = DateTime.formatIso(DateTime.makeUnsafe(nowMs + 1_000)); |
| 115 | + const thread = { |
| 116 | + ...base, |
| 117 | + limitRecovery: { |
| 118 | + runId: base.latestRunId!, |
| 119 | + resetAt: base.usageLimitResetAt!, |
| 120 | + autoResume: true, |
| 121 | + }, |
| 122 | + }; |
| 123 | + const provider = { |
| 124 | + instanceId: base.providerInstanceId, |
| 125 | + driver: ProviderDriverKind.make("codex"), |
| 126 | + usageLimits: { |
| 127 | + credentialFingerprint: fingerprint, |
| 128 | + checkedAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - 60_000)), |
| 129 | + windows: [], |
| 130 | + resetCredits: { availableCount: 1 }, |
| 131 | + }, |
| 132 | + } as unknown as ServerProvider; |
| 133 | + const ledger = yield* Ref.make<ProjectionStore.CodexResetAttempt | null>(null); |
| 134 | + const reports = yield* Ref.make< |
| 135 | + ReadonlyArray<{ |
| 136 | + readonly threadId: ThreadId; |
| 137 | + readonly report: NonNullable<OrchestrationV2LimitRecovery["codexResetRedemption"]>; |
| 138 | + }> |
| 139 | + >([]); |
| 140 | + const dispatches = yield* Ref.make<ReadonlyArray<ThreadId>>([]); |
| 141 | + const layer = Layer.mergeAll( |
| 142 | + Layer.succeed(ProjectionStore.ProjectionStoreV2, { |
| 143 | + getLimitRecoveryCandidates: () => Effect.succeed([thread]), |
| 144 | + getLimitRecoveryWakeAt: () => Effect.succeed(nowMs + HOUR_MS), |
| 145 | + getCodexResetAttempt: () => Ref.get(ledger), |
| 146 | + claimCodexResetAttempt: (attempt: ProjectionStore.CodexResetAttempt) => |
| 147 | + Ref.set(ledger, attempt).pipe(Effect.as(true)), |
| 148 | + updateCodexResetAttempt: (attempt: ProjectionStore.CodexResetAttempt) => |
| 149 | + Ref.set(ledger, attempt), |
| 150 | + releaseCodexResetAttempt: () => Ref.set(ledger, null), |
| 151 | + } as never), |
| 152 | + Layer.mock(ThreadManagement.ThreadManagementService)({ |
| 153 | + dispatch: (command) => |
| 154 | + command.type === "message.dispatch" |
| 155 | + ? Ref.update(dispatches, (ids) => [...ids, command.threadId]).pipe( |
| 156 | + Effect.as({ sequence: 1, storedEvents: [] }), |
| 157 | + ) |
| 158 | + : Effect.die("expected a limit continuation"), |
| 159 | + }), |
| 160 | + Layer.mock(Orchestrator.OrchestratorV2)({ |
| 161 | + streamDomainEvents: Stream.never, |
| 162 | + recordCodexResetRedemption: (input) => |
| 163 | + Ref.update(reports, (entries) => [ |
| 164 | + ...entries, |
| 165 | + { threadId: input.threadId, report: input.codexResetRedemption }, |
| 166 | + ]).pipe(Effect.as(true)), |
| 167 | + }), |
| 168 | + Layer.mock(ServerSettings.ServerSettingsService)({ |
| 169 | + getSettings: Effect.succeed({ |
| 170 | + autoResumeLimitedThreads: true, |
| 171 | + snoozeLimitedThreads: false, |
| 172 | + autoRedeemCodexResetCredits: { [base.providerInstanceId]: true }, |
| 173 | + } as never), |
| 174 | + streamChanges: Stream.never, |
| 175 | + }), |
| 176 | + Layer.mock(ProviderRegistry.ProviderRegistry)({ |
| 177 | + getProviders: Effect.succeed([provider]), |
| 178 | + refreshInstance: () => Effect.succeed([provider]), |
| 179 | + supportsAutoRedeemResetCredit: () => Effect.succeed(true), |
| 180 | + autoRedeemResetCredit: (input) => |
| 181 | + Effect.succeed({ |
| 182 | + status: "sent", |
| 183 | + accountFingerprint: fingerprint, |
| 184 | + creditsBefore: 4, |
| 185 | + outcome: "reset", |
| 186 | + checkedAt, |
| 187 | + allowanceCleared: true, |
| 188 | + idempotencyKey: input.idempotencyKey, |
| 189 | + } as const), |
| 190 | + }), |
| 191 | + ); |
| 192 | + const sweep = yield* UsageLimitRecoveryWorker.makeSweep.pipe(Effect.provide(layer)); |
| 193 | + yield* sweep(); |
| 194 | + |
| 195 | + const driverReport = (yield* Ref.get(reports)).find( |
| 196 | + (entry) => entry.threadId === base.id && entry.report.outcome === "reset", |
| 197 | + ); |
| 198 | + assert.isDefined(driverReport); |
| 199 | + assert.equal(driverReport.report.checkedAt, checkedAt); |
| 200 | + assert.equal(driverReport.report.creditsBefore, 4); |
| 201 | + assert.isTrue(driverReport.report.allowanceCleared); |
| 202 | + assert.deepEqual(yield* Ref.get(dispatches), [base.id]); |
| 203 | + }), |
| 204 | +); |
| 205 | + |
| 206 | +it.effect("resumes a cleared account instance after a reset while the driver remains limited", () => |
| 207 | + Effect.gen(function* () { |
| 208 | + yield* TestClock.setTime(nowMs); |
| 209 | + const base = failedThread(0, -HOUR_MS); |
| 210 | + const driverInstanceId = base.providerInstanceId; |
| 211 | + const secondInstanceId = ProviderInstanceId.make("provider-codex-second-instance"); |
| 212 | + const fingerprint = "codex-account:follow-up"; |
| 213 | + const attemptedAt = DateTime.formatIso(DateTime.makeUnsafe(nowMs - 120_000)); |
| 214 | + const checkedAt = DateTime.formatIso(DateTime.makeUnsafe(nowMs - 60_000)); |
| 215 | + const threadFor = (input: { readonly id: string; readonly instanceId: ProviderInstanceId }) => { |
| 216 | + const threadId = ThreadId.make(input.id); |
| 217 | + const runId = RunId.make(`run:${input.id}`); |
| 218 | + return { |
| 219 | + ...base, |
| 220 | + id: threadId, |
| 221 | + providerInstanceId: input.instanceId, |
| 222 | + modelSelection: { ...base.modelSelection, instanceId: input.instanceId }, |
| 223 | + latestRunId: runId, |
| 224 | + limitRecovery: { |
| 225 | + runId, |
| 226 | + resetAt: base.usageLimitResetAt!, |
| 227 | + autoResume: true, |
| 228 | + codexResetRedemption: { |
| 229 | + providerInstanceId: input.instanceId, |
| 230 | + redeemedByProviderInstanceId: driverInstanceId, |
| 231 | + accountFingerprint: fingerprint, |
| 232 | + idempotencyKey: "codex-auto-reset:follow-up", |
| 233 | + attemptedAt, |
| 234 | + creditsBefore: 1, |
| 235 | + outcome: "reset" as const, |
| 236 | + checkedAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - 180_000)), |
| 237 | + allowanceCleared: false, |
| 238 | + }, |
| 239 | + }, |
| 240 | + }; |
| 241 | + }; |
| 242 | + const driverThread = threadFor({ id: "thread:follow-up-driver", instanceId: driverInstanceId }); |
| 243 | + const secondThread = threadFor({ id: "thread:follow-up-second", instanceId: secondInstanceId }); |
| 244 | + const candidates = [driverThread, secondThread]; |
| 245 | + const previousAttempt: ProjectionStore.CodexResetAttempt = { |
| 246 | + accountFingerprint: fingerprint, |
| 247 | + resetAt: base.usageLimitResetAt!, |
| 248 | + idempotencyKey: "codex-auto-reset:follow-up", |
| 249 | + attemptedAt, |
| 250 | + outcome: "reset", |
| 251 | + allowanceCleared: false, |
| 252 | + checkedAt, |
| 253 | + nextCheckAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - 1_000)), |
| 254 | + }; |
| 255 | + const ledger = yield* Ref.make(previousAttempt); |
| 256 | + const cachedProvider = (instanceId: ProviderInstanceId) => |
| 257 | + ({ |
| 258 | + instanceId, |
| 259 | + driver: ProviderDriverKind.make("codex"), |
| 260 | + usageLimits: { |
| 261 | + credentialFingerprint: fingerprint, |
| 262 | + checkedAt: DateTime.formatIso(DateTime.makeUnsafe(nowMs - 180_000)), |
| 263 | + windows: [], |
| 264 | + resetCredits: { availableCount: 1 }, |
| 265 | + }, |
| 266 | + }) as unknown as ServerProvider; |
| 267 | + const refreshedDriver = { |
| 268 | + ...cachedProvider(driverInstanceId), |
| 269 | + usageLimits: { |
| 270 | + credentialFingerprint: fingerprint, |
| 271 | + checkedAt, |
| 272 | + windows: [ |
| 273 | + { |
| 274 | + id: "session", |
| 275 | + label: "Session", |
| 276 | + usedPercent: 100, |
| 277 | + resetsAt: base.usageLimitResetAt, |
| 278 | + }, |
| 279 | + ], |
| 280 | + resetCredits: { availableCount: 1 }, |
| 281 | + }, |
| 282 | + } as unknown as ServerProvider; |
| 283 | + const refreshedSecond = { |
| 284 | + ...cachedProvider(secondInstanceId), |
| 285 | + usageLimits: { |
| 286 | + credentialFingerprint: fingerprint, |
| 287 | + checkedAt, |
| 288 | + windows: [], |
| 289 | + resetCredits: { availableCount: 1 }, |
| 290 | + }, |
| 291 | + } as unknown as ServerProvider; |
| 292 | + const reports = yield* Ref.make< |
| 293 | + ReadonlyArray<{ |
| 294 | + readonly threadId: ThreadId; |
| 295 | + readonly report: NonNullable<OrchestrationV2LimitRecovery["codexResetRedemption"]>; |
| 296 | + }> |
| 297 | + >([]); |
| 298 | + const dispatches = yield* Ref.make<ReadonlyArray<ThreadId>>([]); |
| 299 | + const layer = Layer.mergeAll( |
| 300 | + Layer.succeed(ProjectionStore.ProjectionStoreV2, { |
| 301 | + getLimitRecoveryCandidates: () => Effect.succeed(candidates), |
| 302 | + getLimitRecoveryWakeAt: () => Effect.succeed(nowMs + HOUR_MS), |
| 303 | + getCodexResetAttempt: () => Ref.get(ledger), |
| 304 | + updateCodexResetAttempt: (attempt: ProjectionStore.CodexResetAttempt) => |
| 305 | + Ref.set(ledger, attempt), |
| 306 | + } as never), |
| 307 | + Layer.mock(ThreadManagement.ThreadManagementService)({ |
| 308 | + dispatch: (command) => |
| 309 | + command.type === "message.dispatch" |
| 310 | + ? Ref.update(dispatches, (ids) => [...ids, command.threadId]).pipe( |
| 311 | + Effect.as({ sequence: 1, storedEvents: [] }), |
| 312 | + ) |
| 313 | + : Effect.die("expected a limit continuation"), |
| 314 | + }), |
| 315 | + Layer.mock(Orchestrator.OrchestratorV2)({ |
| 316 | + streamDomainEvents: Stream.never, |
| 317 | + recordCodexResetRedemption: (input) => |
| 318 | + Ref.update(reports, (entries) => [ |
| 319 | + ...entries, |
| 320 | + { threadId: input.threadId, report: input.codexResetRedemption }, |
| 321 | + ]).pipe(Effect.as(true)), |
| 322 | + }), |
| 323 | + Layer.mock(ServerSettings.ServerSettingsService)({ |
| 324 | + getSettings: Effect.succeed({ |
| 325 | + autoResumeLimitedThreads: true, |
| 326 | + snoozeLimitedThreads: false, |
| 327 | + autoRedeemCodexResetCredits: { |
| 328 | + [driverInstanceId]: true, |
| 329 | + [secondInstanceId]: true, |
| 330 | + }, |
| 331 | + } as never), |
| 332 | + streamChanges: Stream.never, |
| 333 | + }), |
| 334 | + Layer.mock(ProviderRegistry.ProviderRegistry)({ |
| 335 | + getProviders: Effect.succeed([ |
| 336 | + cachedProvider(driverInstanceId), |
| 337 | + cachedProvider(secondInstanceId), |
| 338 | + ]), |
| 339 | + refreshInstance: (instanceId) => |
| 340 | + Effect.succeed([instanceId === driverInstanceId ? refreshedDriver : refreshedSecond]), |
| 341 | + supportsAutoRedeemResetCredit: () => Effect.succeed(true), |
| 342 | + }), |
| 343 | + ); |
| 344 | + const sweep = yield* UsageLimitRecoveryWorker.makeSweep.pipe(Effect.provide(layer)); |
| 345 | + yield* sweep(); |
| 346 | + |
| 347 | + const results = yield* Ref.get(reports); |
| 348 | + const driverReport = results.find((entry) => entry.threadId === driverThread.id); |
| 349 | + const secondReport = results.find((entry) => entry.threadId === secondThread.id); |
| 350 | + assert.isDefined(driverReport); |
| 351 | + assert.isDefined(secondReport); |
| 352 | + assert.isFalse(driverReport.report.allowanceCleared); |
| 353 | + assert.isTrue(secondReport.report.allowanceCleared); |
| 354 | + assert.equal(secondReport.report.checkedAt, checkedAt); |
| 355 | + assert.deepEqual(yield* Ref.get(dispatches), [secondThread.id]); |
| 356 | + }), |
| 357 | +); |
| 358 | + |
108 | 359 | it.effect( |
109 | 360 | "releases a first claim after a pre-send probe failure and throttles the next probe", |
110 | 361 | () => |
|
0 commit comments