@@ -203,157 +203,173 @@ it.effect("reports the driver's fresh redemption timestamp and credit count", ()
203203 } ) ,
204204) ;
205205
206- it . effect ( "resumes a cleared account instance after a reset while the driver remains limited" , ( ) =>
207- Effect . gen ( function * ( ) {
208- yield * TestClock . setTime ( nowMs ) ;
209- const base = failedThread ( 0 , - HOUR_MS ) ;
210- const driverInstanceId = base . providerInstanceId ;
211- const secondInstanceId = ProviderInstanceId . make ( "provider-codex-second-instance" ) ;
212- const fingerprint = "codex-account:follow-up" ;
213- const attemptedAt = DateTime . formatIso ( DateTime . makeUnsafe ( nowMs - 120_000 ) ) ;
214- const checkedAt = DateTime . formatIso ( DateTime . makeUnsafe ( nowMs - 60_000 ) ) ;
215- const threadFor = ( input : { readonly id : string ; readonly instanceId : ProviderInstanceId } ) => {
216- const threadId = ThreadId . make ( input . id ) ;
217- const runId = RunId . make ( `run:${ input . id } ` ) ;
218- return {
219- ...base ,
220- id : threadId ,
221- providerInstanceId : input . instanceId ,
222- modelSelection : { ...base . modelSelection , instanceId : input . instanceId } ,
223- latestRunId : runId ,
224- limitRecovery : {
225- runId,
226- resetAt : base . usageLimitResetAt ! ,
227- autoResume : true ,
228- codexResetRedemption : {
229- providerInstanceId : input . instanceId ,
230- redeemedByProviderInstanceId : driverInstanceId ,
231- accountFingerprint : fingerprint ,
232- idempotencyKey : "codex-auto-reset:follow-up" ,
233- attemptedAt,
234- creditsBefore : 1 ,
235- outcome : "reset" as const ,
206+ it . effect . each ( [ "limited" , "probeFailed" , "stale" ] as const ) (
207+ "resumes another instance and preserves ledger proof when the driver is %s" ,
208+ ( probe ) =>
209+ Effect . gen ( function * ( ) {
210+ yield * TestClock . setTime ( nowMs ) ;
211+ const base = failedThread ( 0 , - HOUR_MS ) ;
212+ const driverInstanceId = base . providerInstanceId ;
213+ const secondInstanceId = ProviderInstanceId . make ( "provider-codex-second-instance" ) ;
214+ const fingerprint = "codex-account:follow-up" ;
215+ const attemptedAt = DateTime . formatIso ( DateTime . makeUnsafe ( nowMs - 120_000 ) ) ;
216+ const checkedAt = DateTime . formatIso ( DateTime . makeUnsafe ( nowMs - 60_000 ) ) ;
217+ const threadFor = ( input : {
218+ readonly id : string ;
219+ readonly instanceId : ProviderInstanceId ;
220+ } ) => {
221+ const threadId = ThreadId . make ( input . id ) ;
222+ const runId = RunId . make ( `run:${ input . id } ` ) ;
223+ return {
224+ ...base ,
225+ id : threadId ,
226+ providerInstanceId : input . instanceId ,
227+ modelSelection : { ...base . modelSelection , instanceId : input . instanceId } ,
228+ latestRunId : runId ,
229+ limitRecovery : {
230+ runId,
231+ resetAt : base . usageLimitResetAt ! ,
232+ autoResume : true ,
233+ codexResetRedemption : {
234+ providerInstanceId : input . instanceId ,
235+ redeemedByProviderInstanceId : driverInstanceId ,
236+ accountFingerprint : fingerprint ,
237+ idempotencyKey : "codex-auto-reset:follow-up" ,
238+ attemptedAt,
239+ creditsBefore : 1 ,
240+ outcome : "reset" as const ,
241+ checkedAt : DateTime . formatIso ( DateTime . makeUnsafe ( nowMs - 180_000 ) ) ,
242+ allowanceCleared : false ,
243+ } ,
244+ } ,
245+ } ;
246+ } ;
247+ const driverThread = threadFor ( {
248+ id : "thread:follow-up-driver" ,
249+ instanceId : driverInstanceId ,
250+ } ) ;
251+ const secondThread = threadFor ( {
252+ id : "thread:follow-up-second" ,
253+ instanceId : secondInstanceId ,
254+ } ) ;
255+ const candidates = [ driverThread , secondThread ] ;
256+ const previousAttempt : ProjectionStore . CodexResetAttempt = {
257+ accountFingerprint : fingerprint ,
258+ resetAt : base . usageLimitResetAt ! ,
259+ idempotencyKey : "codex-auto-reset:follow-up" ,
260+ attemptedAt,
261+ outcome : "reset" ,
262+ allowanceCleared : true ,
263+ checkedAt,
264+ nextCheckAt : DateTime . formatIso ( DateTime . makeUnsafe ( nowMs - 1_000 ) ) ,
265+ } ;
266+ const ledger = yield * Ref . make ( previousAttempt ) ;
267+ const cachedProvider = ( instanceId : ProviderInstanceId ) =>
268+ ( {
269+ instanceId,
270+ driver : ProviderDriverKind . make ( "codex" ) ,
271+ usageLimits : {
272+ credentialFingerprint : fingerprint ,
236273 checkedAt : DateTime . formatIso ( DateTime . makeUnsafe ( nowMs - 180_000 ) ) ,
237- allowanceCleared : false ,
274+ windows : [ ] ,
275+ resetCredits : { availableCount : 1 } ,
238276 } ,
277+ } ) as unknown as ServerProvider ;
278+ const refreshedDriver = {
279+ ...cachedProvider ( driverInstanceId ) ,
280+ usageLimits : {
281+ credentialFingerprint : fingerprint ,
282+ checkedAt,
283+ windows : [
284+ {
285+ id : "session" ,
286+ label : "Session" ,
287+ usedPercent : 100 ,
288+ resetsAt : base . usageLimitResetAt ,
289+ } ,
290+ ] ,
291+ resetCredits : { availableCount : 1 } ,
239292 } ,
240- } ;
241- } ;
242- const driverThread = threadFor ( { id : "thread:follow-up-driver" , instanceId : driverInstanceId } ) ;
243- const secondThread = threadFor ( { id : "thread:follow-up-second" , instanceId : secondInstanceId } ) ;
244- const candidates = [ driverThread , secondThread ] ;
245- const previousAttempt : ProjectionStore . CodexResetAttempt = {
246- accountFingerprint : fingerprint ,
247- resetAt : base . usageLimitResetAt ! ,
248- idempotencyKey : "codex-auto-reset:follow-up" ,
249- attemptedAt,
250- outcome : "reset" ,
251- allowanceCleared : false ,
252- checkedAt,
253- nextCheckAt : DateTime . formatIso ( DateTime . makeUnsafe ( nowMs - 1_000 ) ) ,
254- } ;
255- const ledger = yield * Ref . make ( previousAttempt ) ;
256- const cachedProvider = ( instanceId : ProviderInstanceId ) =>
257- ( {
258- instanceId,
259- driver : ProviderDriverKind . make ( "codex" ) ,
293+ } as unknown as ServerProvider ;
294+ const refreshedSecond = {
295+ ...cachedProvider ( secondInstanceId ) ,
260296 usageLimits : {
261297 credentialFingerprint : fingerprint ,
262- checkedAt : DateTime . formatIso ( DateTime . makeUnsafe ( nowMs - 180_000 ) ) ,
298+ checkedAt,
263299 windows : [ ] ,
264300 resetCredits : { availableCount : 1 } ,
265301 } ,
266- } ) as unknown as ServerProvider ;
267- const refreshedDriver = {
268- ...cachedProvider ( driverInstanceId ) ,
269- usageLimits : {
270- credentialFingerprint : fingerprint ,
271- checkedAt,
272- windows : [
273- {
274- id : "session" ,
275- label : "Session" ,
276- usedPercent : 100 ,
277- resetsAt : base . usageLimitResetAt ,
278- } ,
279- ] ,
280- resetCredits : { availableCount : 1 } ,
281- } ,
282- } as unknown as ServerProvider ;
283- const refreshedSecond = {
284- ...cachedProvider ( secondInstanceId ) ,
285- usageLimits : {
286- credentialFingerprint : fingerprint ,
287- checkedAt,
288- windows : [ ] ,
289- resetCredits : { availableCount : 1 } ,
290- } ,
291- } as unknown as ServerProvider ;
292- const reports = yield * Ref . make <
293- ReadonlyArray < {
294- readonly threadId : ThreadId ;
295- readonly report : NonNullable < OrchestrationV2LimitRecovery [ "codexResetRedemption" ] > ;
296- } >
297- > ( [ ] ) ;
298- const dispatches = yield * Ref . make < ReadonlyArray < ThreadId > > ( [ ] ) ;
299- const layer = Layer . mergeAll (
300- Layer . succeed ( ProjectionStore . ProjectionStoreV2 , {
301- getLimitRecoveryCandidates : ( ) => Effect . succeed ( candidates ) ,
302- getLimitRecoveryWakeAt : ( ) => Effect . succeed ( nowMs + HOUR_MS ) ,
303- getCodexResetAttempt : ( ) => Ref . get ( ledger ) ,
304- updateCodexResetAttempt : ( attempt : ProjectionStore . CodexResetAttempt ) =>
305- Ref . set ( ledger , attempt ) ,
306- } as never ) ,
307- Layer . mock ( ThreadManagement . ThreadManagementService ) ( {
308- dispatch : ( command ) =>
309- command . type === "message.dispatch"
310- ? Ref . update ( dispatches , ( ids ) => [ ...ids , command . threadId ] ) . pipe (
311- Effect . as ( { sequence : 1 , storedEvents : [ ] } ) ,
312- )
313- : Effect . die ( "expected a limit continuation" ) ,
314- } ) ,
315- Layer . mock ( Orchestrator . OrchestratorV2 ) ( {
316- streamDomainEvents : Stream . never ,
317- recordCodexResetRedemption : ( input ) =>
318- Ref . update ( reports , ( entries ) => [
319- ...entries ,
320- { threadId : input . threadId , report : input . codexResetRedemption } ,
321- ] ) . pipe ( Effect . as ( true ) ) ,
322- } ) ,
323- Layer . mock ( ServerSettings . ServerSettingsService ) ( {
324- getSettings : Effect . succeed ( {
325- autoResumeLimitedThreads : true ,
326- snoozeLimitedThreads : false ,
327- autoRedeemCodexResetCredits : {
328- [ driverInstanceId ] : true ,
329- [ secondInstanceId ] : true ,
330- } ,
302+ } as unknown as ServerProvider ;
303+ const reports = yield * Ref . make <
304+ ReadonlyArray < {
305+ readonly threadId : ThreadId ;
306+ readonly report : NonNullable < OrchestrationV2LimitRecovery [ "codexResetRedemption" ] > ;
307+ } >
308+ > ( [ ] ) ;
309+ const dispatches = yield * Ref . make < ReadonlyArray < ThreadId > > ( [ ] ) ;
310+ const layer = Layer . mergeAll (
311+ Layer . succeed ( ProjectionStore . ProjectionStoreV2 , {
312+ getLimitRecoveryCandidates : ( ) => Effect . succeed ( candidates ) ,
313+ getLimitRecoveryWakeAt : ( ) => Effect . succeed ( nowMs + HOUR_MS ) ,
314+ getCodexResetAttempt : ( ) => Ref . get ( ledger ) ,
315+ updateCodexResetAttempt : ( attempt : ProjectionStore . CodexResetAttempt ) =>
316+ Ref . set ( ledger , attempt ) ,
331317 } as never ) ,
332- streamChanges : Stream . never ,
333- } ) ,
334- Layer . mock ( ProviderRegistry . ProviderRegistry ) ( {
335- getProviders : Effect . succeed ( [
336- cachedProvider ( driverInstanceId ) ,
337- cachedProvider ( secondInstanceId ) ,
338- ] ) ,
339- refreshInstance : ( instanceId ) =>
340- Effect . succeed ( [ instanceId === driverInstanceId ? refreshedDriver : refreshedSecond ] ) ,
341- supportsAutoRedeemResetCredit : ( ) => Effect . succeed ( true ) ,
342- } ) ,
343- ) ;
344- const sweep = yield * UsageLimitRecoveryWorker . makeSweep . pipe ( Effect . provide ( layer ) ) ;
345- yield * sweep ( ) ;
318+ Layer . mock ( ThreadManagement . ThreadManagementService ) ( {
319+ dispatch : ( command ) =>
320+ command . type === "message.dispatch"
321+ ? Ref . update ( dispatches , ( ids ) => [ ...ids , command . threadId ] ) . pipe (
322+ Effect . as ( { sequence : 1 , storedEvents : [ ] } ) ,
323+ )
324+ : Effect . die ( "expected a limit continuation" ) ,
325+ } ) ,
326+ Layer . mock ( Orchestrator . OrchestratorV2 ) ( {
327+ streamDomainEvents : Stream . never ,
328+ recordCodexResetRedemption : ( input ) =>
329+ Ref . update ( reports , ( entries ) => [
330+ ...entries ,
331+ { threadId : input . threadId , report : input . codexResetRedemption } ,
332+ ] ) . pipe ( Effect . as ( true ) ) ,
333+ } ) ,
334+ Layer . mock ( ServerSettings . ServerSettingsService ) ( {
335+ getSettings : Effect . succeed ( {
336+ autoResumeLimitedThreads : true ,
337+ snoozeLimitedThreads : false ,
338+ autoRedeemCodexResetCredits : {
339+ [ driverInstanceId ] : true ,
340+ [ secondInstanceId ] : true ,
341+ } ,
342+ } as never ) ,
343+ streamChanges : Stream . never ,
344+ } ) ,
345+ Layer . mock ( ProviderRegistry . ProviderRegistry ) ( {
346+ getProviders : Effect . succeed ( [
347+ cachedProvider ( driverInstanceId ) ,
348+ cachedProvider ( secondInstanceId ) ,
349+ ] ) ,
350+ refreshInstance : ( instanceId ) =>
351+ instanceId === driverInstanceId
352+ ? probe === "probeFailed"
353+ ? Effect . die ( "fixture probe failed" )
354+ : Effect . succeed ( [ probe === "stale" ? cachedProvider ( instanceId ) : refreshedDriver ] )
355+ : Effect . succeed ( [ refreshedSecond ] ) ,
356+ supportsAutoRedeemResetCredit : ( ) => Effect . succeed ( true ) ,
357+ } ) ,
358+ ) ;
359+ const sweep = yield * UsageLimitRecoveryWorker . makeSweep . pipe ( Effect . provide ( layer ) ) ;
360+ yield * sweep ( ) ;
346361
347- const results = yield * Ref . get ( reports ) ;
348- const driverReport = results . find ( ( entry ) => entry . threadId === driverThread . id ) ;
349- const secondReport = results . find ( ( entry ) => entry . threadId === secondThread . id ) ;
350- assert . isDefined ( driverReport ) ;
351- assert . isDefined ( secondReport ) ;
352- assert . isFalse ( driverReport . report . allowanceCleared ) ;
353- assert . isTrue ( secondReport . report . allowanceCleared ) ;
354- assert . equal ( secondReport . report . checkedAt , checkedAt ) ;
355- assert . deepEqual ( yield * Ref . get ( dispatches ) , [ secondThread . id ] ) ;
356- } ) ,
362+ const results = yield * Ref . get ( reports ) ;
363+ const driverReport = results . find ( ( entry ) => entry . threadId === driverThread . id ) ;
364+ const secondReport = results . find ( ( entry ) => entry . threadId === secondThread . id ) ;
365+ assert . isDefined ( driverReport ) ;
366+ assert . isDefined ( secondReport ) ;
367+ assert . isFalse ( driverReport . report . allowanceCleared ) ;
368+ assert . isTrue ( secondReport . report . allowanceCleared ) ;
369+ assert . equal ( secondReport . report . checkedAt , checkedAt ) ;
370+ assert . deepEqual ( yield * Ref . get ( dispatches ) , [ secondThread . id ] ) ;
371+ assert . equal ( ( yield * Ref . get ( ledger ) ) . allowanceCleared , probe !== "limited" ) ;
372+ } ) ,
357373) ;
358374
359375it . effect (
0 commit comments