@@ -23,14 +23,34 @@ import * as Layer from "effect/Layer";
2323import * as SqlClient from "effect/sql/SqlClient" ;
2424
2525import * as SqlitePersistence from "../persistence/Sqlite.ts" ;
26+ import * as ServerSettings from "../serverSettings.ts" ;
2627import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts" ;
2728import * as EffectOutbox from "./EffectOutbox.ts" ;
2829import * as ProjectionStore from "./ProjectionStore.ts" ;
30+ import * as EventSink from "./EventSink.ts" ;
31+ import * as EventStore from "./EventStore.ts" ;
32+ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator" ;
33+ import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts" ;
2934import { restartContinuationRun } from "./RestartContinuation.ts" ;
3035
3136const layerTest = Layer . mergeAll ( ProjectionStore . layer , EffectOutbox . layer ) . pipe (
3237 Layer . provideMerge ( SqlitePersistence . layerMemory ) ,
3338) ;
39+ const layerRecoveryTest = ProviderRuntimeRecovery . layer . pipe (
40+ Layer . provideMerge (
41+ EventSink . layer . pipe (
42+ Layer . provideMerge (
43+ Layer . mergeAll (
44+ EventStore . layer ,
45+ ProjectionStore . layer ,
46+ EffectOutbox . layer ,
47+ IdAllocator . layer ,
48+ ) . pipe ( Layer . provideMerge ( SqlitePersistence . layerMemory ) ) ,
49+ ) ,
50+ ) ,
51+ ) ,
52+ Layer . provide ( ServerSettings . layerTest ( ) ) ,
53+ ) ;
3454const providerInstanceId = ProviderInstanceId . make ( "codex" ) ;
3555const modelSelection = { instanceId : providerInstanceId , model : "gpt-5.4" } ;
3656const driver = ProviderDriverKind . make ( "codex" ) ;
@@ -113,6 +133,89 @@ const createRun = Effect.fn(function* (
113133 return runId ;
114134} ) ;
115135
136+ it . effect . each ( [ "startup" , "shutdown" ] as const ) (
137+ "%s cancels buffered provider wakes left behind by completed turns" ,
138+ ( trigger ) =>
139+ Effect . gen ( function * ( ) {
140+ const projections = yield * ProjectionStore . ProjectionStoreV2 ;
141+ const recovery = yield * ProviderRuntimeRecovery . ProviderRuntimeRecoveryService ;
142+ const now = yield * DateTime . now ;
143+ const bufferedThreads = [ ] as ThreadId [ ] ;
144+ const durableThreads = [ ] as ThreadId [ ] ;
145+ for ( const kind of [
146+ "buffered" ,
147+ "held-buffered" ,
148+ "user" ,
149+ "notification" ,
150+ "delegated" ,
151+ ] as const ) {
152+ const threadId = yield * createThread ( `wake:${ trigger } :${ kind } ` ) ;
153+ const buffered = kind === "buffered" || kind === "held-buffered" ;
154+ ( buffered ? bufferedThreads : durableThreads ) . push ( threadId ) ;
155+ const messageId = MessageId . make ( `message:${ threadId } ` ) ;
156+ const runId = yield * createRun ( threadId , "queued" , {
157+ userMessageId : messageId ,
158+ startedAt : null ,
159+ queueHeld : kind !== "buffered" ,
160+ queuePosition : 1 ,
161+ } ) ;
162+ yield * createRun ( threadId , "completed" , { ordinal : 2 } ) ;
163+ yield * projections . apply ( {
164+ id : EventId . make ( `message:${ threadId } ` ) ,
165+ type : "message.updated" ,
166+ threadId,
167+ runId,
168+ occurredAt : now ,
169+ payload : {
170+ id : messageId ,
171+ threadId,
172+ runId,
173+ nodeId : null ,
174+ role : "user" ,
175+ text : "Queued work" ,
176+ attachments : [ ] ,
177+ streaming : false ,
178+ createdAt : now ,
179+ updatedAt : now ,
180+ createdBy : kind === "user" ? "user" : "agent" ,
181+ creationSource : buffered ? "provider" : "server" ,
182+ ...( kind === "user" || kind === "delegated"
183+ ? { }
184+ : {
185+ notification : {
186+ source : { kind : "background_task" as const } ,
187+ outcome : "updated" as const ,
188+ summary : "Background activity updated" ,
189+ } ,
190+ } ) ,
191+ ...( kind === "delegated"
192+ ? { delegatedCompletion : { parentRunId : runId , generation : 1 , taskIds : [ ] } }
193+ : { } ) ,
194+ } ,
195+ } ) ;
196+ }
197+ assert . deepEqual (
198+ new Set ( yield * projections . getRecoveryThreadIds ( "runtime" ) ) ,
199+ new Set ( bufferedThreads ) ,
200+ ) ;
201+ const summary = yield * recovery . reconcile ( trigger ) ;
202+ assert . equal ( summary . terminalizedRuns , 2 ) ;
203+ for ( const threadId of bufferedThreads ) {
204+ const projection = yield * projections . getThreadProjection ( threadId ) ;
205+ assert . equal ( projection . runs [ 0 ] ?. status , "cancelled" ) ;
206+ assert . isNotNull ( projection . runs [ 0 ] ?. completedAt ) ;
207+ assert . isNull ( projection . runs [ 0 ] ?. queuePosition ) ;
208+ assert . equal ( projection . runs [ 1 ] ?. status , "completed" ) ;
209+ }
210+ for ( const threadId of durableThreads ) {
211+ const projection = yield * projections . getThreadProjection ( threadId ) ;
212+ assert . equal ( projection . runs [ 0 ] ?. status , "queued" ) ;
213+ assert . isTrue ( projection . runs [ 0 ] ?. queueHeld ) ;
214+ }
215+ assert . deepEqual ( yield * projections . getRecoveryThreadIds ( "runtime" ) , [ ] ) ;
216+ } ) . pipe ( Effect . provide ( layerRecoveryTest ) ) ,
217+ ) ;
218+
116219it . effect ( "selects unfinished recovery work without reading settled thread histories" , ( ) =>
117220 Effect . gen ( function * ( ) {
118221 const projections = yield * ProjectionStore . ProjectionStoreV2 ;
0 commit comments