@@ -40,12 +40,15 @@ import * as Ref from "effect/Ref";
4040import * as Scheduler from "effect/Scheduler" ;
4141import * as Schema from "effect/Schema" ;
4242import * as Stream from "effect/Stream" ;
43+ import * as TestClock from "effect/testing/TestClock" ;
4344import * as Tracer from "effect/Tracer" ;
4445import * as SqlClient from "effect/sql/SqlClient" ;
4546import * as Statement from "effect/sql/Statement" ;
4647
4748import { LIVE_STREAM_MAX_ITEMS , LiveStreamBufferError } from "./LiveStreamBudget.ts" ;
49+ import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts" ;
4850import * as SqlitePersistence from "../persistence/Sqlite.ts" ;
51+ import { subscribeOrchestrationV2Thread } from "../ws.ts" ;
4952import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts" ;
5053import * as CommandReceiptStore from "./CommandReceiptStore.ts" ;
5154import * as EffectOutbox from "./EffectOutbox.ts" ;
@@ -55,6 +58,7 @@ import * as EventStore from "./EventStore.ts";
5558import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator" ;
5659import * as ProjectionMaintenance from "./ProjectionMaintenance.ts" ;
5760import * as ProjectionStore from "./ProjectionStore.ts" ;
61+ import * as ThreadManagement from "./ThreadManagementService.ts" ;
5862import * as ProjectStore from "./ProjectStore.ts" ;
5963import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts" ;
6064import * as TurnItemPositionStore from "./TurnItemPositionStore.ts" ;
@@ -219,6 +223,263 @@ it.effect("rebuilds event history one bounded page at a time", () =>
219223 } ) . pipe ( Effect . provide ( layerTest ) ) ,
220224) ;
221225
226+ // Own database: advancing the clock would also expire other tests' history.
227+ it . effect (
228+ "retention prunes expired history to anchors and old cursors resume from a snapshot" ,
229+ ( ) =>
230+ Effect . gen ( function * ( ) {
231+ const eventSink = yield * EventSink . EventSinkV2 ;
232+ const maintenance = yield * ProjectionMaintenance . ProjectionMaintenanceV2 ;
233+ const projections = yield * ProjectionStore . ProjectionStoreV2 ;
234+ const sql = yield * SqlClient . SqlClient ;
235+ const createdAt = yield * DateTime . now ;
236+ const threadId = ThreadId . make ( "thread:retention" ) ;
237+ const thread = makeThread ( threadId , createdAt ) ;
238+ const run : OrchestrationV2Run = {
239+ id : RunId . make ( "run:retention" ) ,
240+ threadId,
241+ ordinal : 1 ,
242+ providerInstanceId,
243+ modelSelection,
244+ providerThreadId : null ,
245+ userMessageId : MessageId . make ( "message:retention-1" ) ,
246+ rootNodeId : null ,
247+ activeAttemptId : RunAttemptId . make ( "run-attempt:retention" ) ,
248+ status : "running" ,
249+ queuePosition : null ,
250+ requestedAt : createdAt ,
251+ startedAt : createdAt ,
252+ completedAt : null ,
253+ checkpointId : null ,
254+ contextHandoffId : null ,
255+ } ;
256+ const runEvent = ( suffix : string ) : OrchestrationV2DomainEvent => ( {
257+ id : EventId . make ( `event:retention:${ suffix } ` ) ,
258+ type : "run.updated" ,
259+ threadId,
260+ runId : run . id ,
261+ occurredAt : createdAt ,
262+ payload : run ,
263+ } ) ;
264+ const messageEvent = (
265+ suffix : string ,
266+ occurredAt : DateTime . Utc = createdAt ,
267+ ) : OrchestrationV2DomainEvent => ( {
268+ id : EventId . make ( `event:retention:${ suffix } ` ) ,
269+ type : "message.updated" ,
270+ threadId,
271+ occurredAt,
272+ payload : {
273+ createdBy : "user" ,
274+ creationSource : "web" ,
275+ id : MessageId . make ( `message:retention-${ suffix } ` ) ,
276+ threadId,
277+ runId : null ,
278+ nodeId : null ,
279+ role : "user" ,
280+ text : suffix ,
281+ attachments : [ ] ,
282+ streaming : false ,
283+ createdAt : occurredAt ,
284+ updatedAt : occurredAt ,
285+ } ,
286+ } ) ;
287+
288+ // Raw rows keep these fixtures out of projections: retention never decodes them.
289+ const insertRaw = ( streamId : string , version : number , payload : object ) => sql `
290+ INSERT INTO orchestration_events (
291+ event_id, aggregate_kind, stream_id, stream_version, event_type,
292+ occurred_at, actor_kind, payload_json, metadata_json, application_event_version
293+ ) VALUES (
294+ ${ `event:${ streamId } :${ version } ` } , 'thread', ${ streamId } , ${ version } , 'turn-item.updated',
295+ ${ DateTime . formatIso ( createdAt ) } , 'server', ${ JSON . stringify ( payload ) } , '{}', 2
296+ )
297+ ` ;
298+ // Slimmed payloads restore their prompt from this row.
299+ yield * insertRaw ( "thread:retention-prompt" , 0 , { id : "item" , prompt : "full prompt" } ) ;
300+ yield * insertRaw ( "thread:retention-prompt" , 1 , { id : "item" , prompt : "" , promptSequence : 1 } ) ;
301+ yield * insertRaw ( "thread:retention-prompt" , 2 , { id : "item" , prompt : "" , promptSequence : 1 } ) ;
302+ // The transcript import still reads its imported event IDs.
303+ yield * insertRaw ( "thread:retention-importing" , 0 , { id : "imported-1" } ) ;
304+ yield * insertRaw ( "thread:retention-importing" , 1 , { id : "imported-2" } ) ;
305+ yield * sql `
306+ INSERT INTO orchestration_v2_legacy_imports (
307+ thread_id, source_updated_at, shell_imported_at, transcript_imported_at,
308+ imported_message_count, last_error
309+ ) VALUES (
310+ 'thread:retention-importing', ${ DateTime . formatIso ( createdAt ) } , ${ DateTime . formatIso ( createdAt ) } , NULL, 0, NULL
311+ )
312+ ` ;
313+ const deletedThread = makeThread ( ThreadId . make ( "thread:retention-deleted" ) , createdAt ) ;
314+ const deletedCreated = threadCreatedEvent ( {
315+ id : "event:retention:deleted-created" ,
316+ thread : deletedThread ,
317+ now : createdAt ,
318+ } ) ;
319+ yield * eventSink . write ( {
320+ events : [
321+ deletedCreated ,
322+ { ...messageEvent ( "deleted-thread-message" ) , threadId : deletedThread . id } ,
323+ ] ,
324+ } ) ;
325+
326+ yield * eventSink . write ( {
327+ events : [
328+ threadCreatedEvent ( { id : "event:retention:created" , thread, now : createdAt } ) ,
329+ runEvent ( "first-running" ) ,
330+ messageEvent ( "expired-1" ) ,
331+ runEvent ( "later-running" ) ,
332+ messageEvent ( "expired-2" ) ,
333+ ] ,
334+ } ) ;
335+ // A retry of this command still reads its result events.
336+ yield * eventSink . commitCommand ( {
337+ commandId : CommandId . make ( "command:retention" ) ,
338+ threadId,
339+ commandType : "thread.send" ,
340+ acceptedAt : createdAt ,
341+ events : [ messageEvent ( "receipt" ) ] ,
342+ effects : [ ] ,
343+ } ) ;
344+ yield * TestClock . adjust ( Duration . hours ( 25 ) ) ;
345+ const recentAt = yield * DateTime . now ;
346+ yield * eventSink . write ( { events : [ messageEvent ( "recent" , recentAt ) ] } ) ;
347+ yield * eventSink . write ( {
348+ events : [
349+ {
350+ id : EventId . make ( "event:retention:deleted" ) ,
351+ type : "thread.deleted" ,
352+ threadId : deletedThread . id ,
353+ providerInstanceId,
354+ occurredAt : recentAt ,
355+ payload : { ...deletedThread , deletedAt : recentAt } ,
356+ } ,
357+ ] ,
358+ } ) ;
359+ const sequenceOf = ( suffix : string ) =>
360+ sql < { readonly sequence : number } > `
361+ SELECT sequence FROM orchestration_events
362+ WHERE event_id = ${ `event:retention:${ suffix } ` }
363+ ` . pipe ( Effect . map ( ( rows ) => rows [ 0 ] ! . sequence ) ) ;
364+ const oldCursor = yield * sequenceOf ( "first-running" ) ;
365+ const lastExpired = yield * sequenceOf ( "expired-2" ) ;
366+ const beforeRetention = yield * projections . getThreadProjection ( threadId ) ;
367+
368+ assert . equal ( ( yield * maintenance . compactEventStore ) . deletedEventCount , 4 ) ;
369+ const survivors = yield * sql < { readonly event_id : string } > `
370+ SELECT event_id FROM orchestration_events
371+ WHERE stream_id IN ('thread:retention-prompt', 'thread:retention-importing')
372+ ORDER BY sequence
373+ ` ;
374+ assert . deepEqual (
375+ survivors . map ( ( row ) => row . event_id ) ,
376+ [
377+ "event:thread:retention-prompt:0" ,
378+ "event:thread:retention-prompt:2" ,
379+ "event:thread:retention-importing:0" ,
380+ "event:thread:retention-importing:1" ,
381+ ] ,
382+ ) ;
383+ const remaining = yield * sql < { readonly event_id : string } > `
384+ SELECT event_id FROM orchestration_events WHERE stream_id = ${ threadId } ORDER BY sequence
385+ ` ;
386+ assert . deepEqual (
387+ remaining . map ( ( row ) => row . event_id ) ,
388+ [
389+ "event:retention:created" ,
390+ "event:retention:first-running" ,
391+ "event:retention:later-running" ,
392+ "event:retention:receipt" ,
393+ "event:retention:recent" ,
394+ ] ,
395+ ) ;
396+ const floors = yield * sql < { readonly replay_floor : number } > `
397+ SELECT replay_floor FROM orchestration_v2_event_retention WHERE thread_id = ${ threadId }
398+ ` ;
399+ assert . deepEqual (
400+ floors . map ( ( row ) => row . replay_floor ) ,
401+ [ lastExpired ] ,
402+ ) ;
403+ assert . deepEqual ( yield * projections . getThreadProjection ( threadId ) , beforeRetention ) ;
404+ assert . isTrue ( ( yield * maintenance . verify ) . valid ) ;
405+ assert . equal ( ( yield * Effect . exit ( maintenance . rebuild ) ) . _tag , "Failure" ) ;
406+ assert . deepEqual ( yield * projections . getThreadProjection ( threadId ) , beforeRetention ) ;
407+
408+ const resume = ( afterSequence : number , resumedThreadId = threadId ) =>
409+ subscribeOrchestrationV2Thread ( {
410+ threadId : resumedThreadId ,
411+ afterSequence,
412+ requestCompletionMarker : true ,
413+ } ) . pipe (
414+ Effect . flatMap ( ( stream ) =>
415+ stream . pipe (
416+ Stream . takeUntil ( ( item ) => item . kind === "synchronized" ) ,
417+ Stream . map ( ( item ) => ( item . kind === "event" ? item . sequence : item . kind ) ) ,
418+ Stream . runCollect ,
419+ ) ,
420+ ) ,
421+ Effect . provideService ( ThreadManagement . ThreadManagementService , {
422+ ensureLegacyTranscript : ( ) => Effect . void ,
423+ getThreadShell : projections . getThreadShell ,
424+ getThreadSnapshot : projections . getThreadSnapshot ,
425+ streamStoredEventsFrom : ( ) => Stream . never ,
426+ } as unknown as ThreadManagement . ThreadManagementService [ "Service" ] ) ,
427+ Effect . provide ( OrchestrationEventStore . layer ) ,
428+ ) ;
429+ // Below the floor the replay would skip deleted events, so the client gets a snapshot.
430+ assert . deepEqual ( Array . from ( yield * resume ( oldCursor ) ) , [ "snapshot" , "synchronized" ] ) ;
431+ assert . deepEqual ( Array . from ( yield * resume ( lastExpired ) ) , [
432+ yield * sequenceOf ( "receipt" ) ,
433+ yield * sequenceOf ( "recent" ) ,
434+ "synchronized" ,
435+ ] ) ;
436+ // A deleted thread has no snapshot; the client learns of the deletion from its tombstone.
437+ assert . deepEqual (
438+ Array . from ( yield * resume ( yield * sequenceOf ( "deleted-created" ) , deletedThread . id ) ) ,
439+ [ yield * sequenceOf ( "deleted" ) , "synchronized" ] ,
440+ ) ;
441+ } ) . pipe ( Effect . provide ( layerTest ) ) ,
442+ ) ;
443+
444+ // A newer CLI's doctor can open a database whose server never ran migration 70.
445+ it . effect ( "retention stays off without the replay floor table" , ( ) =>
446+ Effect . gen ( function * ( ) {
447+ const eventSink = yield * EventSink . EventSinkV2 ;
448+ const maintenance = yield * ProjectionMaintenance . ProjectionMaintenanceV2 ;
449+ const sql = yield * SqlClient . SqlClient ;
450+ const now = yield * DateTime . now ;
451+ const thread = makeThread ( ThreadId . make ( "thread:retention-pre-70" ) , now ) ;
452+ yield * eventSink . write ( {
453+ events : [
454+ threadCreatedEvent ( { id : "event:pre-70:created" , thread, now } ) ,
455+ ...[ "a" , "b" ] . map ( ( suffix ) : OrchestrationV2DomainEvent => ( {
456+ id : EventId . make ( `event:pre-70:${ suffix } ` ) ,
457+ type : "message.updated" ,
458+ threadId : thread . id ,
459+ occurredAt : now ,
460+ payload : {
461+ createdBy : "user" ,
462+ creationSource : "web" ,
463+ id : MessageId . make ( `message:pre-70-${ suffix } ` ) ,
464+ threadId : thread . id ,
465+ runId : null ,
466+ nodeId : null ,
467+ role : "user" ,
468+ text : suffix ,
469+ attachments : [ ] ,
470+ streaming : false ,
471+ createdAt : now ,
472+ updatedAt : now ,
473+ } ,
474+ } ) ) ,
475+ ] ,
476+ } ) ;
477+ yield * sql `DROP TABLE orchestration_v2_event_retention` ;
478+ yield * TestClock . adjust ( Duration . hours ( 25 ) ) ;
479+ assert . equal ( ( yield * maintenance . compactEventStore ) . deletedEventCount , 0 ) ;
480+ } ) . pipe ( Effect . provide ( layerTest ) ) ,
481+ ) ;
482+
222483it . layer ( layerTest ) ( "orchestration V2 foundation persistence" , ( it ) => {
223484 it . effect ( "projects oversized tool bodies before both replay and live RPC retention" , ( ) =>
224485 Effect . scoped (
0 commit comments