@@ -2929,6 +2929,16 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
29292929 const now = yield * DateTime . now ;
29302930 const nowIso = DateTime . formatIso ( now ) ;
29312931 const threadId = ThreadId . make ( "thread:bounded-sql-history" ) ;
2932+ // The raw SQL edits below stand in for committed writes, which always append a thread event.
2933+ let revision = 0 ;
2934+ const historyPage = ( ...args : Parameters < typeof projectionStore . getThreadHistoryPage > ) =>
2935+ sql `INSERT INTO orchestration_events (
2936+ event_id, aggregate_kind, stream_id, stream_version, event_type,
2937+ occurred_at, actor_kind, payload_json, metadata_json, application_event_version
2938+ ) VALUES (
2939+ ${ `event:bounded-sql-history:revision:${ ++ revision } ` } , 'thread', ${ threadId } , ${ revision } ,
2940+ 'turn-item.updated', ${ nowIso } , 'user', '{}', '{}', 2
2941+ )` . pipe ( Effect . andThen ( projectionStore . getThreadHistoryPage ( ...args ) ) ) ;
29322942 yield * projectionStore . apply ( {
29332943 id : EventId . make ( "event:bounded-sql-history:thread" ) ,
29342944 type : "thread.created" ,
@@ -3152,7 +3162,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
31523162 sourceItemId : TurnItemId . make ( "turn-item:bounded-sql-history:71" ) ,
31533163 position : 70 ,
31543164 } ) ;
3155- const bytePage = yield * projectionStore . getThreadHistoryPage ( threadId , byteCursor ) ;
3165+ const bytePage = yield * historyPage ( threadId , byteCursor ) ;
31563166 assert . isAbove ( bytePage . items . length , 0 ) ;
31573167 assert . isBelow ( bytePage . items . length , 70 ) ;
31583168 assert . isAtMost (
@@ -3163,7 +3173,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
31633173 let byteNext = bytePage . nextCursor ;
31643174 for ( let pages = 0 ; byteNext !== null ; pages += 1 ) {
31653175 assert . isBelow ( pages , 70 ) ;
3166- const page = yield * projectionStore . getThreadHistoryPage ( threadId , byteNext ) ;
3176+ const page = yield * historyPage ( threadId , byteNext ) ;
31673177 assert . isAbove ( page . items . length , 0 ) ;
31683178 byteIds . unshift ( ...page . items . map ( ( row ) => String ( row . sourceItemId ) ) ) ;
31693179 byteNext = page . nextCursor ;
@@ -3269,7 +3279,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
32693279 '$.attachments', json('[]'))
32703280 WHERE thread_id = ${ threadId } AND ordinal BETWEEN 600 AND 624
32713281 ` ;
3272- const targetedPage = yield * projectionStore . getThreadHistoryPage (
3282+ const targetedPage = yield * historyPage (
32733283 threadId ,
32743284 encodeThreadHistoryCursor ( {
32753285 snapshotSequence : 0 ,
@@ -3299,7 +3309,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
32993309 '$.type', CASE ordinal WHEN 500 THEN 'run_interrupt_request' ELSE 'run_interrupt_result' END)
33003310 WHERE thread_id = ${ threadId } AND ordinal IN (500, 580)
33013311 ` ;
3302- const interruptPage = yield * projectionStore . getThreadHistoryPage (
3312+ const interruptPage = yield * historyPage (
33033313 threadId ,
33043314 encodeThreadHistoryCursor ( {
33053315 snapshotSequence : 0 ,
@@ -3446,7 +3456,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
34463456 let workCursor : string | null = anchor . item . deferredWork ! . cursor ;
34473457 for ( let pages = 0 ; workCursor !== null ; pages += 1 ) {
34483458 assert . isBelow ( pages , 20 ) ;
3449- const page : OrchestrationV2ThreadHistoryPage = yield * projectionStore . getThreadHistoryPage (
3459+ const page : OrchestrationV2ThreadHistoryPage = yield * historyPage (
34503460 threadId ,
34513461 workCursor ,
34523462 anchor . sourceItemId ,
@@ -3475,7 +3485,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
34753485 window . projection . visibleTurnItems . find ( ( row ) => row . item . deferredWork ) ?. item . ordinal ,
34763486 3 ,
34773487 ) ;
3478- const page = yield * projectionStore . getThreadHistoryPage (
3488+ const page = yield * historyPage (
34793489 threadId ,
34803490 encodeThreadHistoryCursor ( {
34813491 snapshotSequence : 0 ,
@@ -3532,14 +3542,13 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
35323542 const loaded = new Set < TurnItemId > ( ) ;
35333543 for ( let pages = 0 ; cursor !== null ; pages += 1 ) {
35343544 assert . isBelow ( pages , 20 ) ;
3535- const page : OrchestrationV2ThreadHistoryPage =
3536- yield * projectionStore . getThreadHistoryPage (
3537- threadId ,
3538- cursor ,
3539- oldWork ! . sourceItemId ,
3540- false ,
3541- "work" ,
3542- ) ;
3545+ const page : OrchestrationV2ThreadHistoryPage = yield * historyPage (
3546+ threadId ,
3547+ cursor ,
3548+ oldWork ! . sourceItemId ,
3549+ false ,
3550+ "work" ,
3551+ ) ;
35433552 for ( const row of page . items ) loaded . add ( row . sourceItemId ) ;
35443553 cursor = page . nextCursor ;
35453554 }
@@ -4470,6 +4479,276 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
44704479 } ) ,
44714480 ) ;
44724481
4482+ it . effect ( "pages a whole thread without re-reading its full timeline for every page" , ( ) =>
4483+ Effect . gen ( function * ( ) {
4484+ const projectionStore = yield * ProjectionStore . ProjectionStoreV2 ;
4485+ const sql = yield * SqlClient . SqlClient ;
4486+ const now = yield * DateTime . now ;
4487+ const nowIso = DateTime . formatIso ( now ) ;
4488+ const threadId = ThreadId . make ( "thread:paged-scrollback" ) ;
4489+ const runId = RunId . make ( "run:paged-scrollback" ) ;
4490+ const itemCount = 600 ;
4491+ yield * projectionStore . apply ( {
4492+ id : EventId . make ( "event:paged-scrollback:thread" ) ,
4493+ type : "thread.created" ,
4494+ threadId,
4495+ occurredAt : now ,
4496+ payload : {
4497+ createdBy : "user" ,
4498+ creationSource : "web" ,
4499+ id : threadId ,
4500+ projectId : ProjectId . make ( "project:paged-scrollback" ) ,
4501+ title : "Paged scrollback" ,
4502+ providerInstanceId,
4503+ modelSelection,
4504+ runtimeMode : "full-access" ,
4505+ interactionMode : "default" ,
4506+ branch : null ,
4507+ worktreePath : null ,
4508+ activeProviderThreadId : null ,
4509+ lineage : { parentThreadId : null , relationshipToParent : null , rootThreadId : threadId } ,
4510+ forkedFrom : null ,
4511+ createdAt : now ,
4512+ updatedAt : now ,
4513+ archivedAt : null ,
4514+ settledOverride : null ,
4515+ settledAt : null ,
4516+ lastVisitedAt : null ,
4517+ deletedAt : null ,
4518+ } ,
4519+ } ) ;
4520+ yield * sql `
4521+ INSERT INTO orchestration_v2_projection_runs (
4522+ run_id, thread_id, ordinal, provider, provider_thread_id, status,
4523+ requested_at, completed_at, payload_json
4524+ ) VALUES (
4525+ ${ runId } , ${ threadId } , 1, 'codex', NULL, 'completed', ${ nowIso } , ${ nowIso } ,
4526+ ${ encodeUnknownJsonString ( {
4527+ id : runId ,
4528+ threadId,
4529+ ordinal : 1 ,
4530+ providerInstanceId,
4531+ modelSelection,
4532+ providerThreadId : null ,
4533+ userMessageId : "message:paged-scrollback" ,
4534+ rootNodeId : "node:paged-scrollback" ,
4535+ activeAttemptId : null ,
4536+ status : "completed" ,
4537+ requestedAt : nowIso ,
4538+ startedAt : nowIso ,
4539+ completedAt : nowIso ,
4540+ checkpointId : null ,
4541+ contextHandoffId : null ,
4542+ } ) }
4543+ )
4544+ ` ;
4545+ for ( let ordinal = 0 ; ordinal < itemCount ; ordinal += 1 ) {
4546+ const id = `turn-item:paged-scrollback:${ ordinal } ` ;
4547+ const message = ordinal % 9 === 0 ;
4548+ yield * sql `
4549+ INSERT INTO orchestration_v2_projection_turn_items (
4550+ turn_item_id, thread_id, run_id, node_id, provider_thread_id, provider_turn_id,
4551+ parent_item_id, ordinal, type, status, updated_at, payload_json
4552+ ) VALUES (
4553+ ${ id } , ${ threadId } , ${ runId } , NULL, NULL, NULL, NULL, ${ ordinal } ,
4554+ ${ message ? "user_message" : "command_execution" } , 'completed', ${ nowIso } ,
4555+ ${ encodeUnknownJsonString ( {
4556+ id,
4557+ threadId,
4558+ runId,
4559+ nodeId : null ,
4560+ providerThreadId : null ,
4561+ providerTurnId : null ,
4562+ nativeItemRef : null ,
4563+ parentItemId : null ,
4564+ ordinal,
4565+ status : "completed" ,
4566+ title : `item ${ ordinal } ` ,
4567+ startedAt : nowIso ,
4568+ completedAt : nowIso ,
4569+ updatedAt : nowIso ,
4570+ ...( message
4571+ ? {
4572+ type : "user_message" ,
4573+ createdBy : "user" ,
4574+ creationSource : "web" ,
4575+ messageId : `message:paged-scrollback:${ ordinal } ` ,
4576+ inputIntent : "turn_start" ,
4577+ text : `Turn ${ ordinal } ` ,
4578+ attachments : [ ] ,
4579+ }
4580+ : { type : "command_execution" , input : "echo" , output : "ok" , exitCode : 0 } ) ,
4581+ } ) }
4582+ )
4583+ ` ;
4584+ }
4585+ // An unbounded read of every turn item in the thread costs O(thread) per page, so a full
4586+ // scrollback that repeats it on each page is quadratic in thread length.
4587+ const wholeThreadReads = ( statements : ReadonlyArray < string > ) =>
4588+ statements . filter (
4589+ ( statement ) =>
4590+ / F R O M o r c h e s t r a t i o n _ v 2 _ p r o j e c t i o n _ t u r n _ i t e m s ( \s + A S i t e m ) ? \s + W H E R E ( i t e m \. ) ? t h r e a d _ i d = \? \s * ( O R D E R B Y | $ ) / . test (
4591+ statement ,
4592+ ) && ! / \b L I M I T \b / . test ( statement ) ,
4593+ ) . length ;
4594+ const snapshot = yield * projectionStore . getThreadSnapshot ( threadId ) ;
4595+
4596+ const history = traceSqlStatements ( ) ;
4597+ let cursor : string | null = encodeThreadHistoryCursor ( {
4598+ snapshotSequence : snapshot . snapshotSequence ,
4599+ sourceThreadId : threadId ,
4600+ sourceItemId : "absent" ,
4601+ position : itemCount ,
4602+ } ) ;
4603+ const historyIds : Array < string > = [ ] ;
4604+ let historyPages = 0 ;
4605+ while ( cursor !== null ) {
4606+ const page : OrchestrationV2ThreadHistoryPage = yield * projectionStore
4607+ . getThreadHistoryPage ( threadId , cursor )
4608+ . pipe ( Effect . withTracer ( history . tracer ) ) ;
4609+ historyIds . push ( ...page . items . map ( ( row ) => String ( row . sourceItemId ) ) ) ;
4610+ historyPages += 1 ;
4611+ cursor = page . hasMoreHistory ? page . nextCursor : null ;
4612+ }
4613+ assert . strictEqual ( new Set ( historyIds ) . size , itemCount ) ;
4614+ assert . isAtLeast ( historyPages , 4 ) ;
4615+
4616+ const timeline = traceSqlStatements ( ) ;
4617+ const timelineIds : Array < string > = [ ] ;
4618+ let afterPosition = - 1 ;
4619+ let timelinePages = 0 ;
4620+ for ( let hasMore = true ; hasMore ; ) {
4621+ const page = yield * projectionStore
4622+ . getTimelinePage ( threadId , { view : "activity" , afterPosition, limit : 100 } )
4623+ . pipe ( Effect . withTracer ( timeline . tracer ) ) ;
4624+ timelineIds . push ( ...page . items . map ( ( row ) => String ( row . sourceItemId ) ) ) ;
4625+ afterPosition = page . items . at ( - 1 ) ?. position ?? afterPosition ;
4626+ timelinePages += 1 ;
4627+ hasMore = page . hasMore ;
4628+ }
4629+ assert . strictEqual ( new Set ( timelineIds ) . size , itemCount ) ;
4630+ assert . isAtLeast ( timelinePages , 4 ) ;
4631+
4632+ assert . isAtMost (
4633+ wholeThreadReads ( history . statements ) ,
4634+ 1 ,
4635+ `getThreadHistoryPage read the whole thread on each of ${ historyPages } pages` ,
4636+ ) ;
4637+ assert . isAtMost (
4638+ wholeThreadReads ( timeline . statements ) ,
4639+ 1 ,
4640+ `getTimelinePage read the whole thread on each of ${ timelinePages } pages` ,
4641+ ) ;
4642+
4643+ // A write to the thread a fork inherits from must reach the fork's next page.
4644+ const forkThreadId = ThreadId . make ( "thread:paged-scrollback:fork" ) ;
4645+ yield * projectionStore . apply ( {
4646+ id : EventId . make ( "event:paged-scrollback:fork" ) ,
4647+ type : "thread.created" ,
4648+ threadId : forkThreadId ,
4649+ occurredAt : now ,
4650+ payload : {
4651+ createdBy : "user" ,
4652+ creationSource : "web" ,
4653+ id : forkThreadId ,
4654+ projectId : ProjectId . make ( "project:paged-scrollback" ) ,
4655+ title : "Paged scrollback fork" ,
4656+ providerInstanceId,
4657+ modelSelection,
4658+ runtimeMode : "full-access" ,
4659+ interactionMode : "default" ,
4660+ branch : null ,
4661+ worktreePath : null ,
4662+ activeProviderThreadId : null ,
4663+ lineage : {
4664+ parentThreadId : threadId ,
4665+ relationshipToParent : "fork" ,
4666+ rootThreadId : threadId ,
4667+ } ,
4668+ forkedFrom : { type : "run" , threadId, runId } ,
4669+ createdAt : now ,
4670+ updatedAt : now ,
4671+ archivedAt : null ,
4672+ settledOverride : null ,
4673+ settledAt : null ,
4674+ lastVisitedAt : null ,
4675+ deletedAt : null ,
4676+ } ,
4677+ } ) ;
4678+ const forkPage = ( ) =>
4679+ projectionStore . getTimelinePage ( forkThreadId , { view : "activity" , limit : 1 } ) ;
4680+ const forkTotal = ( yield * forkPage ( ) ) . totalItems ;
4681+ assert . strictEqual ( ( yield * forkPage ( ) ) . totalItems , forkTotal ) ;
4682+ yield * projectionStore . apply ( {
4683+ id : EventId . make ( "event:paged-scrollback:late-item" ) ,
4684+ type : "turn-item.updated" ,
4685+ threadId,
4686+ runId,
4687+ driver,
4688+ occurredAt : now ,
4689+ payload : {
4690+ id : TurnItemId . make ( "turn-item:paged-scrollback:late" ) ,
4691+ threadId,
4692+ runId,
4693+ nodeId : null ,
4694+ providerThreadId : null ,
4695+ providerTurnId : null ,
4696+ nativeItemRef : null ,
4697+ parentItemId : null ,
4698+ ordinal : itemCount ,
4699+ status : "completed" ,
4700+ title : "late" ,
4701+ startedAt : now ,
4702+ completedAt : now ,
4703+ updatedAt : now ,
4704+ type : "command_execution" ,
4705+ input : "echo" ,
4706+ } ,
4707+ } ) ;
4708+ assert . strictEqual ( ( yield * forkPage ( ) ) . totalItems , forkTotal + 1 ) ;
4709+
4710+ // Another store instance's write reaches this one only through the committed thread event.
4711+ const directId = "turn-item:paged-scrollback:direct" ;
4712+ yield * sql `
4713+ INSERT INTO orchestration_v2_projection_turn_items (
4714+ turn_item_id, thread_id, run_id, node_id, provider_thread_id, provider_turn_id,
4715+ parent_item_id, ordinal, type, status, updated_at, payload_json
4716+ ) VALUES (
4717+ ${ directId } , ${ threadId } , ${ runId } , NULL, NULL, NULL, NULL, ${ itemCount + 1 } ,
4718+ 'command_execution', 'completed', ${ nowIso } ,
4719+ ${ encodeUnknownJsonString ( {
4720+ id : directId ,
4721+ threadId,
4722+ runId,
4723+ nodeId : null ,
4724+ providerThreadId : null ,
4725+ providerTurnId : null ,
4726+ nativeItemRef : null ,
4727+ parentItemId : null ,
4728+ ordinal : itemCount + 1 ,
4729+ status : "completed" ,
4730+ title : "direct" ,
4731+ startedAt : nowIso ,
4732+ completedAt : nowIso ,
4733+ updatedAt : nowIso ,
4734+ type : "command_execution" ,
4735+ input : "echo" ,
4736+ } ) }
4737+ )
4738+ ` ;
4739+ yield * sql `
4740+ INSERT INTO orchestration_events (
4741+ event_id, aggregate_kind, stream_id, stream_version, event_type,
4742+ occurred_at, actor_kind, payload_json, metadata_json, application_event_version
4743+ ) VALUES (
4744+ 'event:paged-scrollback:direct', 'thread', ${ threadId } , 1,
4745+ 'turn-item.updated', ${ nowIso } , 'user', '{}', '{}', 2
4746+ )
4747+ ` ;
4748+ assert . strictEqual ( ( yield * forkPage ( ) ) . totalItems , forkTotal + 2 ) ;
4749+ } ) ,
4750+ ) ;
4751+
44734752 it . effect ( "scopes actionable provider state to the requested bounded thread" , ( ) =>
44744753 Effect . gen ( function * ( ) {
44754754 const projectionStore = yield * ProjectionStore . ProjectionStoreV2 ;
@@ -4781,6 +5060,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
47815060 number : 42 ,
47825061 url : "https://github.com/acme/project/pull/42" ,
47835062 } ) } )) WHERE thread_id = ${ readThreadId } `;
5063+ const [ latest ] = yield * sql < { readonly sequence : number } > `
5064+ SELECT COALESCE(MAX(sequence), 0) AS sequence FROM orchestration_events` ;
47845065 const trace = traceSqlStatements ( ) ;
47855066 const discovery = yield * projectionStore
47865067 . getPullRequestSnapshot ( { } )
@@ -4791,7 +5072,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
47915072 . filter ( ( id ) => id === readThreadId || id === laterThreadId ) ,
47925073 [ readThreadId ] ,
47935074 ) ;
4794- assert . strictEqual ( discovery . snapshotSequence , 0 ) ;
5075+ assert . strictEqual ( discovery . snapshotSequence , latest ?. sequence ) ;
47955076 assert . strictEqual ( trace . queryTransactions . length , 2 ) ;
47965077 assert . isNotNull ( trace . queryTransactions [ 0 ] ) ;
47975078 assert . strictEqual ( new Set ( trace . queryTransactions ) . size , 1 ) ;
0 commit comments