@@ -474,37 +474,77 @@ const providerErrorEvent = (runId: string, message: string): OrchestrationV2Doma
474474} ) ;
475475
476476// Dispatches to a bound thread succeed at once; the run's events are fed by hand.
477- const makeTurnHarness = Effect . gen ( function * ( ) {
478- const events = yield * Queue . unbounded < OrchestrationV2DomainEvent > ( ) ;
479- const dispatched = yield * Queue . unbounded < MessageId > ( ) ;
480- const layer = ScheduledTaskService . layer . pipe (
481- Layer . provide (
482- Layer . mergeAll (
483- Layer . mock ( ThreadLaunchService . ThreadLaunchService ) ( { } ) ,
484- Layer . mock ( ThreadManagementService . ThreadManagementService ) ( {
485- sendToThread : ( input ) =>
486- Queue . offer ( dispatched , input . messageId ) . pipe ( Effect . as ( undefined as never ) ) ,
487- streamDomainEvents : Stream . fromQueue ( events ) ,
488- } ) ,
489- Layer . mock ( SecretRequests . SecretRequests ) ( { } ) ,
490- Layer . mock ( ManagedProjectFolders . ManagedProjectFolders ) ( {
491- namedProjectsRoot : "/projects" ,
492- isScratchProject : ( ) => Effect . succeed ( false ) ,
493- } ) ,
494- NodeCrypto . layer ,
495- Scheduler . layer ,
477+ const makeTurnHarness = (
478+ options : {
479+ /** Runs inside the dispatch, so the turn's events can land before it returns. */
480+ readonly duringDispatch ?: (
481+ events : Queue . Queue < OrchestrationV2DomainEvent > ,
482+ messageId : MessageId ,
483+ ) => Effect . Effect < void > ;
484+ /** The run the thread's projection says the scheduled message now belongs to. */
485+ readonly messageRunId ?: RunId ;
486+ /** The runs the thread's projection holds, as read back after the event stream drops. */
487+ readonly runs ?: ReadonlyArray < OrchestrationV2Run > ;
488+ /** How many times the event stream fails before it starts to deliver events. */
489+ readonly streamFailures ?: number ;
490+ } = { } ,
491+ ) =>
492+ Effect . gen ( function * ( ) {
493+ const events = yield * Queue . unbounded < OrchestrationV2DomainEvent > ( ) ;
494+ const dispatched = yield * Queue . unbounded < MessageId > ( ) ;
495+ const subscriptions = yield * Ref . make ( 0 ) ;
496+ const layer = ScheduledTaskService . layer . pipe (
497+ Layer . provide (
498+ Layer . mergeAll (
499+ Layer . mock ( ThreadLaunchService . ThreadLaunchService ) ( { } ) ,
500+ Layer . mock ( ThreadManagementService . ThreadManagementService ) ( {
501+ sendToThread : ( input ) =>
502+ Queue . offer ( dispatched , input . messageId ) . pipe (
503+ Effect . andThen ( options . duringDispatch ?.( events , input . messageId ) ?? Effect . void ) ,
504+ Effect . as ( undefined as never ) ,
505+ ) ,
506+ getThreadRecords : ( _threadId , _fields , filter ) =>
507+ Effect . succeed ( {
508+ messages : ( filter ?. messageIds ?? [ ] ) . map ( ( id ) => ( {
509+ id,
510+ runId : options . messageRunId ?? null ,
511+ } ) ) ,
512+ runs : ( options . runs ?? [ ] ) . filter (
513+ ( run ) => filter ?. runIds === undefined || filter . runIds . includes ( run . id ) ,
514+ ) ,
515+ } as never ) ,
516+ streamDomainEvents : Stream . unwrap (
517+ Ref . getAndUpdate ( subscriptions , ( n ) => n + 1 ) . pipe (
518+ Effect . map ( ( attempt ) =>
519+ attempt < ( options . streamFailures ?? 0 )
520+ ? Stream . fail ( new Error ( "event stream dropped" ) as never )
521+ : Stream . fromQueue ( events ) ,
522+ ) ,
523+ ) ,
524+ ) ,
525+ } ) ,
526+ Layer . mock ( SecretRequests . SecretRequests ) ( { } ) ,
527+ Layer . mock ( ManagedProjectFolders . ManagedProjectFolders ) ( {
528+ namedProjectsRoot : "/projects" ,
529+ isScratchProject : ( ) => Effect . succeed ( false ) ,
530+ } ) ,
531+ NodeCrypto . layer ,
532+ Scheduler . layer ,
533+ ) ,
496534 ) ,
497- ) ,
498- ) ;
499- return { events, dispatched, layer } ;
500- } ) ;
535+ ) ;
536+ return { events, dispatched, subscriptions, layer } ;
537+ } ) ;
501538
502- const upsertBoundTask = ( service : ScheduledTaskService . ScheduledTaskService [ "Service" ] ) =>
539+ const upsertBoundTask = (
540+ service : ScheduledTaskService . ScheduledTaskService [ "Service" ] ,
541+ enabled = false ,
542+ ) =>
503543 service . upsert ( {
504544 id : ScheduledTaskId . make ( "task:turn-outcome" ) ,
505545 title : "Check orchestration" ,
506546 prompt : "Check progress" ,
507- enabled : false ,
547+ enabled,
508548 schedule : { type : "interval" , everyMs : 60_000 } ,
509549 projectId : ProjectId . make ( "project:test" ) ,
510550 threadId : turnThreadId ,
@@ -527,7 +567,7 @@ const untilStatus = (
527567
528568it . effect ( "a failed turn replaces the dispatched run's status with the provider error" , ( ) =>
529569 Effect . gen ( function * ( ) {
530- const harness = yield * makeTurnHarness ;
570+ const harness = yield * makeTurnHarness ( ) ;
531571 yield * Effect . gen ( function * ( ) {
532572 const service = yield * ScheduledTaskService . ScheduledTaskService ;
533573 const { task } = yield * upsertBoundTask ( service ) ;
@@ -553,7 +593,7 @@ it.effect("a failed turn replaces the dispatched run's status with the provider
553593
554594it . effect ( "a completed turn records the run as succeeded" , ( ) =>
555595 Effect . gen ( function * ( ) {
556- const harness = yield * makeTurnHarness ;
596+ const harness = yield * makeTurnHarness ( ) ;
557597 yield * Effect . gen ( function * ( ) {
558598 const service = yield * ScheduledTaskService . ScheduledTaskService ;
559599 const { task } = yield * upsertBoundTask ( service ) ;
@@ -572,7 +612,7 @@ it.effect("a completed turn records the run as succeeded", () =>
572612
573613it . effect ( "an earlier run's turn ending cannot overwrite a newer run" , ( ) =>
574614 Effect . gen ( function * ( ) {
575- const harness = yield * makeTurnHarness ;
615+ const harness = yield * makeTurnHarness ( ) ;
576616 yield * Effect . gen ( function * ( ) {
577617 const service = yield * ScheduledTaskService . ScheduledTaskService ;
578618 const { task } = yield * upsertBoundTask ( service ) ;
@@ -594,3 +634,137 @@ it.effect("an earlier run's turn ending cannot overwrite a newer run", () =>
594634 } ) . pipe ( Effect . provide ( harness . layer ) ) ;
595635 } ) . pipe ( Effect . provide ( SqlitePersistence . layerMemory ) ) ,
596636) ;
637+
638+ const messageMovedEvent = ( messageId : MessageId , runId : string ) : OrchestrationV2DomainEvent => ( {
639+ id : EventId . make ( `event:${ messageId } :moved` ) ,
640+ type : "message.updated" ,
641+ threadId : turnThreadId ,
642+ runId : RunId . make ( runId ) ,
643+ occurredAt : epoch ,
644+ payload : {
645+ id : messageId ,
646+ threadId : turnThreadId ,
647+ runId : RunId . make ( runId ) ,
648+ nodeId : null ,
649+ role : "user" ,
650+ text : "Check progress" ,
651+ attachments : [ ] ,
652+ streaming : false ,
653+ createdAt : epoch ,
654+ updatedAt : epoch ,
655+ createdBy : "user" ,
656+ creationSource : "web" ,
657+ } ,
658+ } ) ;
659+
660+ it . effect ( "a turn that ends while the dispatch is being recorded still advances the schedule" , ( ) =>
661+ Effect . gen ( function * ( ) {
662+ const harness = yield * makeTurnHarness ( {
663+ duringDispatch : ( events , messageId ) =>
664+ Queue . offerAll ( events , [
665+ runEvent ( runOf ( "run:fast" , messageId , "queued" ) ) ,
666+ runEvent ( runOf ( "run:fast" , messageId , "failed" ) ) ,
667+ ] ) ,
668+ } ) ;
669+ yield * Effect . gen ( function * ( ) {
670+ const service = yield * ScheduledTaskService . ScheduledTaskService ;
671+ const { task } = yield * upsertBoundTask ( service , true ) ;
672+ yield * TestClock . adjust ( "30 seconds" ) ;
673+ yield * service . runNow ( { id : task . id } ) ;
674+ yield * untilStatus ( service , task . id , "failed" ) ;
675+ const [ settled ] = ( yield * service . list ( ) ) . tasks ;
676+ assert . equal ( settled ?. runCount , 1 ) ;
677+ // Dispatched 30 seconds after the task was saved, so the next run is 60 seconds on from then.
678+ assert . equal (
679+ settled ?. nextRunAt ,
680+ DateTime . formatIso ( DateTime . makeUnsafe ( Date . parse ( task . nextRunAt ! ) + 30_000 ) ) ,
681+ ) ;
682+ } ) . pipe ( Effect . provide ( harness . layer ) ) ;
683+ } ) . pipe ( Effect . provide ( SqlitePersistence . layerMemory ) ) ,
684+ ) ;
685+
686+ it . effect ( "a queued message promoted to a steer keeps its run going in the active run" , ( ) =>
687+ Effect . gen ( function * ( ) {
688+ const harness = yield * makeTurnHarness ( { messageRunId : RunId . make ( "run:active" ) } ) ;
689+ yield * Effect . gen ( function * ( ) {
690+ const service = yield * ScheduledTaskService . ScheduledTaskService ;
691+ const { task } = yield * upsertBoundTask ( service ) ;
692+ yield * service . runNow ( { id : task . id } ) ;
693+ const messageId = yield * Queue . take ( harness . dispatched ) ;
694+ yield * Queue . offerAll ( harness . events , [
695+ runEvent ( runOf ( "run:queued" , messageId , "queued" ) ) ,
696+ // Promotion cancels the queued run before it moves the message into the active one.
697+ runEvent ( runOf ( "run:queued" , messageId , "cancelled" ) ) ,
698+ messageMovedEvent ( messageId , "run:active" ) ,
699+ runEvent ( runOf ( "run:active" , MessageId . make ( "message:active-turn" ) , "completed" ) ) ,
700+ ] ) ;
701+ yield * service . subscribeList ( ) . pipe (
702+ Stream . filter ( ( { tasks } ) => tasks . some ( ( t ) => t . lastRunStatus !== "running" ) ) ,
703+ Stream . take ( 1 ) ,
704+ Stream . runDrain ,
705+ ) ;
706+ const [ settled ] = ( yield * service . list ( ) ) . tasks ;
707+ assert . equal ( settled ?. lastRunStatus , "succeeded" ) ;
708+ assert . isNull ( settled ?. lastRunError ) ;
709+ } ) . pipe ( Effect . provide ( harness . layer ) ) ;
710+ } ) . pipe ( Effect . provide ( SqlitePersistence . layerMemory ) ) ,
711+ ) ;
712+
713+ it . effect ( "a run whose outcome cannot be saved is released instead of left running" , ( ) =>
714+ Effect . gen ( function * ( ) {
715+ const harness = yield * makeTurnHarness ( ) ;
716+ yield * Effect . gen ( function * ( ) {
717+ const service = yield * ScheduledTaskService . ScheduledTaskService ;
718+ const sql = yield * SqlClient . SqlClient ;
719+ const { task } = yield * upsertBoundTask ( service ) ;
720+ yield * service . runNow ( { id : task . id } ) ;
721+ const messageId = yield * Queue . take ( harness . dispatched ) ;
722+ // Only the release path may write the terminal state.
723+ yield * sql `CREATE TRIGGER fail_outcome BEFORE UPDATE ON scheduled_tasks
724+ WHEN OLD.last_run_status = 'running' AND NEW.last_run_status = 'succeeded'
725+ BEGIN SELECT RAISE(FAIL, 'outcome unavailable'); END` ;
726+ yield * Queue . offerAll ( harness . events , [
727+ runEvent ( runOf ( "run:unsaved" , messageId , "completed" ) ) ,
728+ ] ) ;
729+ yield * untilStatus ( service , task . id , "failed" ) ;
730+ const [ settled ] = ( yield * service . list ( ) ) . tasks ;
731+ assert . equal ( settled ?. lastRunError , "Could not record how the run ended." ) ;
732+ } ) . pipe ( Effect . provide ( harness . layer ) ) ;
733+ } ) . pipe ( Effect . provide ( SqlitePersistence . layerMemory ) ) ,
734+ ) ;
735+
736+ it . effect ( "a dropped event stream is resumed, and runs that ended meanwhile are caught up" , ( ) =>
737+ Effect . gen ( function * ( ) {
738+ const runId = RunId . make ( "run:during-gap" ) ;
739+ const harness = yield * makeTurnHarness ( {
740+ streamFailures : 1 ,
741+ messageRunId : runId ,
742+ runs : [ runOf ( runId , MessageId . make ( "unused" ) , "completed" ) ] ,
743+ } ) ;
744+ yield * Effect . gen ( function * ( ) {
745+ const service = yield * ScheduledTaskService . ScheduledTaskService ;
746+ const { task } = yield * upsertBoundTask ( service ) ;
747+ yield * service . runNow ( { id : task . id } ) ;
748+ yield * Queue . take ( harness . dispatched ) ;
749+ // The run completed while no one was listening; the resumed listener finds it.
750+ yield * TestClock . adjust ( "30 seconds" ) ;
751+ yield * untilStatus ( service , task . id , "succeeded" ) ;
752+ assert . equal ( yield * Ref . get ( harness . subscriptions ) , 2 ) ;
753+ } ) . pipe ( Effect . provide ( harness . layer ) ) ;
754+ } ) . pipe ( Effect . provide ( SqlitePersistence . layerMemory ) ) ,
755+ ) ;
756+
757+ it . effect ( "runs are released as failed once the event stream cannot be resumed" , ( ) =>
758+ Effect . gen ( function * ( ) {
759+ const harness = yield * makeTurnHarness ( { streamFailures : 1_000 } ) ;
760+ yield * Effect . gen ( function * ( ) {
761+ const service = yield * ScheduledTaskService . ScheduledTaskService ;
762+ const { task } = yield * upsertBoundTask ( service ) ;
763+ yield * service . runNow ( { id : task . id } ) ;
764+ yield * TestClock . adjust ( "10 minutes" ) ;
765+ yield * untilStatus ( service , task . id , "failed" ) ;
766+ const [ settled ] = ( yield * service . list ( ) ) . tasks ;
767+ assert . equal ( settled ?. lastRunError , "Run outcomes could no longer be followed." ) ;
768+ } ) . pipe ( Effect . provide ( harness . layer ) ) ;
769+ } ) . pipe ( Effect . provide ( SqlitePersistence . layerMemory ) ) ,
770+ ) ;
0 commit comments