@@ -17,20 +17,23 @@ import {
1717 type OrchestrationV2TurnItem ,
1818 ProviderDriverKind ,
1919 ProviderInstanceId ,
20- type ProviderThreadId ,
20+ ProviderThreadId ,
2121 ProviderTurnId ,
2222 RunAttemptId ,
2323 RunId ,
2424 RuntimeRequestId ,
2525 TurnItemId ,
2626} from "@t3tools/contracts" ;
27+ import * as Cause from "effect/Cause" ;
2728import * as DateTime from "effect/DateTime" ;
2829import * as Deferred from "effect/Deferred" ;
2930import * as Fiber from "effect/Fiber" ;
3031import * as Effect from "effect/Effect" ;
32+ import * as Exit from "effect/Exit" ;
3133import * as FileSystem from "effect/FileSystem" ;
3234import * as Layer from "effect/Layer" ;
3335import * as SqlClient from "effect/sql/SqlClient" ;
36+ import * as SqlError from "effect/sql/SqlError" ;
3437import * as Stream from "effect/Stream" ;
3538
3639import { toolOutputImages } from "@t3tools/shared/toolOutput" ;
@@ -65,25 +68,63 @@ const layerGit = GitVcsDriver.layer.pipe(
6568 Layer . provideMerge ( NodeServices . layer ) ,
6669) ;
6770
68- const layerTest = Layer . mergeAll (
69- layerTestDatabase ,
70- layerTestStores ,
71- layerTestEventSink ,
72- IdAllocator . layer ,
73- ThreadCommandExecutor . layer ,
74- layerGit ,
75- ProviderEventIngestor . layer . pipe (
76- Layer . provide (
77- Layer . mergeAll (
78- layerTestStores ,
79- layerTestEventSink ,
80- IdAllocator . layer ,
81- ThreadCommandExecutor . layer ,
82- layerGit ,
71+ const makeLayerTest = < E > ( ingestorEventSink : Layer . Layer < EventSink . EventSinkV2 , E > ) =>
72+ Layer . mergeAll (
73+ layerTestDatabase ,
74+ layerTestStores ,
75+ layerTestEventSink ,
76+ IdAllocator . layer ,
77+ ThreadCommandExecutor . layer ,
78+ layerGit ,
79+ ProviderEventIngestor . layer . pipe (
80+ Layer . provide (
81+ Layer . mergeAll (
82+ layerTestStores ,
83+ ingestorEventSink ,
84+ IdAllocator . layer ,
85+ ThreadCommandExecutor . layer ,
86+ layerGit ,
87+ ) ,
8388 ) ,
8489 ) ,
85- ) ,
86- ) ;
90+ ) ;
91+ const layerTest = makeLayerTest ( layerTestEventSink ) ;
92+
93+ // EventSink reports a lock that outlasted busy_timeout at BEGIN IMMEDIATE this way.
94+ const lockTimeout = new EventSink . EventSinkWriteError ( {
95+ eventCount : 1 ,
96+ cause : new SqlError . SqlError ( { reason : new SqlError . LockTimeoutError ( { cause : null } ) } ) ,
97+ } ) ;
98+ const constraintFailure = new EventSink . EventSinkWriteError ( {
99+ eventCount : 1 ,
100+ cause : new SqlError . SqlError ( { reason : new SqlError . ConstraintError ( { cause : null } ) } ) ,
101+ } ) ;
102+
103+ /** The ingestor's next sink writes of `events` fail with `errors`, in order; the rest go through. */
104+ const layerFailingWrites = (
105+ errors : ReadonlyArray < EventSink . EventSinkWriteError > ,
106+ matches : ( events : ReadonlyArray < OrchestrationV2DomainEvent > ) => boolean = ( ) => true ,
107+ ) =>
108+ makeLayerTest (
109+ Layer . effect (
110+ EventSink . EventSinkV2 ,
111+ Effect . gen ( function * ( ) {
112+ const eventSink = yield * EventSink . EventSinkV2 ;
113+ const pending = [ ...errors ] ;
114+ const failNext = < A , R > (
115+ events : ReadonlyArray < OrchestrationV2DomainEvent > ,
116+ write : Effect . Effect < A , EventSink . EventSinkV2Error , R > ,
117+ ) => Effect . suspend ( ( ) => ( matches ( events ) ? ( pending . shift ( ) ?? write ) : write ) ) ;
118+ return EventSink . EventSinkV2 . of ( {
119+ ...eventSink ,
120+ write : ( input ) => failNext ( input . events , eventSink . write ( input ) ) ,
121+ writeIfRunCurrent : ( input ) => failNext ( input . events , eventSink . writeIfRunCurrent ( input ) ) ,
122+ writeIfProviderThreadOwner : ( input ) =>
123+ failNext ( input . events , eventSink . writeIfProviderThreadOwner ( input ) ) ,
124+ } ) ;
125+ } ) ,
126+ ) . pipe ( Layer . provide ( layerTestEventSink ) ) ,
127+ ) ;
87128
88129const makeSharedWorktree = Effect . gen ( function * ( ) {
89130 const fs = yield * FileSystem . FileSystem ;
@@ -2005,7 +2046,7 @@ layer("ProviderEventIngestorV2", (it) => {
20052046 } ) ,
20062047 ) ;
20072048
2008- it . effect ( "moves a native subagent's thread to the model its provider reports later " , ( ) =>
2049+ it . effect ( "moves a native subagent's thread to its later model through a database lock " , ( ) =>
20092050 Effect . gen ( function * ( ) {
20102051 const now = yield * DateTime . now ;
20112052 const eventSink = yield * EventSink . EventSinkV2 ;
@@ -2124,6 +2165,103 @@ layer("ProviderEventIngestorV2", (it) => {
21242165 ( yield * projectionStore . getThread ( childThreadId ) ) . modelSelection ,
21252166 reportedSelection ,
21262167 ) ;
2127- } ) ,
2168+ } ) . pipe (
2169+ // Fresh: the shared test layer has already built the ingestor this replaces.
2170+ Effect . provide (
2171+ Layer . fresh (
2172+ layerFailingWrites ( [ lockTimeout , lockTimeout ] , ( events ) =>
2173+ events . some ( ( event ) => event . type === "thread.model-selection-updated" ) ,
2174+ ) ,
2175+ ) ,
2176+ ) ,
2177+ ) ,
21282178 ) ;
21292179} ) ;
2180+
2181+ const ingestCompletedProviderTurn = (
2182+ guard : Pick <
2183+ Parameters < ProviderEventIngestor . ProviderEventIngestorV2Shape [ "ingestNormalized" ] > [ 0 ] ,
2184+ "writeIfRunCurrent" | "writeIfProviderThreadOwner"
2185+ > ,
2186+ ) =>
2187+ Effect . gen ( function * ( ) {
2188+ const now = yield * DateTime . now ;
2189+ const eventSink = yield * EventSink . EventSinkV2 ;
2190+ const projectionStore = yield * ProjectionStore . ProjectionStoreV2 ;
2191+ const ingestor = yield * ProviderEventIngestor . ProviderEventIngestorV2 ;
2192+ const idAllocator = yield * IdAllocator . IdAllocatorV2 ;
2193+ const threadEvent = yield * threadCreatedEvent ( now ) ;
2194+ yield * eventSink . write ( { events : [ threadEvent ] } ) ;
2195+ const providerTurnId = ProviderTurnId . make ( "provider-turn:locked" ) ;
2196+ const exit = yield * ingestor
2197+ . ingestNormalized ( {
2198+ providerSessionId : yield * idAllocator . allocate . providerSession ( {
2199+ providerInstanceId : modelSelection . instanceId ,
2200+ threadId : threadEvent . threadId ,
2201+ } ) ,
2202+ providerInstanceId : modelSelection . instanceId ,
2203+ threadId : threadEvent . threadId ,
2204+ event : {
2205+ type : "provider_turn.updated" ,
2206+ driver : CODEX_DRIVER ,
2207+ providerTurn : {
2208+ id : providerTurnId ,
2209+ providerThreadId : ProviderThreadId . make ( "provider-thread:locked" ) ,
2210+ nodeId : NodeId . make ( "node:locked-turn" ) ,
2211+ runAttemptId : null ,
2212+ nativeTurnRef : null ,
2213+ ordinal : 1 ,
2214+ status : "completed" ,
2215+ startedAt : now ,
2216+ completedAt : now ,
2217+ } ,
2218+ } ,
2219+ ...guard ,
2220+ } )
2221+ . pipe ( Effect . exit ) ;
2222+ const projection = yield * projectionStore . getThreadProjection ( threadEvent . threadId ) ;
2223+ return { exit, providerTurnId, persisted : projection . providerTurns . map ( ( turn ) => turn . id ) } ;
2224+ } ) ;
2225+
2226+ // Neither guard matches a stored run, so a guarded write correctly commits nothing.
2227+ const runId = RunId . make ( "run:locked" ) ;
2228+ const activeAttemptId = RunAttemptId . make ( "attempt:locked" ) ;
2229+ it . effect . each ( [
2230+ { route : "write" , guard : { } } ,
2231+ {
2232+ route : "writeIfRunCurrent" ,
2233+ guard : { writeIfRunCurrent : { runId, activeAttemptId, expectedStatus : "running" } } ,
2234+ } ,
2235+ {
2236+ route : "writeIfProviderThreadOwner" ,
2237+ guard : {
2238+ writeIfProviderThreadOwner : {
2239+ providerThreadId : ProviderThreadId . make ( "provider-thread:locked" ) ,
2240+ runId,
2241+ activeAttemptId,
2242+ expectedLastRunOrdinal : 1 ,
2243+ } ,
2244+ } ,
2245+ } ,
2246+ ] as const ) ( "a provider event $route outlasts a brief database lock" , ( { route, guard } ) =>
2247+ Effect . gen ( function * ( ) {
2248+ const { exit, providerTurnId, persisted } = yield * ingestCompletedProviderTurn ( guard ) ;
2249+ assert . isTrue ( Exit . isSuccess ( exit ) ) ;
2250+ assert . deepEqual ( persisted , route === "write" ? [ providerTurnId ] : [ ] ) ;
2251+ } ) . pipe ( Effect . provide ( layerFailingWrites ( [ lockTimeout , lockTimeout ] ) ) ) ,
2252+ ) ;
2253+
2254+ // The write after the queued errors would succeed, so failing proves the retries stopped.
2255+ it . effect . each ( [
2256+ { failure : "stays locked" , errors : [ lockTimeout , lockTimeout , lockTimeout ] } ,
2257+ { failure : "fails for another reason" , errors : [ constraintFailure ] } ,
2258+ ] ) ( "fails a provider event write that $failure" , ( { errors } ) =>
2259+ Effect . gen ( function * ( ) {
2260+ const { exit, persisted } = yield * ingestCompletedProviderTurn ( { } ) ;
2261+ assert . instanceOf (
2262+ Exit . isFailure ( exit ) ? Cause . squash ( exit . cause ) : undefined ,
2263+ ProviderEventIngestor . ProviderEventPublishError ,
2264+ ) ;
2265+ assert . isEmpty ( persisted ) ;
2266+ } ) . pipe ( Effect . provide ( layerFailingWrites ( errors ) ) ) ,
2267+ ) ;
0 commit comments