@@ -19,17 +19,19 @@ import {
1919 ProviderDriverKind ,
2020 ProviderInstanceId ,
2121 PlanId ,
22- type ProviderThreadId ,
22+ ProviderThreadId ,
2323 ProviderTurnId ,
2424 RunAttemptId ,
2525 RunId ,
2626 RuntimeRequestId ,
2727 TurnItemId ,
2828} from "@t3tools/contracts" ;
29+ import * as Cause from "effect/Cause" ;
2930import * as DateTime from "effect/DateTime" ;
3031import * as Deferred from "effect/Deferred" ;
3132import * as Fiber from "effect/Fiber" ;
3233import * as Effect from "effect/Effect" ;
34+ import * as Exit from "effect/Exit" ;
3335import * as FileSystem from "effect/FileSystem" ;
3436import * as Layer from "effect/Layer" ;
3537import * as SqlClient from "effect/sql/SqlClient" ;
@@ -69,25 +71,27 @@ const layerGit = GitVcsDriver.layer.pipe(
6971 Layer . provideMerge ( NodeServices . layer ) ,
7072) ;
7173
72- const layerTest = Layer . mergeAll (
73- layerTestDatabase ,
74- layerTestStores ,
75- layerTestEventSink ,
76- IdAllocator . layer ,
77- ThreadCommandExecutor . layer ,
78- layerGit ,
79- ProviderEventIngestor . layer . pipe (
80- Layer . provide (
81- Layer . mergeAll (
82- layerTestStores ,
83- layerTestEventSink ,
84- IdAllocator . layer ,
85- ThreadCommandExecutor . layer ,
86- layerGit ,
74+ const makeLayerTest = < E > ( ingestorEventSink : Layer . Layer < EventSink . EventSinkV2 , E > ) =>
75+ Layer . mergeAll (
76+ layerTestDatabase ,
77+ layerTestStores ,
78+ layerTestEventSink ,
79+ IdAllocator . layer ,
80+ ThreadCommandExecutor . layer ,
81+ layerGit ,
82+ ProviderEventIngestor . layer . pipe (
83+ Layer . provide (
84+ Layer . mergeAll (
85+ layerTestStores ,
86+ ingestorEventSink ,
87+ IdAllocator . layer ,
88+ ThreadCommandExecutor . layer ,
89+ layerGit ,
90+ ) ,
8791 ) ,
8892 ) ,
89- ) ,
90- ) ;
93+ ) ;
94+ const layerTest = makeLayerTest ( layerTestEventSink ) ;
9195
9296const makeSharedWorktree = Effect . gen ( function * ( ) {
9397 const fs = yield * FileSystem . FileSystem ;
@@ -2244,122 +2248,120 @@ layer("ProviderEventIngestorV2", (it) => {
22442248 ) ;
22452249} ) ;
22462250
2247- const lockTimeoutWriteError = new EventSink . EventSinkWriteError ( {
2251+ // EventSink reports a lock that outlasted busy_timeout at BEGIN IMMEDIATE this way.
2252+ const lockTimeout = new EventSink . EventSinkWriteError ( {
22482253 eventCount : 1 ,
2249- cause : new SqlError . SqlError ( {
2250- reason : new SqlError . LockTimeoutError ( {
2251- cause : new Error ( "database is locked" ) ,
2252- message : "database is locked" ,
2253- } ) ,
2254- } ) ,
2254+ cause : new SqlError . SqlError ( { reason : new SqlError . LockTimeoutError ( { cause : null } ) } ) ,
2255+ } ) ;
2256+ const constraintFailure = new EventSink . EventSinkWriteError ( {
2257+ eventCount : 1 ,
2258+ cause : new SqlError . SqlError ( { reason : new SqlError . ConstraintError ( { cause : null } ) } ) ,
22552259} ) ;
22562260
2257- /** The ingestor sees an event sink whose next writes fail with `errors`, in order. */
2258- function layerFailingNextWrites ( errors : ReadonlyArray < EventSink . EventSinkV2Error > ) {
2259- const attempts = { count : 0 } ;
2260- const layerFailingEventSink = Layer . effect (
2261- EventSink . EventSinkV2 ,
2262- Effect . gen ( function * ( ) {
2263- const eventSink = yield * EventSink . EventSinkV2 ;
2264- return EventSink . EventSinkV2 . of ( {
2265- ...eventSink ,
2266- write : ( input ) =>
2267- Effect . suspend ( ( ) => {
2268- const error = errors [ attempts . count ] ;
2269- attempts . count += 1 ;
2270- return error === undefined ? eventSink . write ( input ) : Effect . fail ( error ) ;
2271- } ) ,
2272- } ) ;
2273- } ) ,
2274- ) . pipe ( Layer . provide ( layerTestEventSink ) ) ;
2275- const layer = Layer . mergeAll (
2276- layerTestDatabase ,
2277- layerTestStores ,
2278- layerTestEventSink ,
2279- IdAllocator . layer ,
2280- ProviderEventIngestor . layer . pipe (
2281- Layer . provide (
2282- Layer . mergeAll (
2283- layerTestStores ,
2284- layerFailingEventSink ,
2285- IdAllocator . layer ,
2286- ThreadCommandExecutor . layer ,
2287- layerGit ,
2288- ) ,
2289- ) ,
2290- ) ,
2261+ /** The ingestor's next sink writes fail with `errors`, in order; later writes go through. */
2262+ const layerFailingNextWrites = ( errors : ReadonlyArray < EventSink . EventSinkWriteError > ) =>
2263+ makeLayerTest (
2264+ Layer . effect (
2265+ EventSink . EventSinkV2 ,
2266+ Effect . gen ( function * ( ) {
2267+ const eventSink = yield * EventSink . EventSinkV2 ;
2268+ const pending = [ ...errors ] ;
2269+ const failNext = < A , R > ( write : Effect . Effect < A , EventSink . EventSinkV2Error , R > ) =>
2270+ Effect . suspend ( ( ) => pending . shift ( ) ?? write ) ;
2271+ return EventSink . EventSinkV2 . of ( {
2272+ ...eventSink ,
2273+ write : ( input ) => failNext ( eventSink . write ( input ) ) ,
2274+ writeIfRunCurrent : ( input ) => failNext ( eventSink . writeIfRunCurrent ( input ) ) ,
2275+ writeIfProviderThreadOwner : ( input ) =>
2276+ failNext ( eventSink . writeIfProviderThreadOwner ( input ) ) ,
2277+ } ) ;
2278+ } ) ,
2279+ ) . pipe ( Layer . provide ( layerTestEventSink ) ) ,
22912280 ) ;
2292- return { attempts, layer } ;
2293- }
22942281
2295- const ingestCompletedProviderTurn = Effect . gen ( function * ( ) {
2296- const now = yield * DateTime . now ;
2297- const eventSink = yield * EventSink . EventSinkV2 ;
2298- const ingestor = yield * ProviderEventIngestor . ProviderEventIngestorV2 ;
2299- const idAllocator = yield * IdAllocator . IdAllocatorV2 ;
2300- const threadEvent = yield * threadCreatedEvent ( now ) ;
2301- yield * eventSink . write ( { events : [ threadEvent ] } ) ;
2302- const providerSessionId = yield * idAllocator . allocate . providerSession ( {
2303- providerInstanceId : modelSelection . instanceId ,
2304- threadId : threadEvent . threadId ,
2305- } ) ;
2306- return yield * ingestor . ingestNormalized ( {
2307- providerSessionId,
2308- providerInstanceId : modelSelection . instanceId ,
2309- threadId : threadEvent . threadId ,
2310- event : {
2311- type : "provider_turn.updated" ,
2312- driver : CODEX_DRIVER ,
2313- providerTurn : {
2314- id : idAllocator . derive . providerTurn ( { driver : CODEX_DRIVER , nativeTurnId : "locked-turn" } ) ,
2315- providerThreadId : idAllocator . derive . providerThread ( {
2316- driver : CODEX_DRIVER ,
2317- nativeThreadId : "locked-thread" ,
2282+ const ingestCompletedProviderTurn = (
2283+ guard : Pick <
2284+ Parameters < ProviderEventIngestor . ProviderEventIngestorV2Shape [ "ingestNormalized" ] > [ 0 ] ,
2285+ "writeIfRunCurrent" | "writeIfProviderThreadOwner"
2286+ > ,
2287+ ) =>
2288+ Effect . gen ( function * ( ) {
2289+ const now = yield * DateTime . now ;
2290+ const eventSink = yield * EventSink . EventSinkV2 ;
2291+ const projectionStore = yield * ProjectionStore . ProjectionStoreV2 ;
2292+ const ingestor = yield * ProviderEventIngestor . ProviderEventIngestorV2 ;
2293+ const idAllocator = yield * IdAllocator . IdAllocatorV2 ;
2294+ const threadEvent = yield * threadCreatedEvent ( now ) ;
2295+ yield * eventSink . write ( { events : [ threadEvent ] } ) ;
2296+ const providerTurnId = ProviderTurnId . make ( "provider-turn:locked" ) ;
2297+ const exit = yield * ingestor
2298+ . ingestNormalized ( {
2299+ providerSessionId : yield * idAllocator . allocate . providerSession ( {
2300+ providerInstanceId : modelSelection . instanceId ,
2301+ threadId : threadEvent . threadId ,
23182302 } ) ,
2319- nodeId : NodeId . make ( "node:locked-turn" ) ,
2320- runAttemptId : null ,
2321- nativeTurnRef : null ,
2322- ordinal : 1 ,
2323- status : "completed" ,
2324- startedAt : now ,
2325- completedAt : now ,
2326- } ,
2327- } ,
2303+ providerInstanceId : modelSelection . instanceId ,
2304+ threadId : threadEvent . threadId ,
2305+ event : {
2306+ type : "provider_turn.updated" ,
2307+ driver : CODEX_DRIVER ,
2308+ providerTurn : {
2309+ id : providerTurnId ,
2310+ providerThreadId : ProviderThreadId . make ( "provider-thread:locked" ) ,
2311+ nodeId : NodeId . make ( "node:locked-turn" ) ,
2312+ runAttemptId : null ,
2313+ nativeTurnRef : null ,
2314+ ordinal : 1 ,
2315+ status : "completed" ,
2316+ startedAt : now ,
2317+ completedAt : now ,
2318+ } ,
2319+ } ,
2320+ ...guard ,
2321+ } )
2322+ . pipe ( Effect . exit ) ;
2323+ const projection = yield * projectionStore . getThreadProjection ( threadEvent . threadId ) ;
2324+ return { exit, providerTurnId, persisted : projection . providerTurns . map ( ( turn ) => turn . id ) } ;
23282325 } ) ;
2329- } ) ;
2330-
2331- it . effect ( "retries a provider event write that timed out on the database lock" , ( ) => {
2332- const { attempts, layer } = layerFailingNextWrites ( [
2333- lockTimeoutWriteError ,
2334- lockTimeoutWriteError ,
2335- ] ) ;
2336- return Effect . gen ( function * ( ) {
2337- const stored = yield * ingestCompletedProviderTurn ;
2338- assert . equal ( attempts . count , 3 ) ;
2339- assert . isNotEmpty ( stored ) ;
2340- } ) . pipe ( Effect . provide ( layer ) ) ;
2341- } ) ;
23422326
2343- it . effect ( "fails a provider event write that stays locked after its retries" , ( ) => {
2344- const { attempts, layer } = layerFailingNextWrites ( [
2345- lockTimeoutWriteError ,
2346- lockTimeoutWriteError ,
2347- lockTimeoutWriteError ,
2348- ] ) ;
2349- return Effect . gen ( function * ( ) {
2350- const error = yield * Effect . flip ( ingestCompletedProviderTurn ) ;
2351- assert . equal ( attempts . count , 3 ) ;
2352- assert . equal ( error . _tag , "ProviderEventPublishError" ) ;
2353- } ) . pipe ( Effect . provide ( layer ) ) ;
2354- } ) ;
2327+ // Neither guard matches a stored run, so a guarded write commits nothing.
2328+ const runId = RunId . make ( "run:locked" ) ;
2329+ const activeAttemptId = RunAttemptId . make ( "attempt:locked" ) ;
2330+ it . effect . each ( [
2331+ { route : "write" , guard : { } } ,
2332+ {
2333+ route : "writeIfRunCurrent" ,
2334+ guard : { writeIfRunCurrent : { runId, activeAttemptId, expectedStatus : "running" } } ,
2335+ } ,
2336+ {
2337+ route : "writeIfProviderThreadOwner" ,
2338+ guard : {
2339+ writeIfProviderThreadOwner : {
2340+ providerThreadId : ProviderThreadId . make ( "provider-thread:locked" ) ,
2341+ runId,
2342+ activeAttemptId,
2343+ expectedLastRunOrdinal : 1 ,
2344+ } ,
2345+ } ,
2346+ } ,
2347+ ] as const ) ( "a provider event $route outlasts a brief database lock" , ( { route, guard } ) =>
2348+ Effect . gen ( function * ( ) {
2349+ const { exit, providerTurnId, persisted } = yield * ingestCompletedProviderTurn ( guard ) ;
2350+ assert . isTrue ( Exit . isSuccess ( exit ) ) ;
2351+ assert . deepEqual ( persisted , route === "write" ? [ providerTurnId ] : [ ] ) ;
2352+ } ) . pipe ( Effect . provide ( layerFailingNextWrites ( [ lockTimeout , lockTimeout ] ) ) ) ,
2353+ ) ;
23552354
2356- it . effect ( "does not retry a provider event write that failed for another reason" , ( ) => {
2357- const { attempts, layer } = layerFailingNextWrites ( [
2358- new EventSink . EventSinkWriteError ( { eventCount : 1 , cause : new Error ( "constraint failed" ) } ) ,
2359- ] ) ;
2360- return Effect . gen ( function * ( ) {
2361- const error = yield * Effect . flip ( ingestCompletedProviderTurn ) ;
2362- assert . equal ( attempts . count , 1 ) ;
2363- assert . equal ( error . _tag , "ProviderEventPublishError" ) ;
2364- } ) . pipe ( Effect . provide ( layer ) ) ;
2365- } ) ;
2355+ it . effect . each ( [
2356+ { failure : "stays locked" , errors : [ lockTimeout , lockTimeout , lockTimeout ] } ,
2357+ { failure : "fails for another reason" , errors : [ constraintFailure ] } ,
2358+ ] ) ( "fails a provider event write that $failure" , ( { errors } ) =>
2359+ Effect . gen ( function * ( ) {
2360+ const { exit, persisted } = yield * ingestCompletedProviderTurn ( { } ) ;
2361+ assert . instanceOf (
2362+ Exit . isFailure ( exit ) ? Cause . squash ( exit . cause ) : undefined ,
2363+ ProviderEventIngestor . ProviderEventPublishError ,
2364+ ) ;
2365+ assert . isEmpty ( persisted ) ;
2366+ } ) . pipe ( Effect . provide ( layerFailingNextWrites ( errors ) ) ) ,
2367+ ) ;
0 commit comments