@@ -33,6 +33,7 @@ import * as Effect from "effect/Effect";
3333import * as FileSystem from "effect/FileSystem" ;
3434import * as Layer from "effect/Layer" ;
3535import * as SqlClient from "effect/sql/SqlClient" ;
36+ import * as SqlError from "effect/sql/SqlError" ;
3637import * as Stream from "effect/Stream" ;
3738import * as TestClock from "effect/testing/TestClock" ;
3839
@@ -2242,3 +2243,123 @@ layer("ProviderEventIngestorV2", (it) => {
22422243 } ) ,
22432244 ) ;
22442245} ) ;
2246+
2247+ const lockTimeoutWriteError = new EventSink . EventSinkWriteError ( {
2248+ eventCount : 1 ,
2249+ cause : new SqlError . SqlError ( {
2250+ reason : new SqlError . LockTimeoutError ( {
2251+ cause : new Error ( "database is locked" ) ,
2252+ message : "database is locked" ,
2253+ } ) ,
2254+ } ) ,
2255+ } ) ;
2256+
2257+ /** The ingestor sees an event sink whose next writes fail with `errors`, in order. */
2258+ function layerFailingNextWrites ( errors : ReadonlyArray < EventSink . EventSinkV2Error > ) {
2259+ const attempts = { count : 0 } ;
2260+ const layerFailingEventSink = Layer . effect (
2261+ EventSink . EventSinkV2 ,
2262+ Effect . gen ( function * ( ) {
2263+ const eventSink = yield * EventSink . EventSinkV2 ;
2264+ return EventSink . EventSinkV2 . of ( {
2265+ ...eventSink ,
2266+ write : ( input ) =>
2267+ Effect . suspend ( ( ) => {
2268+ const error = errors [ attempts . count ] ;
2269+ attempts . count += 1 ;
2270+ return error === undefined ? eventSink . write ( input ) : Effect . fail ( error ) ;
2271+ } ) ,
2272+ } ) ;
2273+ } ) ,
2274+ ) . pipe ( Layer . provide ( layerTestEventSink ) ) ;
2275+ const layer = Layer . mergeAll (
2276+ layerTestDatabase ,
2277+ layerTestStores ,
2278+ layerTestEventSink ,
2279+ IdAllocator . layer ,
2280+ ProviderEventIngestor . layer . pipe (
2281+ Layer . provide (
2282+ Layer . mergeAll (
2283+ layerTestStores ,
2284+ layerFailingEventSink ,
2285+ IdAllocator . layer ,
2286+ ThreadCommandExecutor . layer ,
2287+ layerGit ,
2288+ ) ,
2289+ ) ,
2290+ ) ,
2291+ ) ;
2292+ return { attempts, layer } ;
2293+ }
2294+
2295+ const ingestCompletedProviderTurn = Effect . gen ( function * ( ) {
2296+ const now = yield * DateTime . now ;
2297+ const eventSink = yield * EventSink . EventSinkV2 ;
2298+ const ingestor = yield * ProviderEventIngestor . ProviderEventIngestorV2 ;
2299+ const idAllocator = yield * IdAllocator . IdAllocatorV2 ;
2300+ const threadEvent = yield * threadCreatedEvent ( now ) ;
2301+ yield * eventSink . write ( { events : [ threadEvent ] } ) ;
2302+ const providerSessionId = yield * idAllocator . allocate . providerSession ( {
2303+ providerInstanceId : modelSelection . instanceId ,
2304+ threadId : threadEvent . threadId ,
2305+ } ) ;
2306+ return yield * ingestor . ingestNormalized ( {
2307+ providerSessionId,
2308+ providerInstanceId : modelSelection . instanceId ,
2309+ threadId : threadEvent . threadId ,
2310+ event : {
2311+ type : "provider_turn.updated" ,
2312+ driver : CODEX_DRIVER ,
2313+ providerTurn : {
2314+ id : idAllocator . derive . providerTurn ( { driver : CODEX_DRIVER , nativeTurnId : "locked-turn" } ) ,
2315+ providerThreadId : idAllocator . derive . providerThread ( {
2316+ driver : CODEX_DRIVER ,
2317+ nativeThreadId : "locked-thread" ,
2318+ } ) ,
2319+ nodeId : NodeId . make ( "node:locked-turn" ) ,
2320+ runAttemptId : null ,
2321+ nativeTurnRef : null ,
2322+ ordinal : 1 ,
2323+ status : "completed" ,
2324+ startedAt : now ,
2325+ completedAt : now ,
2326+ } ,
2327+ } ,
2328+ } ) ;
2329+ } ) ;
2330+
2331+ it . effect ( "retries a provider event write that timed out on the database lock" , ( ) => {
2332+ const { attempts, layer } = layerFailingNextWrites ( [
2333+ lockTimeoutWriteError ,
2334+ lockTimeoutWriteError ,
2335+ ] ) ;
2336+ return Effect . gen ( function * ( ) {
2337+ const stored = yield * ingestCompletedProviderTurn ;
2338+ assert . equal ( attempts . count , 3 ) ;
2339+ assert . isNotEmpty ( stored ) ;
2340+ } ) . pipe ( Effect . provide ( layer ) ) ;
2341+ } ) ;
2342+
2343+ it . effect ( "fails a provider event write that stays locked after its retries" , ( ) => {
2344+ const { attempts, layer } = layerFailingNextWrites ( [
2345+ lockTimeoutWriteError ,
2346+ lockTimeoutWriteError ,
2347+ lockTimeoutWriteError ,
2348+ ] ) ;
2349+ return Effect . gen ( function * ( ) {
2350+ const error = yield * Effect . flip ( ingestCompletedProviderTurn ) ;
2351+ assert . equal ( attempts . count , 3 ) ;
2352+ assert . equal ( error . _tag , "ProviderEventPublishError" ) ;
2353+ } ) . pipe ( Effect . provide ( layer ) ) ;
2354+ } ) ;
2355+
2356+ it . effect ( "does not retry a provider event write that failed for another reason" , ( ) => {
2357+ const { attempts, layer } = layerFailingNextWrites ( [
2358+ new EventSink . EventSinkWriteError ( { eventCount : 1 , cause : new Error ( "constraint failed" ) } ) ,
2359+ ] ) ;
2360+ return Effect . gen ( function * ( ) {
2361+ const error = yield * Effect . flip ( ingestCompletedProviderTurn ) ;
2362+ assert . equal ( attempts . count , 1 ) ;
2363+ assert . equal ( error . _tag , "ProviderEventPublishError" ) ;
2364+ } ) . pipe ( Effect . provide ( layer ) ) ;
2365+ } ) ;
0 commit comments