Skip to content

Commit fb8d99b

Browse files
author
PR Batch Tester
committed
fix(server): limit repeated idle storage warnings
1 parent 5fe1333 commit fb8d99b

1 file changed

Lines changed: 11 additions & 5 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1389,6 +1389,7 @@ export const layer: Layer.Layer<
13891389
// A failed idle write keeps its text held and retries with backoff. The provider's next
13901390
// update still writes, and fails ingestion, the normal way.
13911391
let idleFlushFailures = 0;
1392+
let nextIdleFlushWarningAt = Number.NEGATIVE_INFINITY;
13921393
const flushDueAssistantText = ingestLock.withPermits(1)(
13931394
Effect.gen(function* () {
13941395
if (!idleFlushClosed) {
@@ -1406,18 +1407,23 @@ export const layer: Layer.Layer<
14061407
if (Exit.isFailure(written)) {
14071408
if (Cause.hasInterruptsOnly(written.cause)) return yield* written;
14081409
idleFlushFailures += 1;
1409-
yield* Effect.logWarning("orchestration V2 idle assistant text flush failed", {
1410-
runId: input.run.id,
1411-
failures: idleFlushFailures,
1412-
cause: written.cause,
1413-
});
1410+
const failedAt = DateTime.toEpochMillis(yield* DateTime.now);
1411+
if (failedAt >= nextIdleFlushWarningAt) {
1412+
yield* Effect.logWarning("orchestration V2 idle assistant text flush failed", {
1413+
runId: input.run.id,
1414+
failures: idleFlushFailures,
1415+
cause: written.cause,
1416+
});
1417+
nextIdleFlushWarningAt = failedAt + 10 * 60_000;
1418+
}
14141419
const retryAt =
14151420
DateTime.toEpochMillis(yield* DateTime.now) +
14161421
Math.min(1_000 * 2 ** (idleFlushFailures - 1), 30_000);
14171422
for (const pending of due.slice(index)) pending.retryAt(retryAt);
14181423
break;
14191424
}
14201425
idleFlushFailures = 0;
1426+
nextIdleFlushWarningAt = Number.NEGATIVE_INFINITY;
14211427
flush.delivered(now);
14221428
yield* trackChildLifecycle(flush.event, true);
14231429
}

0 commit comments

Comments
 (0)