Skip to content

Commit f03a7fb

Browse files
author
PR Batch Tester
committed
fix(server): wait after failed idle writes before retrying
1 parent 70a2dee commit f03a7fb

2 files changed

Lines changed: 13 additions & 2 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4058,6 +4058,7 @@ it.effect.each([
40584058
const subagentIngested = yield* Deferred.make<void>();
40594059
const rootFinalized = yield* Deferred.make<void>();
40604060
const firstWriteFailed = yield* Deferred.make<void>();
4061+
const failureHandled = yield* Deferred.make<void>();
40614062
const retried = yield* Deferred.make<void>();
40624063
const attempts = yield* Ref.make(0);
40634064
const childWrites = yield* Ref.make<ReadonlyArray<string>>([]);
@@ -4066,7 +4067,9 @@ it.effect.each([
40664067
onIngest: (event) =>
40674068
Effect.gen(function* () {
40684069
if (event.type === "subagent.updated") {
4069-
return yield* Deferred.succeed(subagentIngested, undefined);
4070+
yield* Deferred.succeed(subagentIngested, undefined);
4071+
if ((yield* Ref.get(attempts)) > 0) yield* Deferred.succeed(failureHandled, undefined);
4072+
return;
40704073
}
40714074
if (event.type !== "turn_item.updated" || event.turnItem.type !== "assistant_message")
40724075
return;
@@ -4075,6 +4078,7 @@ it.effect.each([
40754078
(yield* Ref.getAndUpdate(attempts, (n) => n + 1)) === 0
40764079
) {
40774080
yield* Deferred.succeed(firstWriteFailed, undefined);
4081+
yield* Effect.sleep("2 seconds");
40784082
return yield* Effect.die(new Error("disk I/O error"));
40794083
}
40804084
const { text, streaming } = event.turnItem;
@@ -4095,6 +4099,11 @@ it.effect.each([
40954099
}
40964100
yield* TestClock.adjust("1 second");
40974101
yield* Deferred.await(firstWriteFailed);
4102+
yield* TestClock.adjust("2 seconds");
4103+
yield* Queue.offer(events, linkedSubagentEvent(ids, "running"));
4104+
yield* Deferred.await(failureHandled);
4105+
assert.equal(yield* Ref.get(attempts), 1);
4106+
assert.deepEqual(yield* Ref.get(childWrites), []);
40984107
yield* TestClock.adjust("1 second");
40994108
yield* Deferred.await(retried);
41004109
const childText = (text: string, streaming: boolean) => {

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1371,7 +1371,9 @@ export const layer: Layer.Layer<
13711371
failures: idleFlushFailures,
13721372
cause: written.cause,
13731373
});
1374-
const retryAt = now + Math.min(1_000 * 2 ** (idleFlushFailures - 1), 30_000);
1374+
const retryAt =
1375+
DateTime.toEpochMillis(yield* DateTime.now) +
1376+
Math.min(1_000 * 2 ** (idleFlushFailures - 1), 30_000);
13751377
for (const pending of due.slice(index)) pending.retryAt(retryAt);
13761378
break;
13771379
}

0 commit comments

Comments
 (0)