Skip to content

Commit eb11286

Browse files
Merge remote-tracking branch 'origin/main' into fix/issue-18165
2 parents d25e61b + 36d4d71 commit eb11286

2 files changed

Lines changed: 112 additions & 1 deletion

File tree

‎apps/server/src/orchestration-v2/EffectWorker.ts‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -562,6 +562,29 @@ export const layerExecutor: Layer.Layer<
562562
}),
563563
);
564564

565+
// A running effect blocks later work on its thread and its lease is never reclaimed in-process,
566+
// so a stop that hangs must give up and fall through to the normal retry/fail path.
567+
const DELEGATED_TASKS_STOP_TIMEOUT = Duration.seconds(30);
568+
569+
const boundExecution =
570+
(effect: EffectOutbox.OrchestrationEffectV2) =>
571+
(execution: Effect.Effect<void, OrchestrationEffectExecutionError>) =>
572+
effect.request.type === "delegated-tasks.stop"
573+
? execution.pipe(
574+
Effect.timeoutOrElse({
575+
duration: DELEGATED_TASKS_STOP_TIMEOUT,
576+
orElse: () =>
577+
Effect.fail(
578+
new OrchestrationEffectExecutionError({
579+
effectId: effect.id,
580+
effectType: effect.request.type,
581+
cause: `Timed out after ${Duration.format(DELEGATED_TASKS_STOP_TIMEOUT)}.`,
582+
}),
583+
),
584+
}),
585+
)
586+
: execution;
587+
565588
export class OrchestrationEffectWorkerError extends Schema.TaggedError<OrchestrationEffectWorkerError>()(
566589
"OrchestrationEffectWorkerError",
567590
{
@@ -747,7 +770,7 @@ export const layerWithOptions = (
747770

748771
const execution = executor
749772
.execute(effect, { willRetry: effect.attemptCount < maxAttempts })
750-
.pipe(Effect.as("executed" as const));
773+
.pipe(boundExecution(effect), Effect.as("executed" as const));
751774
const exit = yield* Effect.exit(Effect.raceFirst(execution, cancellation)).pipe(
752775
Effect.ensuring(outbox.clearCancellation(effect.id)),
753776
);

‎apps/server/src/orchestration-v2/FoundationPersistence.test.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2594,6 +2594,94 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
25942594
}),
25952595
);
25962596

2597+
it.effect.each([{ replayed: false }, { replayed: true }] as const)(
2598+
"starts later thread work behind a delegated-task stop that never returns (replayed: $replayed)",
2599+
({ replayed }) =>
2600+
Effect.gen(function* () {
2601+
const outbox = yield* EffectOutbox.EffectOutboxV2;
2602+
const key = `foundation-hung-delegated-stop:${replayed}`;
2603+
const threadId = ThreadId.make(`thread:${key}`);
2604+
const stopEffectId = `effect:${key}:stop`;
2605+
const stopStarted = yield* Deferred.make<void>();
2606+
const turnStarted = yield* Deferred.make<void>();
2607+
yield* outbox.enqueue([
2608+
{
2609+
id: stopEffectId,
2610+
commandId: CommandId.make(`command:${key}:stop`),
2611+
threadId,
2612+
request: { type: "delegated-tasks.stop" },
2613+
},
2614+
]);
2615+
if (replayed) {
2616+
assert.isTrue(
2617+
Option.isSome(
2618+
yield* outbox.claimNext({ workerId: "crashed-worker", leaseDurationMs: 30_000 }),
2619+
),
2620+
);
2621+
assert.equal((yield* outbox.reconcileAfterProcessLoss).requeued, 1);
2622+
}
2623+
2624+
const layerExecutor = Layer.succeed(
2625+
EffectWorker.OrchestrationEffectExecutorV2,
2626+
EffectWorker.OrchestrationEffectExecutorV2.of({
2627+
execute: (effect) =>
2628+
effect.request.type === "delegated-tasks.stop"
2629+
? Deferred.succeed(stopStarted, undefined).pipe(Effect.andThen(Effect.never))
2630+
: Deferred.succeed(turnStarted, undefined),
2631+
}),
2632+
);
2633+
const layerWorker = EffectWorker.layerWithOptions({
2634+
workerId: "hung-delegated-stop-worker",
2635+
}).pipe(
2636+
Layer.provide(
2637+
Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
2638+
),
2639+
);
2640+
2641+
yield* Effect.gen(function* () {
2642+
yield* EffectWorker.runDaemonWithOptions({ concurrency: 2 }).pipe(Effect.forkScoped);
2643+
yield* Deferred.await(stopStarted);
2644+
yield* outbox.enqueue([
2645+
{
2646+
id: `effect:${key}:turn`,
2647+
commandId: CommandId.make(`command:${key}:turn`),
2648+
threadId,
2649+
request: { type: "provider-turn.start", runId: RunId.make(`run:${key}`) },
2650+
},
2651+
]);
2652+
yield* outbox.notifyAvailable();
2653+
2654+
const turn = yield* Deferred.await(turnStarted).pipe(
2655+
Effect.timeoutOption("1 hour"),
2656+
Effect.forkChild,
2657+
);
2658+
yield* TestClock.adjust("1 hour");
2659+
assert.isTrue(
2660+
Option.isSome(yield* Fiber.join(turn)),
2661+
"provider-turn.start never ran behind the hung delegated-tasks.stop",
2662+
);
2663+
const stop = yield* outbox.get(stopEffectId);
2664+
assert.isTrue(Option.isSome(stop));
2665+
if (Option.isSome(stop)) {
2666+
assert.equal(stop.value.status, "failed");
2667+
assert.include(stop.value.lastError ?? "", "Timed out");
2668+
}
2669+
}).pipe(
2670+
Effect.provide(layerWorker),
2671+
Effect.scoped,
2672+
Effect.ensuring(
2673+
outbox
2674+
.cancelUnsettled({
2675+
threadId,
2676+
effectTypes: ["delegated-tasks.stop", "provider-turn.start"],
2677+
reason: "fixture complete",
2678+
})
2679+
.pipe(Effect.orDie),
2680+
),
2681+
);
2682+
}),
2683+
);
2684+
25972685
it.effect(
25982686
"persists shutdown continuation intent through the real event sink without domain events",
25992687
() =>

0 commit comments

Comments
 (0)