Skip to content

Commit 6108ef3

Browse files
t3dotggclaude
andauthored
fix(server): runs no longer get stuck (pingdotgg#15048)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent fed41fa commit 6108ef3

9 files changed

Lines changed: 959 additions & 115 deletions

‎apps/server/src/orchestration-v2/EffectOutbox.ts‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -278,9 +278,16 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
278278
available,
279279
Array.from({ length: Math.min(64, Math.max(0, Math.floor(count))) }, () => undefined),
280280
).pipe(Effect.asVoid);
281+
// Each thread runs its effects one at a time, in enqueue (rowid) order. An earlier
282+
// effect waiting out a retry backoff still blocks later ones, so a turn
283+
// cannot start while a failed rollback is about to restore files. A claim
284+
// that skips restart continuations is not blocked by them either.
281285
// Title generation is correlated metadata work, so it has its own
282286
// per-thread lane and cannot delay provider lifecycle effects.
283-
const claimableCandidatePredicate = (availableBefore?: string) =>
287+
const claimableCandidatePredicate = (
288+
availableBefore?: string,
289+
excludeRestartContinuations = false,
290+
) =>
284291
sql`
285292
${
286293
availableBefore === undefined
@@ -292,7 +299,18 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
292299
SELECT 1
293300
FROM orchestration_v2_effect_outbox AS active
294301
WHERE active.thread_id = candidate.thread_id
295-
AND active.status = 'running'
302+
AND (
303+
active.status = 'running'
304+
OR (
305+
active.status = 'pending'
306+
AND active.rowid < candidate.rowid
307+
AND ${
308+
excludeRestartContinuations
309+
? sql`active.effect_type != 'provider-runtime.continue'`
310+
: sql`1 = 1`
311+
}
312+
)
313+
)
296314
AND (
297315
(
298316
candidate.effect_type = 'thread-title.generate'
@@ -492,7 +510,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
492510
WHERE effect_id = (
493511
SELECT candidate.effect_id
494512
FROM orchestration_v2_effect_outbox AS candidate
495-
WHERE ${claimableCandidatePredicate(nowIso)}
513+
WHERE ${claimableCandidatePredicate(nowIso, excludeRestartContinuations)}
496514
AND ${excludeRestartContinuations ? sql`candidate.effect_type != 'provider-runtime.continue'` : sql`1 = 1`}
497515
ORDER BY candidate.available_at ASC, candidate.created_at ASC, candidate.effect_id ASC
498516
LIMIT 1

‎apps/server/src/orchestration-v2/EventSink.ts‎

Lines changed: 59 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import * as Effect from "effect/Effect";
1717
import * as Layer from "effect/Layer";
1818
import * as Option from "effect/Option";
1919
import * as PubSub from "effect/PubSub";
20+
import * as Semaphore from "effect/Semaphore";
2021
import * as Schema from "effect/Schema";
2122
import * as Stream from "effect/Stream";
2223
import * as SqlClient from "effect/unstable/sql/SqlClient";
@@ -224,6 +225,40 @@ const baseLayer: Layer.Layer<
224225
);
225226
}
226227
});
228+
const publishStoredEvents = (events: ReadonlyArray<OrchestrationV2StoredEvent>) =>
229+
eventStore.publishCommitted(events).pipe(Effect.andThen(publishLiveEvents(events)));
230+
231+
// Transactions commit one at a time, but each writer publishes after its
232+
// commit. If a writer is descheduled in between, a later commit reaches
233+
// subscribers first, and clients drop any event at or below the newest
234+
// sequence they have applied. So a writer takes this lane as the last step
235+
// of its transaction and holds it until it has published. Publishing never
236+
// waits, so a writer that holds the transaction while it waits for the
237+
// lane is not blocked for long.
238+
const publishLane = yield* Semaphore.make(1);
239+
const commitThenPublish = <A, E, R>(
240+
transaction: Effect.Effect<A, E, R>,
241+
publish: (committed: A) => Effect.Effect<void>,
242+
) =>
243+
Effect.suspend(() => {
244+
let holdsLane = false;
245+
const takeLane = publishLane.take(1).pipe(
246+
Effect.andThen(
247+
Effect.sync(() => {
248+
holdsLane = true;
249+
}),
250+
),
251+
Effect.uninterruptible,
252+
);
253+
return sql
254+
.withTransaction(Effect.tap(transaction, () => takeLane))
255+
.pipe(
256+
Effect.tap(publish),
257+
Effect.ensuring(
258+
Effect.suspend(() => (holdsLane ? publishLane.release(1) : Effect.void)),
259+
),
260+
);
261+
});
227262

228263
// A user can answer after terminal normalization reads the pending request.
229264
// Recheck inside the write transaction so stale cleanup cannot erase answers.
@@ -332,7 +367,7 @@ const baseLayer: Layer.Layer<
332367
"orchestration_v2.thread_id": input.events[0]?.threadId ?? null,
333368
});
334369

335-
const storedEvents = yield* sql.withTransaction(
370+
return yield* commitThenPublish(
336371
Effect.gen(function* () {
337372
const normalized = yield* normalizeEvents(
338373
input.guardPendingUserInputCancellations === true
@@ -347,13 +382,14 @@ const baseLayer: Layer.Layer<
347382
yield* effectOutbox.enqueue(input.effects);
348383
return committed;
349384
}),
385+
(storedEvents) =>
386+
Effect.gen(function* () {
387+
if (input.effects.length > 0) {
388+
yield* effectOutbox.notifyAvailable(input.effects.length);
389+
}
390+
yield* publishStoredEvents(storedEvents);
391+
}),
350392
);
351-
if (input.effects.length > 0) {
352-
yield* effectOutbox.notifyAvailable(input.effects.length);
353-
}
354-
yield* eventStore.publishCommitted(storedEvents);
355-
yield* publishLiveEvents(storedEvents);
356-
return storedEvents;
357393
});
358394

359395
const writeIfRunCurrentEffect = Effect.fn("orchestrationV2.EventSink.writeIfRunCurrent")(
@@ -365,7 +401,7 @@ const baseLayer: Layer.Layer<
365401
"orchestration_v2.thread_id": input.threadId,
366402
});
367403

368-
const result = yield* sql.withTransaction(
404+
return yield* commitThenPublish(
369405
Effect.gen(function* () {
370406
const rows = yield* sql<{
371407
readonly status: string;
@@ -403,12 +439,8 @@ const baseLayer: Layer.Layer<
403439
yield* applyStoredEvents(storedEvents);
404440
return { committed: true as const, storedEvents };
405441
}),
442+
(result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void),
406443
);
407-
if (result.committed) {
408-
yield* eventStore.publishCommitted(result.storedEvents);
409-
yield* publishLiveEvents(result.storedEvents);
410-
}
411-
return result;
412444
},
413445
);
414446

@@ -424,7 +456,7 @@ const baseLayer: Layer.Layer<
424456
"orchestration_v2.expected_last_run_ordinal": input.expectedLastRunOrdinal,
425457
});
426458

427-
const result = yield* sql.withTransaction(
459+
return yield* commitThenPublish(
428460
Effect.gen(function* () {
429461
const rows = yield* sql<{
430462
readonly active_attempt_id: string | null;
@@ -464,12 +496,8 @@ const baseLayer: Layer.Layer<
464496
yield* applyStoredEvents(storedEvents);
465497
return { committed: true as const, storedEvents };
466498
}),
499+
(result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void),
467500
);
468-
if (result.committed) {
469-
yield* eventStore.publishCommitted(result.storedEvents);
470-
yield* publishLiveEvents(result.storedEvents);
471-
}
472-
return result;
473501
});
474502

475503
const existingCommandResult = (commandId: CommandId) =>
@@ -490,7 +518,7 @@ const baseLayer: Layer.Layer<
490518
const commitCommandEffect = Effect.fn("orchestrationV2.EventSink.commitCommand")(function* (
491519
input: Parameters<EventSinkV2Shape["commitCommand"]>[0],
492520
) {
493-
const result = yield* sql.withTransaction(
521+
const result = yield* commitThenPublish(
494522
Effect.gen(function* () {
495523
const reserved = yield* commandReceipts.insertIfAbsent({
496524
commandId: input.commandId,
@@ -538,15 +566,15 @@ const baseLayer: Layer.Layer<
538566
});
539567
return { receipt, storedEvents, committed: true as const, cancelledEffectIds };
540568
}),
569+
(result) =>
570+
Effect.gen(function* () {
571+
yield* effectOutbox.signalCancellations(result.cancelledEffectIds);
572+
if (result.committed && input.effects.length > 0) {
573+
yield* effectOutbox.notifyAvailable(input.effects.length);
574+
}
575+
if (result.committed) yield* publishStoredEvents(result.storedEvents);
576+
}),
541577
);
542-
yield* effectOutbox.signalCancellations(result.cancelledEffectIds);
543-
if (result.committed && input.effects.length > 0) {
544-
yield* effectOutbox.notifyAvailable(input.effects.length);
545-
}
546-
if (result.committed) {
547-
yield* eventStore.publishCommitted(result.storedEvents);
548-
yield* publishLiveEvents(result.storedEvents);
549-
}
550578
return {
551579
receipt: result.receipt,
552580
storedEvents: result.storedEvents,
@@ -593,7 +621,7 @@ const baseLayer: Layer.Layer<
593621

594622
const commitProjectCommandEffect = Effect.fn("orchestrationV2.EventSink.commitProjectCommand")(
595623
function* (input: Parameters<EventSinkV2Shape["commitProjectCommand"]>[0]) {
596-
const result = yield* sql.withTransaction(
624+
const result = yield* commitThenPublish(
597625
Effect.gen(function* () {
598626
const reserved: CommandReceiptStore.ProjectCommandReceiptV2 = {
599627
commandId: input.commandId,
@@ -613,10 +641,9 @@ const baseLayer: Layer.Layer<
613641
yield* commandReceipts.upsert(receipt);
614642
return { receipt, event };
615643
}),
644+
(result) =>
645+
result.event === undefined ? Effect.void : eventStore.publishCommitted([result.event]),
616646
);
617-
if (result.event !== undefined) {
618-
yield* eventStore.publishCommitted([result.event]);
619-
}
620647
return { receipt: result.receipt, committed: result.event !== undefined };
621648
},
622649
);

0 commit comments

Comments
 (0)