Skip to content

Commit ef80b84

Browse files
fix(client-runtime): queued runs that start after a steer show up in the thread
A queued run keeps the run ordinal it got when it was queued, and the server bands its turn items by that ordinal. A steer or new turn sent later gets a higher ordinal and runs first, and on a partially loaded thread its items raise the client's latestLocalTurnOrdinal watermark. When the queued run finally started, every one of its items sat below the watermark, so the partial-timeline guard treated them as unloaded history and dropped them. The resume cursor still moved past those events and the thread cache kept the gap, so reloading did not bring them back. The guard now keeps a missing item when its run is still queued in the client's projection (the orchestrator commits a queued run's first item before the run.updated that starts it), and when the window already shows an earlier item of the same run (which also covers #16437's case). Items of other runs below the window are still dropped. Thread cache entries now carry their own schema version (4), so web and mobile discard thread snapshots saved with the gap and fetch a fresh bounded snapshot. Shell caches keep version 3 and stay warm. Fixes #16987 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 57b3780 commit ef80b84

7 files changed

Lines changed: 172 additions & 7 deletions

File tree

‎apps/mobile/src/connection/environment-cache-store.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
ORCHESTRATION_CACHE_SCHEMA_VERSION,
3+
ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
34
StoredOrchestrationShellSnapshot,
45
} from "@t3tools/client-runtime/platform";
56
import {
@@ -251,7 +252,7 @@ describe("mobile SQLite environment cache store", () => {
251252
ORCHESTRATION_CACHE_SCHEMA_VERSION,
252253
);
253254
expect(memory.schemaVersions.get(cacheId(ENVIRONMENT_ID, "thread", THREAD_ID))).toBe(
254-
ORCHESTRATION_CACHE_SCHEMA_VERSION,
255+
ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
255256
);
256257
}),
257258
);

‎apps/mobile/src/connection/environment-cache-store.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
ORCHESTRATION_CACHE_SCHEMA_VERSION,
3+
ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
34
StoredOrchestrationThreadSnapshot,
45
Persistence,
56
} from "@t3tools/client-runtime/platform";
@@ -148,13 +149,19 @@ export const make = Effect.fn("MobileEnvironmentCacheStore.make")(function* () {
148149
saveThread: Effect.fn("MobileEnvironmentCache.saveThread")(function* (environmentId, snapshot) {
149150
const threadId = snapshot.projection.thread.id;
150151
const payload = yield* encodeStoredThreadSnapshot({
151-
schemaVersion: ORCHESTRATION_CACHE_SCHEMA_VERSION,
152+
schemaVersion: ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
152153
environmentId,
153154
threadId,
154155
snapshot,
155156
}).pipe(Effect.mapError((cause) => persistenceError("save-thread", cause)));
156157
yield* database
157-
.saveCache(environmentId, "thread", threadId, ORCHESTRATION_CACHE_SCHEMA_VERSION, payload)
158+
.saveCache(
159+
environmentId,
160+
"thread",
161+
threadId,
162+
ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
163+
payload,
164+
)
158165
.pipe(Effect.mapError(mapDatabaseError("save-thread")));
159166
}),
160167
removeThread: Effect.fn("MobileEnvironmentCache.removeThread")((environmentId, threadId) =>

‎apps/web/src/connection/storage.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
type ConnectionCatalogDocument as ConnectionCatalogDocumentType,
44
EMPTY_CONNECTION_CATALOG_DOCUMENT,
55
ORCHESTRATION_CACHE_SCHEMA_VERSION,
6+
ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
67
StoredOrchestrationShellSnapshot,
78
StoredOrchestrationThreadSnapshot,
89
decodeOrDiscardOrchestrationCache,
@@ -810,7 +811,7 @@ export const layer = Layer.effectContext(
810811
saveThread: (environmentId, snapshot) =>
811812
Effect.gen(function* () {
812813
const encoded = yield* encodeStoredThreadSnapshot({
813-
schemaVersion: ORCHESTRATION_CACHE_SCHEMA_VERSION,
814+
schemaVersion: ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
814815
environmentId,
815816
threadId: snapshot.projection.thread.id,
816817
snapshot,

‎packages/client-runtime/src/platform/orchestrationCache.test.ts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import * as Schema from "effect/Schema";
1515

1616
import {
1717
ORCHESTRATION_CACHE_SCHEMA_VERSION,
18+
ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
1819
StoredOrchestrationShellSnapshot,
1920
StoredOrchestrationThreadSnapshot,
2021
decodeOrDiscardOrchestrationCache,
@@ -102,7 +103,7 @@ describe("orchestration cache envelopes", () => {
102103
expect(projection).not.toBeNull();
103104
if (projection === null) throw new Error("Expected live notice projection");
104105
const encoded = encodeStoredThreadSnapshotJson({
105-
schemaVersion: ORCHESTRATION_CACHE_SCHEMA_VERSION,
106+
schemaVersion: ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
106107
environmentId,
107108
threadId: v2ThreadId,
108109
snapshot: { snapshotSequence: 5, projection },
@@ -121,7 +122,7 @@ describe("orchestration cache envelopes", () => {
121122
});
122123
const shell = decodeStoredShellSnapshotJson(encodedShell);
123124
const encodedThread = encodeStoredThreadSnapshotJson({
124-
schemaVersion: ORCHESTRATION_CACHE_SCHEMA_VERSION,
125+
schemaVersion: ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
125126
environmentId,
126127
threadId: v2ThreadId,
127128
snapshot: { snapshotSequence: 4, projection: v2Projection },
@@ -159,6 +160,23 @@ describe("orchestration cache envelopes", () => {
159160
}),
160161
);
161162

163+
it("rejects thread snapshots saved before queued-run items were kept, but not shells", () => {
164+
// Version 3 thread windows may have dropped a queued run's items (#16987).
165+
const staleThread = encodeStoredThreadSnapshotJson({
166+
schemaVersion: ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION,
167+
environmentId,
168+
threadId: v2ThreadId,
169+
snapshot: { snapshotSequence: 4, projection: v2Projection },
170+
}).replace(`"schemaVersion":${ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION}`, `"schemaVersion":3`);
171+
expect(() => decodeStoredThreadSnapshotJson(staleThread)).toThrow();
172+
const shell = encodeStoredShellSnapshot({
173+
schemaVersion: ORCHESTRATION_CACHE_SCHEMA_VERSION,
174+
environmentId,
175+
snapshot: v2ShellSnapshot,
176+
});
177+
expect(decodeStoredShellSnapshotSync(shell).schemaVersion).toBe(3);
178+
});
179+
162180
it.effect("discards V1-versioned cache envelopes after a decode failure", () =>
163181
Effect.gen(function* () {
164182
let discardCount = 0;

‎packages/client-runtime/src/platform/orchestrationCache.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ import * as Option from "effect/Option";
1010
import * as Schema from "effect/Schema";
1111

1212
export const ORCHESTRATION_CACHE_SCHEMA_VERSION = 3 as const;
13+
/**
14+
* Thread snapshots version on their own so a thread-only invalidation keeps
15+
* shell caches warm. 4 drops bounded windows that lost queued runs' items (#16987).
16+
*/
17+
export const ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION = 4 as const;
1318

1419
export const StoredOrchestrationShellSnapshot = Schema.Struct({
1520
schemaVersion: Schema.Literal(ORCHESTRATION_CACHE_SCHEMA_VERSION),
@@ -18,7 +23,7 @@ export const StoredOrchestrationShellSnapshot = Schema.Struct({
1823
});
1924

2025
export const StoredOrchestrationThreadSnapshot = Schema.Struct({
21-
schemaVersion: Schema.Literal(ORCHESTRATION_CACHE_SCHEMA_VERSION),
26+
schemaVersion: Schema.Literal(ORCHESTRATION_THREAD_CACHE_SCHEMA_VERSION),
2227
environmentId: EnvironmentId,
2328
threadId: ThreadId,
2429
snapshot: OrchestrationV2ThreadDetailSnapshot.mapFields((fields) => ({

‎packages/client-runtime/src/state/orchestrationV2Projection.test.ts‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,111 @@ describe("applyOrchestrationV2ProjectionEvent", () => {
275275
expect(next?.visibleTurnItems.map((row) => row.position)).toEqual([0, 1]);
276276
});
277277

278+
describe("partial timeline", () => {
279+
const queuedRunId = RunId.make("run-queued");
280+
const steerRunId = RunId.make("run-steer");
281+
const queuedRun = { ...run, id: queuedRunId, ordinal: 2, status: "queued" } as const;
282+
const steerRun = { ...run, id: steerRunId, ordinal: 3, status: "completed" } as const;
283+
const runItem = (id: string, itemRunId: RunId, ordinal: number) => ({
284+
...commandItem(id, id, ordinal),
285+
runId: itemRunId,
286+
});
287+
const windowOf = (
288+
runs: ReadonlyArray<OrchestrationV2Run>,
289+
items: ReadonlyArray<OrchestrationV2TurnItem>,
290+
): OrchestrationV2ThreadProjection => ({
291+
...emptyProjection,
292+
runs,
293+
turnItems: items,
294+
visibleTurnItems: items.map((item, position) => ({
295+
position,
296+
visibility: "local" as const,
297+
sourceThreadId: threadId,
298+
sourceItemId: item.id,
299+
item,
300+
})),
301+
});
302+
const itemEvent = (payload: OrchestrationV2TurnItem) =>
303+
({
304+
id: `event-${payload.id}`,
305+
type: "turn-item.updated",
306+
threadId,
307+
occurredAt: now,
308+
payload,
309+
}) as OrchestrationV2DomainEvent;
310+
const runEvent = (payload: OrchestrationV2Run) =>
311+
({
312+
id: `event-${payload.id}-${payload.status}`,
313+
type: "run.updated",
314+
threadId,
315+
occurredAt: now,
316+
payload,
317+
}) as OrchestrationV2DomainEvent;
318+
319+
it("shows a queued run that starts after a newer steer raised the watermark", () => {
320+
// Run 2 was queued first, run 3 was steered in and ran first. Its items
321+
// sit in run 3's band and already raised the watermark past run 2's band.
322+
const steerItems = [
323+
runItem("item-steer-user", steerRunId, 3_000_001),
324+
runItem("item-steer-reply", steerRunId, 3_000_002),
325+
];
326+
let projection: OrchestrationV2ThreadProjection | null = windowOf(
327+
[run, queuedRun, steerRun],
328+
steerItems,
329+
);
330+
let latestLocalTurnOrdinal = 3_000_002;
331+
// Same order the orchestrator commits a queued start: the user message
332+
// lands while the run is still queued, then the run starts and replies.
333+
const queuedUser = runItem("item-queued-user", queuedRunId, 2_000_001);
334+
const queuedReply = runItem("item-queued-reply", queuedRunId, 2_000_002);
335+
for (const event of [
336+
itemEvent(queuedUser),
337+
runEvent({ ...queuedRun, status: "running" }),
338+
itemEvent(queuedReply),
339+
runEvent({ ...queuedRun, status: "completed" }),
340+
]) {
341+
projection = applyOrchestrationV2ProjectionEvent(projection, event, {
342+
partialTimeline: true,
343+
latestLocalTurnOrdinal,
344+
});
345+
if (event.type === "turn-item.updated") {
346+
latestLocalTurnOrdinal = Math.max(latestLocalTurnOrdinal, event.payload.ordinal);
347+
}
348+
}
349+
350+
expect(projection?.visibleTurnItems.map((row) => row.item.id)).toEqual([
351+
queuedUser.id,
352+
queuedReply.id,
353+
...steerItems.map((item) => item.id),
354+
]);
355+
expect(projection?.turnItems.map((item) => item.id)).toContain(queuedReply.id);
356+
});
357+
358+
it.each([
359+
{ name: "keeps an item of a run already in the window", itemRunId: steerRunId, kept: true },
360+
{ name: "drops an item of a finished run outside the window", itemRunId: runId, kept: false },
361+
])("$name that arrives below the watermark", ({ itemRunId, kept }) => {
362+
const first = runItem("item-first", steerRunId, 3_000_001);
363+
const later = runItem("item-later", steerRunId, 3_000_003);
364+
const late = runItem("item-late", itemRunId, 3_000_002);
365+
const projection = windowOf([run, steerRun], [first, later]);
366+
367+
const next = applyOrchestrationV2ProjectionEvent(projection, itemEvent(late), {
368+
partialTimeline: true,
369+
latestLocalTurnOrdinal: later.ordinal,
370+
});
371+
if (kept) {
372+
expect(next?.visibleTurnItems.map((row) => row.item.id)).toEqual([
373+
first.id,
374+
late.id,
375+
later.id,
376+
]);
377+
} else {
378+
expect(next).toBe(projection);
379+
}
380+
});
381+
});
382+
278383
it("removes only hidden local items while preserving inherited rows", () => {
279384
const inherited = commandItem("item-inherited");
280385
const local = commandItem("item-local");

‎packages/client-runtime/src/state/orchestrationV2Projection.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,34 @@ function shouldDropMissingPartialTurnItem(
9494
if (projection.visibleTurnItems.some((row) => row.sourceItemId === item.id)) {
9595
return false;
9696
}
97+
if (item.runId === null) {
98+
return isBelowPartialWindow(projection, item, latestLocalTurnOrdinal);
99+
}
100+
// A queued run keeps the ordinal it got when queued, so a steer sent later
101+
// can raise the watermark past it. The orchestrator commits its first item
102+
// while the run is still queued, so that item is new, not unloaded history.
103+
if (projection.runs.some((run) => run.id === item.runId && run.status === "queued")) {
104+
return false;
105+
}
106+
// Later items of a run already in the window can raise the watermark first.
107+
if (
108+
projection.visibleTurnItems.some(
109+
(row) =>
110+
row.visibility === "local" &&
111+
row.item.runId === item.runId &&
112+
row.item.ordinal <= item.ordinal,
113+
)
114+
) {
115+
return false;
116+
}
117+
return isBelowPartialWindow(projection, item, latestLocalTurnOrdinal);
118+
}
119+
120+
function isBelowPartialWindow(
121+
projection: OrchestrationV2ThreadProjection,
122+
item: OrchestrationV2TurnItem,
123+
latestLocalTurnOrdinal: number | null | undefined,
124+
): boolean {
97125
if (
98126
latestLocalTurnOrdinal !== null &&
99127
latestLocalTurnOrdinal !== undefined &&

0 commit comments

Comments
 (0)