Skip to content

Commit 9fe2ada

Browse files
mthsmbclaude
andcommitted
feat(web): session import keeps going past the 100-conversation batch limit
Each agentSessions.import call imports at most 100 transcripts and 4 GiB per project. Everything past that budget was counted in skippedCount, so the welcome wizard reported it as "could not be imported" and users had to reopen setup by hand, sometimes several times, to finish a large history. The scanner now reports transcripts left out by the per-attempt transcript or byte budget as Deferred, and the import result gains an optional deferredCount. skippedCount stays the total not imported, so older clients behave as before. Transcripts that were read but could not be parsed are remembered by file identity, so later attempts skip them without spending the budget again. The wizard keeps importing a project while each batch leaves fewer deferred threads, shows per-project progress, and offers Stop importing between batches. If a batch makes no progress, the rest count as not imported. The summary separates threads that could not be imported from threads not imported yet, and offers Continue import when work is left. Servers without deferredCount get a single batch, as today. Built with Claude Code (Opus 5.5 orchestrating, Sonnet 5.5 and Haiku 5.5 subagents). Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 6577c6b commit 9fe2ada

10 files changed

Lines changed: 507 additions & 61 deletions

File tree

‎apps/server/src/project/AgentSessionImporter.test.ts‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ const seedOwner = Effect.fn(function* (
170170
});
171171

172172
const makeImporter = Effect.fn(function* (
173-
candidates: ReadonlyArray<ReturnType<typeof candidate>>,
173+
candidates: ReadonlyArray<AgentSessionScanner.AgentSessionRecentThread>,
174174
beforeTransaction: Effect.Effect<void> = Effect.void,
175175
settings: Layer.Layer<
176176
ServerSettings.ServerSettingsService,
@@ -289,11 +289,14 @@ it.effect(
289289
deleted,
290290
ownedPi,
291291
otherInstance,
292+
{ _tag: "Deferred" },
292293
unowned,
294+
{ _tag: "Skipped" },
293295
unownedPi,
296+
{ _tag: "Deferred" },
294297
]);
295298

296-
expect(yield* importer.run).toEqual({ importedCount: 3, skippedCount: 0 });
299+
expect(yield* importer.run).toEqual({ importedCount: 3, skippedCount: 3, deferredCount: 2 });
297300
for (const source of [active, archived, deleted, ownedPi]) {
298301
const threadId = importId(source);
299302
expect(yield* Effect.option(projections.getThread(threadId))).toEqual(Option.none());
@@ -364,8 +367,12 @@ it.effect("rechecks ownership after the import reaches the commit transaction",
364367
// Commit ownership after all pre-transaction reads, before the import acquires SQLite.
365368
const ownerId = yield* seedOwner(source);
366369
yield* Deferred.succeed(continueTransaction, undefined);
367-
expect(yield* Fiber.join(pending)).toEqual({ importedCount: 0, skippedCount: 0 });
368-
expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 0 });
370+
expect(yield* Fiber.join(pending)).toEqual({
371+
importedCount: 0,
372+
skippedCount: 0,
373+
deferredCount: 0,
374+
});
375+
expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 0, deferredCount: 0 });
369376
const projections = yield* ProjectionStore.ProjectionStoreV2;
370377
const events = yield* EventStore.EventStoreV2;
371378
const runtimes = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository;
@@ -392,23 +399,23 @@ it.effect(
392399
const threadId = importId(source);
393400
const importer = yield* makeImporter([source]);
394401
yield* sql`CREATE TEMP TRIGGER fail_import BEFORE INSERT ON orchestration_events WHEN NEW.event_type = 'message.updated' BEGIN SELECT RAISE(ABORT, 'test publication failure'); END`;
395-
expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 1 });
402+
expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 1, deferredCount: 0 });
396403
expect(yield* events.read({ threadId }).pipe(Stream.runCollect)).toEqual([]);
397404
expect(yield* Effect.option(projections.getThread(threadId))).toEqual(Option.none());
398405
expect(yield* runtimes.list()).toEqual([]);
399406
expect(importer.published).toEqual([]);
400407
yield* sql`DROP TRIGGER fail_import`;
401408

402409
yield* sql`CREATE TEMP TRIGGER fail_runtime BEFORE INSERT ON provider_session_runtime BEGIN SELECT RAISE(ABORT, 'test runtime failure'); END`;
403-
expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 1 });
410+
expect(yield* importer.run).toEqual({ importedCount: 0, skippedCount: 1, deferredCount: 0 });
404411
const committed = yield* events.read({ threadId }).pipe(Stream.runCollect);
405412
expect(committed).toHaveLength(6);
406413
expect(importer.published).toHaveLength(6);
407414
expect(yield* runtimes.list()).toEqual([]);
408415
yield* sql`DROP TRIGGER fail_runtime`;
409416

410-
expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 });
411-
expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 });
417+
expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0, deferredCount: 0 });
418+
expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0, deferredCount: 0 });
412419
expect(yield* events.read({ threadId }).pipe(Stream.runCollect)).toEqual(committed);
413420
expect(importer.published).toHaveLength(6);
414421
expect(
@@ -442,7 +449,7 @@ it.effect("imports under a policy ceiling in the project's default, kept under t
442449
},
443450
),
444451
);
445-
expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0 });
452+
expect(yield* importer.run).toEqual({ importedCount: 1, skippedCount: 0, deferredCount: 0 });
446453
expect((yield* projections.getThread(threadId)).runtimeMode).toBe("auto-accept-edits");
447454
expect(Option.getOrThrow(yield* runtimes.getByThreadId({ threadId })).runtimeMode).toBe(
448455
"auto-accept-edits",

‎apps/server/src/project/AgentSessionImporter.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,13 +235,19 @@ const make = Effect.gen(function* () {
235235
const importedThreadIds = new Set<ThreadId>();
236236
let importedCount = 0;
237237
let skippedCount = 0;
238+
let deferredCount = 0;
238239

239240
yield* Stream.runForEach(outcomes, (outcome) =>
240241
Effect.gen(function* () {
241242
if (outcome._tag === "Skipped") {
242243
skippedCount += 1;
243244
return;
244245
}
246+
if (outcome._tag === "Deferred") {
247+
skippedCount += 1;
248+
deferredCount += 1;
249+
return;
250+
}
245251
const source = outcome.source;
246252
const threadId = ThreadId.make(
247253
`import:${source.providerInstanceId}:${source.providerSessionId}`,
@@ -449,7 +455,7 @@ const make = Effect.gen(function* () {
449455
}),
450456
);
451457

452-
return { importedCount, skippedCount } satisfies AgentSessionImportResult;
458+
return { importedCount, skippedCount, deferredCount } satisfies AgentSessionImportResult;
453459
});
454460

455461
return { importRecentAgentThreads };

‎apps/server/src/project/AgentSessionScanner.test.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1623,6 +1623,83 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => {
16231623
}),
16241624
);
16251625

1626+
it.effect(
1627+
"defers transcripts past the per-attempt budget and skips known failures for free",
1628+
() =>
1629+
Effect.gen(function* () {
1630+
const path = yield* Path.Path;
1631+
const nowMs = Date.parse("2026-08-24T12:00:00.000Z");
1632+
yield* TestClock.setTime(nowMs);
1633+
const claudeHomePath = yield* makeTempDir("t3code-transcript-budget-claude-");
1634+
const codexHomePath = yield* makeTempDir("t3code-transcript-budget-codex-");
1635+
const workspace = yield* makeTempDir("t3code-transcript-budget-workspace-");
1636+
const writeRollout = (index: number, withPrompt: boolean) =>
1637+
writeTranscript({
1638+
filePath: path.join(
1639+
codexHomePath,
1640+
"sessions",
1641+
"2026",
1642+
"08",
1643+
"24",
1644+
`rollout-${String(index).padStart(3, "0")}.jsonl`,
1645+
),
1646+
contents: [
1647+
{ type: "session_meta", payload: { id: `session-${index}`, cwd: workspace } },
1648+
...(withPrompt
1649+
? [
1650+
{
1651+
type: "event_msg",
1652+
payload: { type: "user_message", message: "Import this prompt" },
1653+
},
1654+
]
1655+
: []),
1656+
]
1657+
.map((record) => encodeTranscriptRecord(record))
1658+
.join("\n"),
1659+
mtimeMs: nowMs - index * 60_000,
1660+
});
1661+
// The 100 newest rollouts have no user message and fill the first attempt's budget.
1662+
for (let index = 0; index <= 200; index += 1) {
1663+
yield* writeRollout(index, index >= 100);
1664+
}
1665+
const tags = (outcomes: Iterable<AgentSessionScanner.AgentSessionRecentThread>) =>
1666+
Array.from(outcomes, (outcome) => outcome._tag);
1667+
const repeat = (count: number, tag: string) => Array.from({ length: count }, () => tag);
1668+
1669+
yield* Effect.gen(function* () {
1670+
const scanner = yield* AgentSessionScanner.AgentSessionScanner;
1671+
const first = yield* Stream.runCollect(scanner.recentThreads(workspace));
1672+
expect(tags(first)).toEqual([...repeat(100, "Skipped"), ...repeat(101, "Deferred")]);
1673+
1674+
const second = yield* Stream.runCollect(scanner.recentThreads(workspace));
1675+
expect(tags(second)).toEqual([
1676+
...repeat(100, "Skipped"),
1677+
...repeat(100, "Importable"),
1678+
"Deferred",
1679+
]);
1680+
const completed = Array.from(second).flatMap((outcome) =>
1681+
outcome._tag === "Importable" ? [outcome.source] : [],
1682+
);
1683+
1684+
// A changed transcript is read again instead of reusing its earlier failure.
1685+
yield* writeRollout(0, true);
1686+
const third = Array.from(
1687+
yield* Stream.runCollect(scanner.recentThreads(workspace, completed)),
1688+
);
1689+
expect(tags(third)).toEqual([
1690+
"Importable",
1691+
...repeat(99, "Skipped"),
1692+
...repeat(100, "AlreadyImported"),
1693+
"Importable",
1694+
]);
1695+
expect([third[0], third[200]]).toMatchObject([
1696+
{ thread: { providerSessionId: "session-0" } },
1697+
{ thread: { providerSessionId: "session-200" } },
1698+
]);
1699+
}).pipe(Effect.provide(layerScannerTest({ claudeHomePath, codexHomePath })));
1700+
}),
1701+
);
1702+
16261703
for (const source of ["claudeAgent", "codex"] as const) {
16271704
for (const replacement of [
16281705
"same root",

‎apps/server/src/project/AgentSessionScanner.ts‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,8 @@ export type AgentSessionRecentThread =
199199
}
200200
| { readonly _tag: "AlreadyImported"; readonly source: AgentSessionImportSource }
201201
| { readonly _tag: "Duplicate"; readonly source: AgentSessionImportSource }
202-
| { readonly _tag: "Skipped" };
202+
| { readonly _tag: "Skipped" }
203+
| { readonly _tag: "Deferred" };
203204

204205
export class AgentSessionScanner extends Context.Service<
205206
AgentSessionScanner,
@@ -1524,6 +1525,9 @@ export const make = Effect.gen(function* () {
15241525
});
15251526

15261527
let cachedCandidates: ReadonlyArray<RawCandidate> | null = null;
1528+
// An unchanged transcript that was read but could not be parsed fails the same way again, so
1529+
// later attempts skip it without spending the per-attempt budget.
1530+
const unimportableTranscripts = new Map<string, TranscriptIdentity>();
15271531

15281532
const scan: AgentSessionScanner["Service"]["scan"] = Effect.gen(function* () {
15291533
const { candidates: raw, truncated } = yield* collectCandidates();
@@ -1763,11 +1767,15 @@ export const make = Effect.gen(function* () {
17631767
return Stream.fromIteratorSucceed(eligibleTranscripts.values(), 1).pipe(
17641768
Stream.mapEffect(({ candidate, transcript }) =>
17651769
Effect.gen(function* () {
1766-
const completed = completedByFile.get(
1767-
`${candidate.providerInstanceId}\0${transcript.filePath}`,
1768-
);
1769-
if (completed === undefined && (transcriptsRemaining === 0 || bytesRemaining === 0)) {
1770-
return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });
1770+
const fileKey = `${candidate.providerInstanceId}\0${transcript.filePath}`;
1771+
const completed = completedByFile.get(fileKey);
1772+
const unimportable = unimportableTranscripts.get(fileKey);
1773+
if (
1774+
completed === undefined &&
1775+
unimportable === undefined &&
1776+
(transcriptsRemaining === 0 || bytesRemaining === 0)
1777+
) {
1778+
return Option.some<AgentSessionRecentThread>({ _tag: "Deferred" });
17711779
}
17721780
const stats = yield* statOption(transcript.filePath);
17731781
if (Option.isNone(stats) || stats.value.type !== "File") {
@@ -1788,12 +1796,14 @@ export const make = Effect.gen(function* () {
17881796
});
17891797
}
17901798
if (
1791-
transcriptsRemaining === 0 ||
17921799
identity.size > MAX_IMPORTED_TRANSCRIPT_BYTES ||
1793-
identity.size > bytesRemaining
1800+
(unimportable !== undefined && sameTranscriptIdentity(unimportable, identity))
17941801
) {
17951802
return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });
17961803
}
1804+
if (transcriptsRemaining === 0 || identity.size > bytesRemaining) {
1805+
return Option.some<AgentSessionRecentThread>({ _tag: "Deferred" });
1806+
}
17971807
transcriptsRemaining -= 1;
17981808
bytesRemaining -= identity.size;
17991809
const snapshot = yield* readTranscript(transcript.filePath, identity, candidate.source);
@@ -1802,6 +1812,7 @@ export const make = Effect.gen(function* () {
18021812
}
18031813

18041814
if (Option.isNone(snapshot.cwd)) {
1815+
unimportableTranscripts.set(fileKey, identity);
18051816
return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });
18061817
}
18071818
const expandedCwd = expandHomePath(snapshot.cwd.value.trim(), homeDir);
@@ -1833,6 +1844,10 @@ export const make = Effect.gen(function* () {
18331844
snapshot.claudeFields,
18341845
);
18351846
if (parsedThread === null) {
1847+
// A Codex rollout missing its inherited history may parse once that history fits.
1848+
if (candidate.source !== "codex" || inheritedRecords !== null) {
1849+
unimportableTranscripts.set(fileKey, identity);
1850+
}
18361851
return Option.some<AgentSessionRecentThread>({ _tag: "Skipped" });
18371852
}
18381853
let savedTitle: string | undefined;

0 commit comments

Comments
 (0)