Skip to content

Commit f2c52b2

Browse files
fix(history): preserve paging consistency and isolate broken sources
Bind search continuations to the ordered listing and resumed transcript version. Recover from individual inaccessible or disappearing entries without caching incomplete listings. Isolate invalid Claude profiles while retaining full deletion of legacy export history.
1 parent 8e267c3 commit f2c52b2

6 files changed

Lines changed: 140 additions & 20 deletions

File tree

‎apps/server/src/project/externalHistory/ExternalHistory.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("external history
205205
enabled: false,
206206
config: { homePath: claude },
207207
},
208+
[ProviderInstanceId.make("invalid-claude")]: {
209+
driver: ProviderDriverKind.make("claudeAgent"),
210+
enabled: false,
211+
config: { homePath: 17 },
212+
},
208213
[ProviderInstanceId.make("opencode-downloads")]: {
209214
driver: ProviderDriverKind.make("opencode"),
210215
environment: [

‎apps/server/src/project/externalHistory/JsonlHistoryStore.test.ts‎

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { expect, it } from "@effect/vitest";
44
import * as Effect from "effect/Effect";
55
import * as FileSystem from "effect/FileSystem";
66
import * as Layer from "effect/Layer";
7+
import * as PlatformError from "effect/PlatformError";
78
import { makeClaudeHistoryStore, makePiHistoryStore } from "./JsonlHistoryStore.ts";
89
import { makeLocalHistory } from "./LocalHistory.ts";
910

@@ -89,8 +90,10 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("local JSONL hist
8990
.read(store, { operation: "read", ref: first.matches[0]!.ref })
9091
.pipe(Effect.flip)).reason,
9192
).toBe("changed");
92-
const restarted = yield* history.search(store, { ...paged, cursor: second.nextCursor! });
93-
expect(restarted.matches[0]?.itemId).toBe("new-0");
93+
expect(
94+
(yield* history.search(store, { ...paged, cursor: second.nextCursor! }).pipe(Effect.flip))
95+
.reason,
96+
).toBe("changed");
9497
expect(
9598
(yield* store
9699
.loadThread({ key: "../../outside.jsonl", version: "", updatedAt: 0 })
@@ -135,6 +138,80 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("local JSONL hist
135138
}).pipe(Effect.scoped),
136139
);
137140

141+
it.effect("isolates unreadable entries and retries incomplete listings after recovery", () =>
142+
Effect.gen(function* () {
143+
const fs = yield* FileSystem.FileSystem;
144+
const root = yield* fs.makeTempDirectoryScoped();
145+
const directory = `${root}/projects`;
146+
for (const project of ["good", "blocked"]) {
147+
yield* fs.makeDirectory(`${directory}/${project}`, { recursive: true });
148+
yield* fs.writeFileString(
149+
`${directory}/${project}/session.jsonl`,
150+
JSON.stringify({ type: "user", uuid: project, message: { content: "needle" } }) + "\n",
151+
);
152+
}
153+
const history = yield* makeLocalHistory;
154+
const allKeys = ["blocked/session.jsonl", "good/session.jsonl"];
155+
for (const failure of [
156+
{ method: "stat", path: `${directory}/blocked`, reason: "NotFound" },
157+
{ method: "readDirectory", path: `${directory}/blocked`, reason: "PermissionDenied" },
158+
{ method: "stat", path: `${directory}/blocked/session.jsonl`, reason: "NotFound" },
159+
{ method: "stat", path: `${directory}/blocked/session.jsonl`, reason: "PermissionDenied" },
160+
] as const) {
161+
let unavailable = true;
162+
const error = PlatformError.systemError({
163+
_tag: failure.reason,
164+
module: "FileSystem",
165+
method: failure.method,
166+
pathOrDescriptor: failure.path,
167+
});
168+
const partialFs = FileSystem.make({
169+
...fs,
170+
stat: (path) =>
171+
unavailable && failure.method === "stat" && path === failure.path
172+
? Effect.fail(error)
173+
: fs.stat(path),
174+
readDirectory: (path) =>
175+
unavailable && failure.method === "readDirectory" && path === failure.path
176+
? Effect.fail(error)
177+
: fs.readDirectory(path),
178+
});
179+
const store = yield* makeClaudeHistoryStore(root, history.cache).pipe(
180+
Effect.provideService(FileSystem.FileSystem, partialFs),
181+
);
182+
expect((yield* store.listThreads).map((ref) => ref.key)).toEqual(["good/session.jsonl"]);
183+
unavailable = false;
184+
expect((yield* store.listThreads).map((ref) => ref.key).toSorted()).toEqual(allKeys);
185+
if (failure.method === "stat") {
186+
unavailable = true;
187+
expect((yield* store.listThreads).map((ref) => ref.key)).toEqual(["good/session.jsonl"]);
188+
unavailable = false;
189+
expect((yield* store.listThreads).map((ref) => ref.key).toSorted()).toEqual(allKeys);
190+
}
191+
}
192+
const inaccessibleRoot = yield* makeClaudeHistoryStore(root, history.cache).pipe(
193+
Effect.provideService(
194+
FileSystem.FileSystem,
195+
FileSystem.make({
196+
...fs,
197+
readDirectory: (path) =>
198+
path === directory
199+
? Effect.fail(
200+
PlatformError.systemError({
201+
_tag: "PermissionDenied",
202+
module: "FileSystem",
203+
method: "readDirectory",
204+
pathOrDescriptor: path,
205+
}),
206+
)
207+
: fs.readDirectory(path),
208+
}),
209+
),
210+
);
211+
expect((yield* inaccessibleRoot.listThreads.pipe(Effect.flip)).reason).toBe("unavailable");
212+
}).pipe(Effect.scoped),
213+
);
214+
138215
it.effect("searches sessions larger than the old allocation cap and verifies their source", () =>
139216
Effect.gen(function* () {
140217
const fs = yield* FileSystem.FileSystem;

‎apps/server/src/project/externalHistory/JsonlHistoryStore.ts‎

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,18 @@ const makeJsonlHistoryStore = Effect.fn("ExternalHistory.openJsonlStore")(functi
8989
| { directories: ReadonlyMap<string, string>; refs: ReadonlyArray<LocalThreadRef> }
9090
| undefined;
9191
const listThreads = Effect.gen(function* () {
92-
if (listing) {
92+
let complete = true;
93+
const skipEntryError = (error: PlatformError.PlatformError) => {
94+
if (error.reason._tag === "NotFound" || error.reason._tag === "PermissionDenied") {
95+
complete = false;
96+
return Effect.undefined;
97+
}
98+
return Effect.fail(error);
99+
};
100+
const cachedListing = listing;
101+
if (cachedListing) {
93102
let unchanged = true;
94-
for (const [dir, version] of listing.directories) {
103+
for (const [dir, version] of cachedListing.directories) {
95104
const stat = yield* fs.stat(dir).pipe(Effect.option);
96105
if (Option.isNone(stat) || fileVersion(stat.value) !== version) {
97106
unchanged = false;
@@ -101,7 +110,7 @@ const makeJsonlHistoryStore = Effect.fn("ExternalHistory.openJsonlStore")(functi
101110
if (unchanged) {
102111
// Appending a transcript does not change its directory's mtime.
103112
const refreshed = yield* Effect.forEach(
104-
listing.refs,
113+
cachedListing.refs,
105114
(ref) =>
106115
fs.stat(path.join(directory, ...ref.key.split("/"))).pipe(
107116
Effect.map((info) =>
@@ -115,16 +124,13 @@ const makeJsonlHistoryStore = Effect.fn("ExternalHistory.openJsonlStore")(functi
115124
}
116125
: undefined,
117126
),
118-
Effect.catchTags({
119-
PlatformError: (error) =>
120-
error.reason._tag === "NotFound" ? Effect.undefined : Effect.fail(error),
121-
}),
127+
Effect.catchTags({ PlatformError: skipEntryError }),
122128
),
123129
{ concurrency: 16 },
124130
);
125131
const refs = refreshed.filter((ref) => ref !== undefined);
126132
refs.sort((a, b) => b.updatedAt - a.updatedAt || a.key.localeCompare(b.key));
127-
listing = { directories: listing.directories, refs };
133+
listing = complete ? { directories: cachedListing.directories, refs } : undefined;
128134
return refs;
129135
}
130136
}
@@ -138,22 +144,30 @@ const makeJsonlHistoryStore = Effect.fn("ExternalHistory.openJsonlStore")(functi
138144
for (const child of children) {
139145
const childPath = path.join(directory, child);
140146
if (yield* Effect.isSuccess(fs.readLink(childPath))) continue;
141-
const info = yield* fs.stat(childPath);
147+
const info = yield* fs
148+
.stat(childPath)
149+
.pipe(Effect.catchTags({ PlatformError: skipEntryError }));
150+
if (!info) continue;
142151
const files =
143152
info.type === "Directory"
144-
? yield* fs.readDirectory(childPath)
153+
? yield* fs
154+
.readDirectory(childPath)
155+
.pipe(Effect.catchTags({ PlatformError: skipEntryError }))
145156
: source === "pi" && child.endsWith(".jsonl")
146157
? [child]
147158
: [];
159+
if (!files) continue;
148160
if (info.type === "Directory") directories.set(childPath, fileVersion(info));
149161
if (directories.size > 50_000 || files.length + refs.length > 50_000)
150162
return yield* new ExternalHistoryError({ reason: "limit_exceeded" });
151163
for (const name of files) {
152164
if (!name.endsWith(".jsonl")) continue;
153165
const filePath = info.type === "Directory" ? path.join(childPath, name) : childPath;
154166
if (yield* Effect.isSuccess(fs.readLink(filePath))) continue;
155-
const stat = yield* fs.stat(filePath);
156-
if (stat.type !== "File") continue;
167+
const stat = yield* fs
168+
.stat(filePath)
169+
.pipe(Effect.catchTags({ PlatformError: skipEntryError }));
170+
if (stat?.type !== "File") continue;
157171
const key = path.relative(directory, filePath).split(path.sep).join("/");
158172
if (key.length > 512) return yield* new ExternalHistoryError({ reason: "limit_exceeded" });
159173
refs.push({
@@ -164,7 +178,8 @@ const makeJsonlHistoryStore = Effect.fn("ExternalHistory.openJsonlStore")(functi
164178
}
165179
}
166180
refs.sort((a, b) => b.updatedAt - a.updatedAt || a.key.localeCompare(b.key));
167-
listing = { directories, refs };
181+
// An omitted entry may recover without changing its parent's mtime.
182+
listing = complete ? { directories, refs } : undefined;
168183
return refs;
169184
}).pipe(Effect.mapError(mapError));
170185

‎apps/server/src/project/externalHistory/LocalHistory.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,18 @@ it.layer(NodeCrypto.layer)("local history search paging", (it) => {
103103
"last:hit",
104104
]);
105105
expect(requests).toBeGreaterThan(3);
106+
107+
let refs = yield* store.listThreads;
108+
const changingStore = { ...store, listThreads: Effect.sync(() => refs) };
109+
const history = yield* makeLocalHistory;
110+
const request = { operation: "search", sourceId: "local", query: "needle" } as const;
111+
const first = yield* history.search(changingStore, request);
112+
refs = [refs.at(-1)!, ...refs.slice(0, -1)];
113+
expect(
114+
(yield* history
115+
.search(changingStore, { ...request, cursor: first.nextCursor! })
116+
.pipe(Effect.flip)).reason,
117+
).toBe("changed");
106118
}),
107119
);
108120

‎apps/server/src/project/externalHistory/LocalHistory.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,11 @@ import {
2020
} from "./LocalHistoryStore.ts";
2121

2222
const SearchCursor = Schema.Struct({
23-
v: Schema.Literal(2),
23+
v: Schema.Literal(3),
2424
sourceId: Schema.String,
2525
query: Schema.String,
2626
filterCwd: Schema.NullOr(Schema.String),
27+
listingDigest: Schema.String,
2728
afterKey: Schema.NullOr(Schema.String),
2829
itemOffset: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
2930
version: Schema.NullOr(Schema.String),
@@ -86,17 +87,22 @@ export const makeLocalHistory = Effect.gen(function* () {
8687
)
8788
return yield* invalidCursor();
8889
const refs = yield* store.listThreads;
90+
// Appends can reorder sessions between pages; never silently skip or repeat them.
91+
const listingDigest = yield* digest(JSON.stringify(refs.map((ref) => ref.key)));
92+
if (cursor && cursor.listingDigest !== listingDigest)
93+
return yield* new ExternalHistoryError({ reason: "changed" });
8994
const first =
9095
cursor?.afterKey == null ? 0 : refs.findIndex((ref) => ref.key === cursor.afterKey);
9196
if (first < 0) return yield* new ExternalHistoryError({ reason: "changed" });
9297
const cursorAt = (index: number, itemOffset: number, version: string | null) => {
9398
const ref = refs[index];
9499
return ref
95100
? encodeSearchCursor({
96-
v: 2,
101+
v: 3,
97102
sourceId: input.sourceId,
98103
query: input.query,
99104
filterCwd: input.filterCwd ?? null,
105+
listingDigest,
100106
afterKey: ref.key,
101107
itemOffset,
102108
version,
@@ -137,6 +143,8 @@ export const makeLocalHistory = Effect.gen(function* () {
137143
() => Effect.undefined,
138144
),
139145
);
146+
if (resumed?.version != null && resumed.version !== thread?.version)
147+
return yield* new ExternalHistoryError({ reason: "changed" });
140148
if (!thread) {
141149
examinedChars += FAILED_LOAD_CHARS;
142150
continue;
@@ -150,7 +158,7 @@ export const makeLocalHistory = Effect.gen(function* () {
150158
normalizeProjectPathForComparison(input.filterCwd)))
151159
)
152160
continue;
153-
const offset = resumed?.version === thread.version ? resumed.itemOffset : 0;
161+
const offset = resumed?.itemOffset ?? 0;
154162
if (offset > thread.items.length) return yield* invalidCursor();
155163
for (let position = offset; position < thread.items.length; position++) {
156164
const item = thread.items[position]!;

‎apps/server/src/project/externalHistory/historySupport.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { piHome, resolvePiSessionsDir } from "@t3tools/provider-pi/server/usage"
44
import * as HostProcess from "@t3tools/shared/HostProcess";
55
import * as Effect from "effect/Effect";
66
import * as FileSystem from "effect/FileSystem";
7+
import * as Option from "effect/Option";
78
import * as Path from "effect/Path";
89
import * as Schema from "effect/Schema";
910
import { resolveClaudeRuntime } from "../../provider/Drivers/ClaudeHome.ts";
@@ -32,7 +33,7 @@ type HistorySupport = { readonly label: string } & (
3233
| { readonly kind: "unsupported"; readonly reason: string }
3334
);
3435

35-
const decodeClaudeSettings = Schema.decodeUnknownSync(ClaudeSettings);
36+
const decodeClaudeSettings = Schema.decodeUnknownOption(ClaudeSettings);
3637
const unsupported = (label: string): HistorySupport => ({
3738
kind: "unsupported",
3839
label,
@@ -54,9 +55,11 @@ export function makeHistorySupport(stores: {
5455
label: "Claude Code",
5556
open: stores.claudeAgent,
5657
roots: Effect.fn("ExternalHistory.claudeRoots")(function* (profile) {
58+
const config = decodeClaudeSettings(profile.config ?? {});
59+
if (Option.isNone(config)) return [];
5760
const runtime = yield* resolveClaudeRuntime({
5861
...profile,
59-
config: decodeClaudeSettings(profile.config ?? {}),
62+
config: config.value,
6063
});
6164
return [runtime.configDir];
6265
}),

0 commit comments

Comments
 (0)