Skip to content

Commit 215fd76

Browse files
yotambrauncliffhall
authored andcommitted
fix(memory): keep unreadable lines when saving the graph
Since #4717, loadGraph skips lines it cannot read (malformed JSON, entries that fail validation, unknown record types). Every mutation then rewrote the file from the filtered graph, so the next unrelated write permanently deleted those lines: an entity with one null observation lost all of its valid observations, and relations to it were left dangling. Keep skipping them in the in-memory graph, but write them back unchanged on save. Cherry-picked from #4886 (0ef28b0) onto v2/main. Conflicts with v2/main's Prettier formatting resolved by hand; saveGraph's unreadable parameter is now required, since every caller passes it. Signed-off-by: cliffhall <[email protected]>
1 parent 571e2b3 commit 215fd76

2 files changed

Lines changed: 77 additions & 15 deletions

File tree

‎src/memory/__tests__/knowledge-graph.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,45 @@ describe("KnowledgeGraphManager", () => {
789789
expect(graph.entities).toHaveLength(2);
790790
expect(graph.entities.map((e) => e.name)).toEqual(["Alice", "Bob"]);
791791
});
792+
793+
it("keeps lines it cannot read when an unrelated write saves the graph", async () => {
794+
const unreadable = [
795+
JSON.stringify({
796+
type: "entity",
797+
name: "Broken",
798+
observations: ["missing entityType"],
799+
}),
800+
JSON.stringify({
801+
type: "entity",
802+
name: "BadObs",
803+
entityType: "person",
804+
observations: ["allergic to penicillin", null],
805+
}),
806+
"{this is not valid json",
807+
JSON.stringify({ type: "note", text: "written by another tool" }),
808+
];
809+
const lines = [
810+
JSON.stringify({
811+
type: "entity",
812+
name: "Alice",
813+
entityType: "person",
814+
observations: [],
815+
}),
816+
...unreadable,
817+
];
818+
await fs.writeFile(testFilePath, lines.join("\n") + "\n");
819+
820+
await manager.createEntities([
821+
{ name: "Carol", entityType: "person", observations: [] },
822+
]);
823+
824+
const saved = (await fs.readFile(testFilePath, "utf-8")).split("\n");
825+
for (const line of unreadable) {
826+
expect(saved).toContain(line);
827+
}
828+
const graph = await manager.readGraph();
829+
expect(graph.entities.map((e) => e.name)).toEqual(["Alice", "Carol"]);
830+
});
792831
});
793832

794833
describe("concurrent mutations", () => {

‎src/memory/index.ts‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -127,22 +127,37 @@ export class KnowledgeGraphManager {
127127
}
128128

129129
private async loadGraph(): Promise<KnowledgeGraph> {
130+
return (await this.loadGraphWithUnreadable()).graph;
131+
}
132+
133+
// Lines that can't be read (malformed JSON, entries that fail validation,
134+
// unknown record types) stay out of the in-memory graph, but mutations pass
135+
// them back to saveGraph so they are written out unchanged. Otherwise the
136+
// next unrelated write would silently delete them from disk, together with
137+
// every valid observation of an entity that has one bad field.
138+
private async loadGraphWithUnreadable(): Promise<{
139+
graph: KnowledgeGraph;
140+
unreadable: string[];
141+
}> {
130142
try {
131143
const data = await fs.readFile(this.memoryFilePath, "utf-8");
132144
const lines = data.split("\n").filter((line) => line.trim() !== "");
133145
const graph: KnowledgeGraph = { entities: [], relations: [] };
146+
const unreadable: string[] = [];
134147

135148
for (const line of lines) {
136149
let item: unknown;
137150
try {
138151
item = JSON.parse(line);
139152
} catch {
140153
console.error("Skipping malformed line in memory file");
154+
unreadable.push(line);
141155
continue;
142156
}
143157

144158
if (typeof item !== "object" || item === null) {
145159
console.error("Skipping non-object line in memory file");
160+
unreadable.push(line);
146161
continue;
147162
}
148163

@@ -152,6 +167,7 @@ export class KnowledgeGraphManager {
152167
if (parsed.success) {
153168
graph.entities.push(parsed.data);
154169
} else {
170+
unreadable.push(line);
155171
console.error(
156172
"Skipping invalid entity in memory file:",
157173
parsed.error.issues
@@ -164,30 +180,36 @@ export class KnowledgeGraphManager {
164180
if (parsed.success) {
165181
graph.relations.push(parsed.data);
166182
} else {
183+
unreadable.push(line);
167184
console.error(
168185
"Skipping invalid relation in memory file:",
169186
parsed.error.issues
170187
.map((issue) => `${issue.path.join(".")}: ${issue.message}`)
171188
.join(", "),
172189
);
173190
}
191+
} else {
192+
unreadable.push(line);
174193
}
175194
}
176195

177-
return graph;
196+
return { graph, unreadable };
178197
} catch (error) {
179198
if (
180199
error instanceof Error &&
181200
"code" in error &&
182201
error.code === "ENOENT"
183202
) {
184-
return { entities: [], relations: [] };
203+
return { graph: { entities: [], relations: [] }, unreadable: [] };
185204
}
186205
throw error;
187206
}
188207
}
189208

190-
private async saveGraph(graph: KnowledgeGraph): Promise<void> {
209+
private async saveGraph(
210+
graph: KnowledgeGraph,
211+
unreadable: string[],
212+
): Promise<void> {
191213
const lines = [
192214
...graph.entities.map((e) =>
193215
JSON.stringify({
@@ -205,6 +227,7 @@ export class KnowledgeGraphManager {
205227
relationType: r.relationType,
206228
}),
207229
),
230+
...unreadable,
208231
];
209232

210233
// Write to a temporary file in the same directory, then rename it over
@@ -233,7 +256,7 @@ export class KnowledgeGraphManager {
233256

234257
async createEntities(entities: Entity[]): Promise<Entity[]> {
235258
return this.withLock(async () => {
236-
const graph = await this.loadGraph();
259+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
237260
const newEntities = entities.filter(
238261
(e, index) =>
239262
!graph.entities.some(
@@ -243,14 +266,14 @@ export class KnowledgeGraphManager {
243266
!entities.slice(0, index).some((earlier) => earlier.name === e.name),
244267
);
245268
graph.entities.push(...newEntities);
246-
await this.saveGraph(graph);
269+
await this.saveGraph(graph, unreadable);
247270
return newEntities;
248271
});
249272
}
250273

251274
async createRelations(relations: Relation[]): Promise<Relation[]> {
252275
return this.withLock(async () => {
253-
const graph = await this.loadGraph();
276+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
254277
const entityNames = new Set(graph.entities.map((e) => e.name));
255278

256279
relations.forEach((r) => {
@@ -275,7 +298,7 @@ export class KnowledgeGraphManager {
275298
.some((earlier) => isSameRelation(earlier, r)),
276299
);
277300
graph.relations.push(...newRelations);
278-
await this.saveGraph(graph);
301+
await this.saveGraph(graph, unreadable);
279302
return newRelations;
280303
});
281304
}
@@ -284,7 +307,7 @@ export class KnowledgeGraphManager {
284307
observations: { entityName: string; contents: string[] }[],
285308
): Promise<{ entityName: string; addedObservations: string[] }[]> {
286309
return this.withLock(async () => {
287-
const graph = await this.loadGraph();
310+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
288311
const results = observations.map((o) => {
289312
const entity = graph.entities.find((e) => e.name === o.entityName);
290313
if (!entity) {
@@ -296,7 +319,7 @@ export class KnowledgeGraphManager {
296319
entity.observations.push(...newObservations);
297320
return { entityName: o.entityName, addedObservations: newObservations };
298321
});
299-
await this.saveGraph(graph);
322+
await this.saveGraph(graph, unreadable);
300323
return results;
301324
});
302325
}
@@ -305,7 +328,7 @@ export class KnowledgeGraphManager {
305328
entityNames: string[],
306329
): Promise<{ deleted: string[]; notFound: string[] }> {
307330
return this.withLock(async () => {
308-
const graph = await this.loadGraph();
331+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
309332
const present = new Set(graph.entities.map((e) => e.name));
310333
const deleted = entityNames.filter((name) => present.has(name));
311334
const notFound = entityNames.filter((name) => !present.has(name));
@@ -315,7 +338,7 @@ export class KnowledgeGraphManager {
315338
graph.relations = graph.relations.filter(
316339
(r) => !entityNames.includes(r.from) && !entityNames.includes(r.to),
317340
);
318-
await this.saveGraph(graph);
341+
await this.saveGraph(graph, unreadable);
319342
return { deleted, notFound };
320343
});
321344
}
@@ -324,7 +347,7 @@ export class KnowledgeGraphManager {
324347
deletions: { entityName: string; observations: string[] }[],
325348
): Promise<{ deletedCount: number; missingEntities: string[] }> {
326349
return this.withLock(async () => {
327-
const graph = await this.loadGraph();
350+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
328351
let deletedCount = 0;
329352
const missingEntities: string[] = [];
330353
deletions.forEach((d) => {
@@ -339,7 +362,7 @@ export class KnowledgeGraphManager {
339362
missingEntities.push(d.entityName);
340363
}
341364
});
342-
await this.saveGraph(graph);
365+
await this.saveGraph(graph, unreadable);
343366
return { deletedCount, missingEntities };
344367
});
345368
}
@@ -348,7 +371,7 @@ export class KnowledgeGraphManager {
348371
relations: Relation[],
349372
): Promise<{ deletedCount: number }> {
350373
return this.withLock(async () => {
351-
const graph = await this.loadGraph();
374+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
352375
const before = graph.relations.length;
353376
graph.relations = graph.relations.filter(
354377
(r) =>
@@ -359,7 +382,7 @@ export class KnowledgeGraphManager {
359382
r.relationType === delRelation.relationType,
360383
),
361384
);
362-
await this.saveGraph(graph);
385+
await this.saveGraph(graph, unreadable);
363386
return { deletedCount: before - graph.relations.length };
364387
});
365388
}

0 commit comments

Comments
 (0)