Skip to content

Commit 0ef28b0

Browse files
committed
fix(memory): keep unreadable lines when saving the graph
Since #4717, loadGraph skips lines it cannot read (malformed JSON, entries that fail validation, unknown record types). Every mutation then rewrote the file from the filtered graph, so the next unrelated write permanently deleted those lines: an entity with one null observation lost all of its valid observations, and relations to it were left dangling. Keep skipping them in the in-memory graph, but write them back unchanged on save.
1 parent f46d957 commit 0ef28b0

2 files changed

Lines changed: 55 additions & 15 deletions

File tree

‎src/memory/__tests__/knowledge-graph.test.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -695,6 +695,29 @@ describe('KnowledgeGraphManager', () => {
695695
expect(graph.entities).toHaveLength(2);
696696
expect(graph.entities.map(e => e.name)).toEqual(['Alice', 'Bob']);
697697
});
698+
699+
it('keeps lines it cannot read when an unrelated write saves the graph', async () => {
700+
const unreadable = [
701+
JSON.stringify({ type: 'entity', name: 'Broken', observations: ['missing entityType'] }),
702+
JSON.stringify({ type: 'entity', name: 'BadObs', entityType: 'person', observations: ['allergic to penicillin', null] }),
703+
'{this is not valid json',
704+
JSON.stringify({ type: 'note', text: 'written by another tool' }),
705+
];
706+
const lines = [
707+
JSON.stringify({ type: 'entity', name: 'Alice', entityType: 'person', observations: [] }),
708+
...unreadable,
709+
];
710+
await fs.writeFile(testFilePath, lines.join('\n') + '\n');
711+
712+
await manager.createEntities([{ name: 'Carol', entityType: 'person', observations: [] }]);
713+
714+
const saved = (await fs.readFile(testFilePath, 'utf-8')).split('\n');
715+
for (const line of unreadable) {
716+
expect(saved).toContain(line);
717+
}
718+
const graph = await manager.readGraph();
719+
expect(graph.entities.map(e => e.name)).toEqual(['Alice', 'Carol']);
720+
});
698721
});
699722

700723
describe('concurrent mutations', () => {

‎src/memory/index.ts‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -107,22 +107,34 @@ export class KnowledgeGraphManager {
107107
}
108108

109109
private async loadGraph(): Promise<KnowledgeGraph> {
110+
return (await this.loadGraphWithUnreadable()).graph;
111+
}
112+
113+
// Lines that can't be read (malformed JSON, entries that fail validation,
114+
// unknown record types) stay out of the in-memory graph, but mutations pass
115+
// them back to saveGraph so they are written out unchanged. Otherwise the
116+
// next unrelated write would silently delete them from disk, together with
117+
// every valid observation of an entity that has one bad field.
118+
private async loadGraphWithUnreadable(): Promise<{ graph: KnowledgeGraph; unreadable: string[] }> {
110119
try {
111120
const data = await fs.readFile(this.memoryFilePath, "utf-8");
112121
const lines = data.split("\n").filter(line => line.trim() !== "");
113122
const graph: KnowledgeGraph = { entities: [], relations: [] };
123+
const unreadable: string[] = [];
114124

115125
for (const line of lines) {
116126
let item: unknown;
117127
try {
118128
item = JSON.parse(line);
119129
} catch {
120130
console.error("Skipping malformed line in memory file");
131+
unreadable.push(line);
121132
continue;
122133
}
123134

124135
if (typeof item !== "object" || item === null) {
125136
console.error("Skipping non-object line in memory file");
137+
unreadable.push(line);
126138
continue;
127139
}
128140

@@ -132,6 +144,7 @@ export class KnowledgeGraphManager {
132144
if (parsed.success) {
133145
graph.entities.push(parsed.data);
134146
} else {
147+
unreadable.push(line);
135148
console.error(
136149
"Skipping invalid entity in memory file:",
137150
parsed.error.issues.map(issue => `${issue.path.join(".")}: ${issue.message}`).join(", ")
@@ -142,24 +155,27 @@ export class KnowledgeGraphManager {
142155
if (parsed.success) {
143156
graph.relations.push(parsed.data);
144157
} else {
158+
unreadable.push(line);
145159
console.error(
146160
"Skipping invalid relation in memory file:",
147161
parsed.error.issues.map(issue => `${issue.path.join(".")}: ${issue.message}`).join(", ")
148162
);
149163
}
164+
} else {
165+
unreadable.push(line);
150166
}
151167
}
152168

153-
return graph;
169+
return { graph, unreadable };
154170
} catch (error) {
155171
if (error instanceof Error && 'code' in error && (error as any).code === "ENOENT") {
156-
return { entities: [], relations: [] };
172+
return { graph: { entities: [], relations: [] }, unreadable: [] };
157173
}
158174
throw error;
159175
}
160176
}
161177

162-
private async saveGraph(graph: KnowledgeGraph): Promise<void> {
178+
private async saveGraph(graph: KnowledgeGraph, unreadable: string[] = []): Promise<void> {
163179
const lines = [
164180
...graph.entities.map(e => JSON.stringify({
165181
type: "entity",
@@ -173,6 +189,7 @@ export class KnowledgeGraphManager {
173189
to: r.to,
174190
relationType: r.relationType
175191
})),
192+
...unreadable,
176193
];
177194

178195
// Write to a temporary file in the same directory, then rename it over
@@ -201,21 +218,21 @@ export class KnowledgeGraphManager {
201218

202219
async createEntities(entities: Entity[]): Promise<Entity[]> {
203220
return this.withLock(async () => {
204-
const graph = await this.loadGraph();
221+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
205222
const newEntities = entities.filter((e, index) =>
206223
!graph.entities.some(existingEntity => existingEntity.name === e.name) &&
207224
// Also skip duplicates appearing earlier in this same batch
208225
!entities.slice(0, index).some(earlier => earlier.name === e.name)
209226
);
210227
graph.entities.push(...newEntities);
211-
await this.saveGraph(graph);
228+
await this.saveGraph(graph, unreadable);
212229
return newEntities;
213230
});
214231
}
215232

216233
async createRelations(relations: Relation[]): Promise<Relation[]> {
217234
return this.withLock(async () => {
218-
const graph = await this.loadGraph();
235+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
219236
const entityNames = new Set(graph.entities.map(e => e.name));
220237

221238
relations.forEach(r => {
@@ -237,14 +254,14 @@ export class KnowledgeGraphManager {
237254
!relations.slice(0, index).some(earlier => isSameRelation(earlier, r))
238255
);
239256
graph.relations.push(...newRelations);
240-
await this.saveGraph(graph);
257+
await this.saveGraph(graph, unreadable);
241258
return newRelations;
242259
});
243260
}
244261

245262
async addObservations(observations: { entityName: string; contents: string[] }[]): Promise<{ entityName: string; addedObservations: string[] }[]> {
246263
return this.withLock(async () => {
247-
const graph = await this.loadGraph();
264+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
248265
const results = observations.map(o => {
249266
const entity = graph.entities.find(e => e.name === o.entityName);
250267
if (!entity) {
@@ -254,27 +271,27 @@ export class KnowledgeGraphManager {
254271
entity.observations.push(...newObservations);
255272
return { entityName: o.entityName, addedObservations: newObservations };
256273
});
257-
await this.saveGraph(graph);
274+
await this.saveGraph(graph, unreadable);
258275
return results;
259276
});
260277
}
261278

262279
async deleteEntities(entityNames: string[]): Promise<{ deleted: string[]; notFound: string[] }> {
263280
return this.withLock(async () => {
264-
const graph = await this.loadGraph();
281+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
265282
const present = new Set(graph.entities.map(e => e.name));
266283
const deleted = entityNames.filter(name => present.has(name));
267284
const notFound = entityNames.filter(name => !present.has(name));
268285
graph.entities = graph.entities.filter(e => !entityNames.includes(e.name));
269286
graph.relations = graph.relations.filter(r => !entityNames.includes(r.from) && !entityNames.includes(r.to));
270-
await this.saveGraph(graph);
287+
await this.saveGraph(graph, unreadable);
271288
return { deleted, notFound };
272289
});
273290
}
274291

275292
async deleteObservations(deletions: { entityName: string; observations: string[] }[]): Promise<{ deletedCount: number; missingEntities: string[] }> {
276293
return this.withLock(async () => {
277-
const graph = await this.loadGraph();
294+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
278295
let deletedCount = 0;
279296
const missingEntities: string[] = [];
280297
deletions.forEach(d => {
@@ -287,21 +304,21 @@ export class KnowledgeGraphManager {
287304
missingEntities.push(d.entityName);
288305
}
289306
});
290-
await this.saveGraph(graph);
307+
await this.saveGraph(graph, unreadable);
291308
return { deletedCount, missingEntities };
292309
});
293310
}
294311

295312
async deleteRelations(relations: Relation[]): Promise<{ deletedCount: number }> {
296313
return this.withLock(async () => {
297-
const graph = await this.loadGraph();
314+
const { graph, unreadable } = await this.loadGraphWithUnreadable();
298315
const before = graph.relations.length;
299316
graph.relations = graph.relations.filter(r => !relations.some(delRelation =>
300317
r.from === delRelation.from &&
301318
r.to === delRelation.to &&
302319
r.relationType === delRelation.relationType
303320
));
304-
await this.saveGraph(graph);
321+
await this.saveGraph(graph, unreadable);
305322
return { deletedCount: before - graph.relations.length };
306323
});
307324
}

0 commit comments

Comments
 (0)