Skip to content

Commit 9606208

Browse files
committed
👁️ Observe a crashed run's store without writing to it
1 parent 3dd969b commit 9606208

3 files changed

Lines changed: 76 additions & 47 deletions

File tree

‎packages/cli/tests/support/run-database.ts‎

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,39 @@
66
* published rather than what some handle is holding. One state needs more
77
* than that grant. A writer killed mid-transaction leaves a hot rollback
88
* journal, and SQLite refuses to read past one on a connection that cannot
9-
* write the rollback. The next write-capable owner of the run performs that
10-
* rollback on its first read — discarding only what was never committed — so
11-
* a reader that meets the refusal recovers the same way, then reads on a
12-
* fresh readonly connection.
9+
* write the rollback — playing the journal back is a write, and it belongs to
10+
* the run's next owner, not to an observer.
11+
*
12+
* So the reader never writes to the store it observes. When it meets the
13+
* refusal, it copies the database and its journal to scratch, lets SQLite
14+
* play the journal back on the copy — which discards only what was never
15+
* committed — reads the copy, and discards it. The real store still holds its
16+
* hot journal afterwards, exactly as the kill left it, for whatever the test
17+
* does next. A hot journal also proves its writer is gone — a live writer
18+
* holds the lock that makes a journal merely busy — so the files are static
19+
* while the copy is taken.
1320
*/
1421

1522
import { DatabaseSync } from "node:sqlite";
23+
import { ensure, scoped } from "effection";
24+
import type { Operation } from "effection";
25+
import { copyFile, ensureDir, exists, rm } from "@effectionx/fs";
26+
import { randomUUID } from "node:crypto";
27+
import { join } from "node:path";
28+
import { tmpdir } from "node:os";
1629

17-
export function readRunDatabase<T>(path: string, read: (database: DatabaseSync) => T): T {
30+
export function* readRunDatabase<T>(
31+
path: string,
32+
read: (database: DatabaseSync) => T,
33+
): Operation<T> {
1834
try {
1935
return readonly(path, read);
2036
} catch (error) {
2137
if (!refusesHotJournal(error)) {
2238
throw error;
2339
}
24-
rollBackHotJournal(path);
25-
return readonly(path, read);
2640
}
41+
return yield* readRecoveredCopy(path, read);
2742
}
2843

2944
function readonly<T>(path: string, read: (database: DatabaseSync) => T): T {
@@ -39,11 +54,23 @@ function refusesHotJournal(error: unknown): boolean {
3954
return error instanceof Error && error.message.includes("attempt to write a readonly database");
4055
}
4156

42-
function rollBackHotJournal(path: string): void {
43-
const database = new DatabaseSync(path);
44-
try {
45-
database.prepare("SELECT count(*) FROM sqlite_schema").get();
46-
} finally {
47-
database.close();
48-
}
57+
function readRecoveredCopy<T>(path: string, read: (database: DatabaseSync) => T): Operation<T> {
58+
return scoped(function* () {
59+
const dir = join(tmpdir(), `xmd-recovered-read-${randomUUID()}`);
60+
yield* ensureDir(dir);
61+
yield* ensure(() => rm(dir, { recursive: true, force: true }));
62+
63+
const copy = join(dir, "run.db");
64+
yield* copyFile(path, copy);
65+
if (yield* exists(`${path}-journal`)) {
66+
yield* copyFile(`${path}-journal`, `${copy}-journal`);
67+
}
68+
69+
const database = new DatabaseSync(copy);
70+
try {
71+
return read(database);
72+
} finally {
73+
database.close();
74+
}
75+
});
4976
}

‎packages/cli/tests/workflow-crash.test.ts‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ interface FileEffect {
144144
* transaction are invisible here until that transaction commits, so this
145145
* reports what has been *published* rather than what some handle is holding.
146146
*/
147-
function committedEffects(path: string): FileEffect[] {
147+
function committedEffects(path: string): Operation<FileEffect[]> {
148148
return readRunDatabase(path, (database) => {
149149
const rows = database
150150
.prepare("SELECT event_id AS id, record FROM journal_events ORDER BY sequence")
@@ -163,15 +163,15 @@ function committedEffects(path: string): FileEffect[] {
163163
}
164164

165165
/** The run's retained status, as a second connection sees it. */
166-
function committedStatus(path: string): string {
166+
function committedStatus(path: string): Operation<string> {
167167
return readRunDatabase(path, (database) => {
168168
const row = database.prepare("SELECT status FROM workflow_run WHERE id = 1").get();
169169
return String(row?.["status"]);
170170
});
171171
}
172172

173173
/** The current Workspace root, as a second connection sees it. */
174-
function committedRoot(path: string): string {
174+
function committedRoot(path: string): Operation<string> {
175175
return readRunDatabase(path, (database) => {
176176
const row = database.prepare("SELECT current_root_id AS root FROM workspace_state").get();
177177
return String(row?.["root"]);
@@ -193,7 +193,7 @@ interface ExecutionRow {
193193
* them by identity is what separates "an execution was closed" from "*this*
194194
* execution was closed".
195195
*/
196-
function committedExecutions(path: string): ExecutionRow[] {
196+
function committedExecutions(path: string): Operation<ExecutionRow[]> {
197197
return readRunDatabase(path, (database) =>
198198
database
199199
.prepare(
@@ -283,12 +283,12 @@ function signalledRun(
283283
yield* when(
284284
function* () {
285285
expect(exists(path)).toBe(true);
286-
expect(committedEffects(path).length).toBeGreaterThanOrEqual(reached);
286+
expect((yield* committedEffects(path)).length).toBeGreaterThanOrEqual(reached);
287287
},
288288
{ timeout: 60_000 },
289289
);
290-
const before = committedEffects(path);
291-
const rootBefore = committedRoot(path);
290+
const before = yield* committedEffects(path);
291+
const rootBefore = yield* committedRoot(path);
292292
process.kill(child.pid, signal);
293293
const status = yield* child.join();
294294

@@ -320,11 +320,11 @@ describe("Tier WFX — a killed workflow run resumes from its frontier", () => {
320320

321321
// Nothing ran after the signal, so the run is still `running`: no status
322322
// was published and no execution record was closed.
323-
expect(committedStatus(path)).toBe("running");
323+
expect(yield* committedStatus(path)).toBe("running");
324324

325325
// The killed owner's execution, by its own identity. It is open, because
326326
// closing one is work and the signal left no opportunity to do any.
327-
const abandoned = committedExecutions(path);
327+
const abandoned = yield* committedExecutions(path);
328328
expect(abandoned).toHaveLength(1);
329329
expect(abandoned[0]?.stopped).toBe(false);
330330
const owner = abandoned[0]?.executionId;
@@ -338,9 +338,9 @@ describe("Tier WFX — a killed workflow run resumes from its frontier", () => {
338338
expect(resumed.code).toBe(0);
339339
expect(resumed.stderr).toContain(`workflow run: ${RUN_ID}`);
340340
expect(resumed.stderr).toContain("workflow status: completed");
341-
expect(committedStatus(path)).toBe("completed");
341+
expect(yield* committedStatus(path)).toBe("completed");
342342

343-
const after = committedEffects(path);
343+
const after = yield* committedEffects(path);
344344

345345
// Every effect the kill left committed is still there, by its own event
346346
// id and in the same order: the resume replayed them rather than
@@ -358,13 +358,13 @@ describe("Tier WFX — a killed workflow run resumes from its frontier", () => {
358358

359359
// The resume continued from the root the last committed effect left, so
360360
// the frontier moved on rather than starting again.
361-
expect(committedRoot(path)).not.toBe(killed.rootBefore);
361+
expect(yield* committedRoot(path)).not.toBe(killed.rootBefore);
362362

363363
// And the killed owner's own execution — that exact one, not merely some
364364
// execution — was closed as interrupted, saying what became of the attempt
365365
// the signal ended. The resume's execution is a different one, and it
366366
// completed.
367-
const settled = committedExecutions(path);
367+
const settled = yield* committedExecutions(path);
368368
expect(settled).toHaveLength(2);
369369
expect(settled[0]?.executionId).toBe(owner);
370370
expect(settled[0]?.stopped).toBe(true);
@@ -397,8 +397,8 @@ describe("Tier WFX — a killed workflow run resumes from its frontier", () => {
397397
// anything resumes: the process that was interrupted settled its own run.
398398
// A kill leaves `running` with an open execution for the next owner to
399399
// reconcile; a Ctrl-C leaves the outcome already published.
400-
expect(committedStatus(path)).toBe("interrupted");
401-
const closed = committedExecutions(path);
400+
expect(yield* committedStatus(path)).toBe("interrupted");
401+
const closed = yield* committedExecutions(path);
402402
expect(closed).toHaveLength(1);
403403
expect(closed[0]?.stopped).toBe(true);
404404
expect(closed[0]?.stopStatus).toBe("interrupted");
@@ -417,18 +417,18 @@ describe("Tier WFX — a killed workflow run resumes from its frontier", () => {
417417

418418
expect(resumed.code).toBe(0);
419419
expect(resumed.stderr).toContain("workflow status: completed");
420-
expect(committedStatus(path)).toBe("completed");
420+
expect(yield* committedStatus(path)).toBe("completed");
421421

422422
// Nothing the interruption had already committed was performed again, by
423423
// event id and in order, and the document's effects appear exactly once.
424-
const after = committedEffects(path);
424+
const after = yield* committedEffects(path);
425425
expect(after.slice(0, interrupted.before.length)).toEqual(interrupted.before);
426426
expect(after).toHaveLength(HELD_EFFECTS);
427427
expect(new Set(after.map((effect) => effect.name)).size).toBe(HELD_EFFECTS);
428428

429429
// The interrupted execution stays closed as it was, and the resume's is a
430430
// second, different one.
431-
const settled = committedExecutions(path);
431+
const settled = yield* committedExecutions(path);
432432
expect(settled).toHaveLength(2);
433433
expect(settled[0]?.executionId).toBe(owner);
434434
expect(settled[0]?.stopStatus).toBe("interrupted");
@@ -448,8 +448,8 @@ describe("Tier WFX — a killed workflow run resumes from its frontier", () => {
448448
document: "flows/holding.md",
449449
reached: HELD_EFFECTS,
450450
});
451-
expect(committedStatus(path)).toBe("running");
452-
expect(committedExecutions(path)).toHaveLength(1);
451+
expect(yield* committedStatus(path)).toBe("running");
452+
expect(yield* committedExecutions(path)).toHaveLength(1);
453453

454454
const options = {
455455
cwd: fixture.repository,
@@ -478,14 +478,14 @@ describe("Tier WFX — a killed workflow run resumes from its frontier", () => {
478478
// The loser was refused before it could begin anything: the run holds the
479479
// killed owner's execution and the winner's, and no third. A refusal that
480480
// still recorded an execution would be a second owner advancing the run.
481-
const settled = committedExecutions(path);
481+
const settled = yield* committedExecutions(path);
482482
expect(settled).toHaveLength(2);
483483
expect(settled[0]?.stopStatus).toBe("interrupted");
484484
expect(settled[1]?.stopStatus).toBe("completed");
485485

486486
// And one winner means one set of effects.
487-
expect(committedStatus(path)).toBe("completed");
488-
const after = committedEffects(path);
487+
expect(yield* committedStatus(path)).toBe("completed");
488+
const after = yield* committedEffects(path);
489489
expect(after).toHaveLength(HELD_EFFECTS);
490490
expect(new Set(after.map((effect) => effect.name)).size).toBe(HELD_EFFECTS);
491491
});

‎packages/cli/tests/workflow-fetch.test.ts‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ interface JournalRecord {
160160
}
161161

162162
/** Everything a second connection can read of the run's journal. */
163-
function rows(path: string): JournalRecord[] {
163+
function rows(path: string): Operation<JournalRecord[]> {
164164
return readRunDatabase(path, (database) =>
165165
database
166166
.prepare("SELECT record FROM journal_events ORDER BY sequence")
@@ -169,17 +169,19 @@ function rows(path: string): JournalRecord[] {
169169
);
170170
}
171171

172-
function committedFetches(path: string): JournalRecord[] {
173-
return rows(path).filter(
172+
function* committedFetches(path: string): Operation<JournalRecord[]> {
173+
const records = yield* rows(path);
174+
return records.filter(
174175
(record) =>
175176
record.type === "yield" &&
176177
record.description?.type === "fetch" &&
177178
record.result?.status === "ok",
178179
);
179180
}
180181

181-
function rootCloses(path: string): number {
182-
return rows(path).filter((record) => record.type === "close" && record.coroutineId === "root")
182+
function* rootCloses(path: string): Operation<number> {
183+
const records = yield* rows(path);
184+
return records.filter((record) => record.type === "close" && record.coroutineId === "root")
183185
.length;
184186
}
185187

@@ -211,15 +213,15 @@ describe("Tier FE — a workflow run's Fetch", () => {
211213

212214
yield* when(
213215
function* () {
214-
expect(committedFetches(path)).toHaveLength(1);
216+
expect(yield* committedFetches(path)).toHaveLength(1);
215217
},
216218
{ timeout: 60_000 },
217219
);
218220
process.kill(child.pid, "SIGKILL");
219221
yield* child.join();
220222
});
221223

222-
const [committed] = committedFetches(path);
224+
const [committed] = yield* committedFetches(path);
223225
expect(server.requests).toEqual(["GET /greeting"]);
224226
expect(committed?.description?.input).toEqual({
225227
url: `${server.origin}/greeting`,
@@ -245,7 +247,7 @@ describe("Tier FE — a workflow run's Fetch", () => {
245247
expect(names).toEqual(names.map((name) => name.toLowerCase()));
246248
expect(named["content-type"]).toBe("application/json");
247249
// The killed run left the root open, which is what makes it resumable.
248-
expect(rootCloses(path)).toBe(0);
250+
expect(yield* rootCloses(path)).toBe(0);
249251

250252
// Open the gate the document waits on, then resume.
251253
yield* writeTextFile(fixture.gate, "open\n");
@@ -261,10 +263,10 @@ describe("Tier FE — a workflow run's Fetch", () => {
261263
// The response came from the journal: the server was never asked again,
262264
// and the run still holds exactly one Fetch observation.
263265
expect(server.requests).toEqual(["GET /greeting"]);
264-
const after = committedFetches(path);
266+
const after = yield* committedFetches(path);
265267
expect(after).toHaveLength(1);
266268
expect(after[0]).toEqual(committed);
267-
expect(rootCloses(path)).toBe(1);
269+
expect(yield* rootCloses(path)).toBe(1);
268270
});
269271
});
270272
});

0 commit comments

Comments
 (0)