Skip to content

Commit 4d0eb0e

Browse files
Merge remote-tracking branch 'origin/main' into fix/grok-usage-window-duration
2 parents 4160402 + 034f3ba commit 4d0eb0e

6 files changed

Lines changed: 235 additions & 11 deletions

File tree

‎apps/server/src/serviceLauncher.test.ts‎

Lines changed: 121 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import type * as NodeChildProcess from "node:child_process";
33
// @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback.
44
import * as NodeFS from "node:fs";
5+
import * as NodeSqlite from "node:sqlite";
56

67
import * as NodeServices from "@effect/platform-node/NodeServices";
78
import { assert, it } from "@effect/vitest";
@@ -464,6 +465,125 @@ if (context.update?.status === "pending") {
464465
}),
465466
);
466467

468+
it.effect.each(["wal", "delete"] as const)(
469+
"backs up a %s-mode database (online when WAL) and restores its final state",
470+
(journalMode) =>
471+
Effect.gen(function* () {
472+
const fs = yield* FileSystem.FileSystem;
473+
const path = yield* Path.Path;
474+
const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-online-" });
475+
const statePath = path.join(root, "runtime", "service-state.json");
476+
const databasePath = path.join(root, "userdata", "state.sqlite");
477+
const observationsPath = path.join(root, "observations.json");
478+
yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
479+
// SQLite keeps -wal beside the symlink's target, so the backup must restore it there.
480+
yield* fs.symlink("actual.sqlite", databasePath);
481+
const seed = new NodeSqlite.DatabaseSync(databasePath);
482+
seed.exec(`
483+
PRAGMA journal_mode = ${journalMode};
484+
CREATE TABLE events (id INTEGER PRIMARY KEY, payload TEXT NOT NULL);
485+
WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 500)
486+
INSERT INTO events (payload) SELECT 'before the update ' || i FROM n;
487+
`);
488+
seed.close();
489+
490+
const encoded = JSON.stringify({ root, databasePath, observationsPath, journalMode });
491+
const childSource = `
492+
import { readdirSync, writeFileSync } from "node:fs";
493+
import { DatabaseSync } from "node:sqlite";
494+
const paths = ${encoded};
495+
const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
496+
if (context.update?.status === "pending") {
497+
writeFileSync(paths.observationsPath + ".trial", JSON.stringify(
498+
readdirSync(paths.root + "/runtime/db-backup/" + context.update.id).sort(),
499+
));
500+
const trial = new DatabaseSync(context.update.dbPath);
501+
trial.exec("PRAGMA journal_mode = " + paths.journalMode + "; INSERT INTO events (payload) VALUES ('trial');");
502+
trial.close();
503+
process.exit(1);
504+
} else if (context.update === undefined) {
505+
const db = new DatabaseSync(paths.databasePath);
506+
db.exec("PRAGMA busy_timeout = 5000;");
507+
db.exec("PRAGMA journal_mode = " + paths.journalMode + ";");
508+
const insert = db.prepare("INSERT INTO events (payload) VALUES (?)");
509+
const writer = setInterval(() => insert.run("written while serving"), 1);
510+
process.once("SIGTERM", () => {
511+
clearInterval(writer);
512+
insert.run("last write before stopping");
513+
const backupRoot = paths.root + "/runtime/db-backup";
514+
writeFileSync(paths.observationsPath, JSON.stringify({
515+
backupsAtStop: readdirSync(backupRoot).map((entry) => readdirSync(backupRoot + "/" + entry)),
516+
rows: db.prepare("SELECT count(*) AS n FROM events").get().n,
517+
}));
518+
db.close();
519+
process.exit(0);
520+
});
521+
process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath });
522+
} else {
523+
process.exit(0);
524+
}
525+
`;
526+
for (const version of ["1.0.0", "1.1.0"]) {
527+
yield* writeFakeRuntime(
528+
fs,
529+
path,
530+
path.join(root, "runtime", "versions", version),
531+
childSource,
532+
);
533+
}
534+
yield* Effect.promise(() =>
535+
writeServiceState(statePath, {
536+
protocol: SERVICE_LAUNCHER_PROTOCOL,
537+
activeVersion: "1.0.0",
538+
}),
539+
);
540+
541+
const launcher = new Launcher(
542+
root,
543+
yield* Effect.promise(() => readServiceState(statePath)),
544+
{
545+
backupSpaceProbe: { canClone: async () => false },
546+
},
547+
);
548+
yield* Effect.promise(() =>
549+
launcher.run().then(
550+
() => Promise.reject(new Error("launcher unexpectedly completed")),
551+
() => Promise.resolve(),
552+
),
553+
);
554+
555+
const state = yield* Effect.promise(() => readServiceState(statePath));
556+
assert.equal(state.update?.status, "rolled-back");
557+
const observed = JSON.parse(yield* fs.readFileString(observationsPath));
558+
// A WAL database's main file is copied before the server is asked to stop; others fall back offline.
559+
assert.deepEqual(observed.backupsAtStop, journalMode === "wal" ? [["database"]] : []);
560+
assert.isAbove(observed.rows, 501);
561+
// Only the online copy holds no -shm: the offline fallback copies the one the server left.
562+
assert.deepEqual(
563+
JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)),
564+
journalMode === "wal" ? ["database", "database-wal"] : ["database"],
565+
);
566+
const restored = new NodeSqlite.DatabaseSync(databasePath);
567+
try {
568+
assert.equal(restored.prepare("PRAGMA integrity_check").get()?.integrity_check, "ok");
569+
assert.equal(
570+
restored.prepare("SELECT count(*) AS n FROM events").get()?.n,
571+
observed.rows,
572+
);
573+
assert.equal(
574+
restored
575+
.prepare(
576+
"SELECT count(*) AS n FROM events WHERE payload = 'last write before stopping'",
577+
)
578+
.get()?.n,
579+
1,
580+
);
581+
} finally {
582+
restored.close();
583+
}
584+
}),
585+
);
586+
467587
const requestRejectedUpdate = (backupSpaceProbe: Partial<BackupSpaceProbe>) =>
468588
Effect.gen(function* () {
469589
const fs = yield* FileSystem.FileSystem;
@@ -670,7 +790,7 @@ setInterval(() => {}, 1_000);
670790
assert.include((yield* check(0.5 * 1024 ** 3, true)) ?? "", "1.0 GB needed, 0.5 GB free");
671791
const blocker = yield* check(8 * 1024 ** 3, false);
672792
assert.include(blocker ?? "", "Not enough free disk space");
673-
assert.include(blocker ?? "", "10.5 GB needed, 8.0 GB free");
793+
assert.include(blocker ?? "", "10.8 GB needed, 8.0 GB free");
674794
assert.include(blocker ?? "", "The server was not stopped");
675795
}),
676796
);

‎apps/server/src/serviceLauncher.ts‎

Lines changed: 95 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,8 @@ const GIB = 1024 ** 3;
162162
// Cloned backups still need headroom for trial writes and migrations to diverge.
163163
const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB;
164164
const CLONED_BACKUP_HEADROOM_FRACTION = 0.1;
165+
// While the online backup holds its snapshot, the live WAL and its copy can each reach the server's 128 MiB gate.
166+
const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2;
165167

166168
async function statDatabaseFiles(dbPath: string): Promise<ReadonlyArray<NodeFS.Stats>> {
167169
const stats: Array<NodeFS.Stats> = [];
@@ -242,7 +244,7 @@ export async function databaseBackupBlocker(
242244
await removeAbandonedBackupCopies(backupRoot);
243245
const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
244246
const size = files.reduce((total, file) => total + file.size, 0);
245-
const fullCopyBytes = size * 1.05;
247+
const fullCopyBytes = size * 1.05 + ONLINE_BACKUP_WAL_HEADROOM_BYTES;
246248
if (free >= fullCopyBytes) return undefined;
247249
const clonedBytes = Math.max(
248250
CLONED_BACKUP_MIN_HEADROOM_BYTES,
@@ -254,6 +256,83 @@ export async function databaseBackupBlocker(
254256
return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`;
255257
}
256258

259+
// A child-process copy: closing any descriptor this process opens on the database drops the snapshot's POSIX locks.
260+
async function copyWithoutOpening(source: string, destination: string): Promise<void> {
261+
// oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
262+
if (process.platform === "win32") return NodeFSP.copyFile(source, destination);
263+
if (!(await runCp([source, destination]))) throw new Error(`cp ${source} failed.`);
264+
}
265+
266+
// Copies the main file while the server runs; an open read transaction stops SQLite resetting the WAL,
267+
// which `finish` copies after the stop and which overrides any page the copy caught mid-checkpoint.
268+
// `-shm` is left out: its backfill count would make SQLite trust those pages. Past the WAL size gate,
269+
// a write-heavy server's new writes wait, uncommitted, until the server stops.
270+
async function startOnlineDatabaseBackup(
271+
baseDir: string,
272+
pending: PendingServiceUpdate,
273+
probe: Partial<BackupSpaceProbe> = {},
274+
): Promise<(() => Promise<void>) | undefined> {
275+
const backupDir = databaseBackupDir(baseDir, pending.id);
276+
if (await pathExists(backupDir)) return undefined;
277+
const { canClone } = { ...filesystemBackupSpaceProbe, ...probe };
278+
if (await canClone(pending.dbPath, NodePath.dirname(backupDir)).catch(() => true))
279+
return undefined;
280+
const NodeSqlite = await import("node:sqlite").catch(() => undefined);
281+
if (NodeSqlite === undefined) return undefined;
282+
283+
const stagingDir = `${backupDir}.staging`;
284+
const discardStaging = () =>
285+
NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined);
286+
const fallBack = async (cause: unknown) => {
287+
await discardStaging();
288+
process.stderr.write(
289+
`[service-launcher] Copying the database while the server runs failed; backing it up offline: ${cause instanceof Error ? cause.message : String(cause)}\n`,
290+
);
291+
return undefined;
292+
};
293+
let snapshot: InstanceType<typeof NodeSqlite.DatabaseSync> | undefined;
294+
// SQLite keeps -wal beside the resolved file, not beside a symlink to it.
295+
let dbPath = pending.dbPath;
296+
const release = () => {
297+
try {
298+
snapshot?.close();
299+
} catch {}
300+
snapshot = undefined;
301+
};
302+
try {
303+
await discardStaging();
304+
await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 });
305+
dbPath = await NodeFSP.realpath(pending.dbPath);
306+
snapshot = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true });
307+
const mode = String(snapshot.prepare("PRAGMA journal_mode").get()?.journal_mode);
308+
if (mode.toLowerCase() !== "wal") throw new Error(`journal_mode is ${mode}, not wal.`);
309+
snapshot.exec("BEGIN");
310+
snapshot.prepare("SELECT count(*) FROM sqlite_schema").get();
311+
const mainCopy = databaseBackupFile(stagingDir, "");
312+
await copyWithoutOpening(dbPath, mainCopy);
313+
await syncFile(mainCopy);
314+
} catch (cause) {
315+
release();
316+
return fallBack(cause);
317+
}
318+
return async () => {
319+
try {
320+
const wal = `${dbPath}-wal`;
321+
if (!(await pathExists(wal)))
322+
throw new Error("the WAL disappeared while the snapshot was open.");
323+
const walCopy = databaseBackupFile(stagingDir, "-wal");
324+
await copyWithoutOpening(wal, walCopy);
325+
await syncFile(walCopy);
326+
release();
327+
await NodeFSP.rename(stagingDir, backupDir);
328+
await syncDirectory(NodePath.dirname(backupDir));
329+
} catch (cause) {
330+
release();
331+
await fallBack(cause);
332+
}
333+
};
334+
}
335+
257336
// Preserve completed backups; restarted trials may already have changed the live database.
258337
async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise<void> {
259338
const backupDir = databaseBackupDir(baseDir, pending.id);
@@ -330,8 +409,13 @@ async function restoreDatabaseBackup(
330409

331410
await markDatabaseRestorePending(backupDir);
332411
const directories = new Set([NodePath.dirname(pending.dbPath)]);
412+
// SQLite keeps -wal and -shm beside the resolved database file, not beside a symlink to it.
413+
const mainTarget = await restoreTarget(pending.dbPath);
333414
for (const suffix of DB_FILE_SUFFIXES) {
334-
const livePath = `${pending.dbPath}${suffix}`;
415+
const livePath = suffix === "" ? pending.dbPath : `${mainTarget}${suffix}`;
416+
if (livePath !== `${pending.dbPath}${suffix}`) {
417+
await NodeFSP.rm(`${pending.dbPath}${suffix}`, { force: true });
418+
}
335419
const source = databaseBackupFile(backupDir, suffix);
336420
if (await pathExists(source)) {
337421
// Restore into the symlink target; cloning would replace the link.
@@ -708,8 +792,17 @@ export class Launcher {
708792
return;
709793
}
710794
this.#timer = undefined;
795+
// Anything thrown outside its own fallback still leaves the offline backup to run.
796+
const finishOnlineBackup = await startOnlineDatabaseBackup(
797+
this.#baseDir,
798+
pending,
799+
this.#backupSpaceProbe,
800+
).catch(() => undefined);
711801
this.#child = null;
712802
await terminateChild(child.process);
803+
await finishOnlineBackup?.();
804+
// A stop during a minutes-long copy should not start a trial; restart resumes the pending update.
805+
if (this.#stopRequested) return;
713806
await this.#startTrial(pending);
714807
}
715808

‎apps/web/src/components/RightPanelTabs.browser.test.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ describe("RightPanelTabs", () => {
6464
navigateHistory.mockReset();
6565
const { bridge, scroll } = trackpad();
6666
using _bridge = stubGlobal("desktopBridge", bridge);
67+
using _platform = vi.spyOn(navigator, "platform", "get").mockReturnValue("MacIntel");
6768
const panel = document.createElement("div");
6869
panel.style.cssText = "width: 320px; height: 200px; display: flex";
6970
const noop = () => {};

‎packages/contracts/src/assistantCitations.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,15 @@ import { EnvironmentId, MessageId, NonNegativeInt, ThreadId } from "./baseSchema
44
export const ASSISTANT_CITATION_MAX_TEXT_LENGTH = 8_000;
55
export const ASSISTANT_CITATION_MAX_COMMENT_LENGTH = 8_000;
66
export const ASSISTANT_CITATION_CONTEXT_LENGTH = 32;
7+
// Provider message IDs nest and re-encode their thread and turn IDs, so they outgrow a few hundred characters.
8+
export const ASSISTANT_CITATION_MAX_ID_LENGTH = 4_096;
79

810
/** Positions are UTF-16 offsets, not Markdown offsets. */
911
export const AssistantCitation = Schema.Struct({
1012
version: Schema.Literal(1),
11-
environmentId: EnvironmentId.check(Schema.isMaxLength(512)),
12-
threadId: ThreadId.check(Schema.isMaxLength(512)),
13-
messageId: MessageId.check(Schema.isMaxLength(512)),
13+
environmentId: EnvironmentId.check(Schema.isMaxLength(ASSISTANT_CITATION_MAX_ID_LENGTH)),
14+
threadId: ThreadId.check(Schema.isMaxLength(ASSISTANT_CITATION_MAX_ID_LENGTH)),
15+
messageId: MessageId.check(Schema.isMaxLength(ASSISTANT_CITATION_MAX_ID_LENGTH)),
1416
text: Schema.String.check(
1517
Schema.isNonEmpty(),
1618
Schema.isMaxLength(ASSISTANT_CITATION_MAX_TEXT_LENGTH),

‎packages/shared/src/assistantCitations.test.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test";
22
import {
33
ASSISTANT_CITATION_CONTEXT_LENGTH,
44
ASSISTANT_CITATION_MAX_COMMENT_LENGTH,
5+
ASSISTANT_CITATION_MAX_ID_LENGTH,
56
ASSISTANT_CITATION_MAX_TEXT_LENGTH,
67
EnvironmentId,
78
MessageId,
@@ -126,9 +127,9 @@ describe("assistant citation references", () => {
126127
it("accepts complete 8k CJK quotes and comments with maximum-sized source selectors", () => {
127128
const largeCitation: AssistantCitation = {
128129
...citation,
129-
environmentId: EnvironmentId.make("環".repeat(512)),
130-
threadId: ThreadId.make("線".repeat(512)),
131-
messageId: MessageId.make("文".repeat(512)),
130+
environmentId: EnvironmentId.make("環".repeat(ASSISTANT_CITATION_MAX_ID_LENGTH)),
131+
threadId: ThreadId.make("線".repeat(ASSISTANT_CITATION_MAX_ID_LENGTH)),
132+
messageId: MessageId.make("文".repeat(ASSISTANT_CITATION_MAX_ID_LENGTH)),
132133
text: "引".repeat(ASSISTANT_CITATION_MAX_TEXT_LENGTH),
133134
comment: "注".repeat(ASSISTANT_CITATION_MAX_COMMENT_LENGTH),
134135
start: Number.MAX_SAFE_INTEGER - ASSISTANT_CITATION_MAX_TEXT_LENGTH,

‎packages/shared/src/assistantCitations.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import {
2+
ASSISTANT_CITATION_CONTEXT_LENGTH,
23
ASSISTANT_CITATION_MAX_COMMENT_LENGTH,
4+
ASSISTANT_CITATION_MAX_ID_LENGTH,
35
ASSISTANT_CITATION_MAX_TEXT_LENGTH,
46
AssistantCitation,
57
} from "@t3tools/contracts";
@@ -8,9 +10,14 @@ import * as Schema from "effect/Schema";
810

911
const CITATION_PROTOCOL = "t3-citation:";
1012
const CITATION_HREF_PREFIX = `${CITATION_PROTOCOL}//v1/`;
11-
// Percent encoding needs up to nine characters per UTF-16 code unit; 16k covers selectors.
13+
// Percent encoding needs up to nine characters per UTF-16 code unit; 1k covers keys and offsets.
1214
const MAX_CITATION_HREF_LENGTH =
13-
9 * (ASSISTANT_CITATION_MAX_TEXT_LENGTH + ASSISTANT_CITATION_MAX_COMMENT_LENGTH) + 16_000;
15+
9 *
16+
(ASSISTANT_CITATION_MAX_TEXT_LENGTH +
17+
ASSISTANT_CITATION_MAX_COMMENT_LENGTH +
18+
3 * ASSISTANT_CITATION_MAX_ID_LENGTH +
19+
2 * ASSISTANT_CITATION_CONTEXT_LENGTH) +
20+
1_000;
1421
const CITATION_LINK = new RegExp(
1522
String.raw`\[Assistant quote\]\((${CITATION_HREF_PREFIX}[^\s)]{1,${MAX_CITATION_HREF_LENGTH - CITATION_HREF_PREFIX.length}})\)`,
1623
"g",

0 commit comments

Comments
 (0)