Skip to content

Commit c3cf6d5

Browse files
committed
fix(usage): unchanged Antigravity databases are not decoded again after a restart
The Antigravity memo from #17139 lived only in memory, so the first usage scan after every restart decoded the whole history again. The Antigravity usage source now saves it in the state directory and loads it on its first scan, keyed as before by the size, mtime and ctime of each database and its WAL, and holding candidates as read, before aliases merge and the date window applies. A malformed entry is dropped and its database read again.
1 parent 61eb4af commit c3cf6d5

4 files changed

Lines changed: 420 additions & 10 deletions

File tree

‎apps/server/src/provider/Drivers/AntigravityUsage.ts‎

Lines changed: 75 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@
22
* AntigravityUsage - Antigravity's usage history as a usage source.
33
*
44
* Antigravity keeps conversations in SQLite databases under its data
5-
* directories and each instance's profile. Parsed databases stay in memory
6-
* while a database and its WAL keep the same `(size, mtime, ctime)`, so a
7-
* warm scan decodes only what changed.
5+
* directories and each instance's profile. Parsed databases are reused while a
6+
* database and its WAL keep the same `(size, mtime, ctime)`, so a warm scan
7+
* decodes only what changed. They persist in the server's state directory, so
8+
* an unchanged history is not decoded again after a restart.
89
*
910
* @module provider/Drivers/AntigravityUsage
1011
*/
@@ -17,16 +18,31 @@ import type {
1718
ProviderUsageReader,
1819
ProviderUsageScan,
1920
} from "@t3tools/provider-core/server/usage";
21+
import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite";
2022
import * as HostProcess from "@t3tools/shared/HostProcess";
2123
import * as Context from "effect/Context";
2224
import * as Crypto from "effect/Crypto";
2325
import * as Effect from "effect/Effect";
26+
import * as Fiber from "effect/Fiber";
2427
import * as FileSystem from "effect/FileSystem";
2528
import * as Layer from "effect/Layer";
2629
import * as Path from "effect/Path";
30+
import * as Schema from "effect/Schema";
31+
import * as Semaphore from "effect/Semaphore";
2732

2833
import { resolveAntigravityInstanceDirectories } from "../antigravityAuthSupport.ts";
29-
import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts";
34+
import {
35+
ANTIGRAVITY_USAGE_CACHE_FILE_NAME,
36+
decodeAntigravityUsageCache,
37+
makeAntigravityUsageCache,
38+
makeAntigravityUsageCacheWriter,
39+
readAntigravityUsage,
40+
} from "./antigravityUsageReader.ts";
41+
42+
/** The cache file is narrowed by hand in `antigravityUsageReader`, so JSON is enough here. */
43+
const decodeCacheFile = Schema.decodeUnknownEffect(
44+
Schema.fromJsonString(Schema.Unknown as unknown as Schema.Codec<unknown>),
45+
);
3046

3147
export class AntigravityUsage extends Context.Service<
3248
AntigravityUsage,
@@ -36,6 +52,8 @@ export class AntigravityUsage extends Context.Service<
3652
readonly instances: ReadonlyArray<ProviderUsageInstance<AntigravitySettings>>;
3753
readonly windowStartMs: number;
3854
}) => Effect.Effect<ReadonlyArray<ProviderUsageScan>, UsageReadError>;
55+
/** Waits for every cache write scheduled so far, as a restart would need. */
56+
readonly awaitPersisted: Effect.Effect<void>;
3957
}
4058
>()("t3/provider/Drivers/AntigravityUsage") {}
4159

@@ -46,6 +64,53 @@ const make = Effect.gen(function* () {
4664
const host = yield* ProviderHost.ProviderHost;
4765
const hostEnvironment = yield* HostProcess.Environment;
4866
const cache = makeAntigravityUsageCache();
67+
const cachePath = path.join(host.paths.stateDir, ANTIGRAVITY_USAGE_CACHE_FILE_NAME);
68+
const writeCache = makeAntigravityUsageCacheWriter();
69+
let cacheDirty = false;
70+
71+
/** Loads the saved cache once; concurrent first scans await the same load. */
72+
const ensureLoaded = yield* Effect.cached(
73+
fileSystem.readFileString(cachePath).pipe(
74+
Effect.flatMap((raw) => decodeCacheFile(raw)),
75+
Effect.catchCause(() => Effect.succeed(null)),
76+
Effect.map((document) => {
77+
for (const [key, entry] of decodeAntigravityUsageCache(document)) cache.set(key, entry);
78+
}),
79+
),
80+
);
81+
82+
// Serialized so an older snapshot never lands after a newer one. The dirty
83+
// flag is cleared before encoding, so a change while the write is in flight
84+
// marks it dirty again; a failed write restores it so the next persist retries.
85+
const persistLock = yield* Semaphore.make(1);
86+
const persist = Effect.gen(function* () {
87+
if (!cacheDirty) return;
88+
cacheDirty = false;
89+
yield* Effect.sync(() => writeCache(cache)).pipe(
90+
Effect.flatMap((contents) => writeFileStringAtomically({ filePath: cachePath, contents })),
91+
Effect.provideService(FileSystem.FileSystem, fileSystem),
92+
Effect.provideService(Path.Path, path),
93+
Effect.catchCause(() =>
94+
Effect.sync(() => {
95+
cacheDirty = true;
96+
}),
97+
),
98+
);
99+
}).pipe(persistLock.withPermit, Effect.withSpan("AntigravityUsage.persist"));
100+
101+
const pendingPersists = new Set<Fiber.Fiber<void>>();
102+
/** Writes the cache in the background, after the scan that changed it answers. */
103+
const schedulePersist = Effect.forkDetach(persist).pipe(
104+
Effect.map((fiber) => {
105+
pendingPersists.add(fiber);
106+
fiber.addObserver(() => pendingPersists.delete(fiber));
107+
}),
108+
);
109+
const awaitPersisted = Effect.suspend(() => Fiber.awaitAll([...pendingPersists])).pipe(
110+
Effect.asVoid,
111+
);
112+
// A write still running at shutdown finishes first, so the next start keeps it.
113+
yield* Effect.addFinalizer(() => awaitPersisted);
49114

50115
/** `ANTIGRAVITY_DATA_DIR` (comma-separated) or the defaults, canonicalized. */
51116
const dataRoots = Effect.gen(function* () {
@@ -74,6 +139,7 @@ const make = Effect.gen(function* () {
74139
instances,
75140
windowStartMs,
76141
}) {
142+
yield* ensureLoaded;
77143
const roots = yield* dataRoots;
78144
// Only configured instances have a profile; the implicit default never ran.
79145
for (const { instanceId } of instances.filter((instance) => instance.configured)) {
@@ -109,6 +175,10 @@ const make = Effect.gen(function* () {
109175
Effect.provideService(FileSystem.FileSystem, fileSystem),
110176
Effect.provideService(Path.Path, path),
111177
);
178+
if (result.cacheChanged) {
179+
cacheDirty = true;
180+
yield* schedulePersist;
181+
}
112182
const scanned: ProviderUsageScan[] = [];
113183
for (const dir of conversationDirs) {
114184
const exists = yield* fileSystem
@@ -127,7 +197,7 @@ const make = Effect.gen(function* () {
127197
return scanned;
128198
});
129199

130-
return AntigravityUsage.of({ scan });
200+
return AntigravityUsage.of({ scan, awaitPersisted });
131201
});
132202

133203
export const layer = Layer.effect(AntigravityUsage, make);

‎apps/server/src/provider/Drivers/antigravityUsageReader.test.ts‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,12 @@ import * as Layer from "effect/Layer";
1111
import * as Path from "effect/Path";
1212
import * as SqlClient from "effect/sql/SqlClient";
1313

14-
import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts";
14+
import {
15+
decodeAntigravityUsageCache,
16+
makeAntigravityUsageCache,
17+
makeAntigravityUsageCacheWriter,
18+
readAntigravityUsage,
19+
} from "./antigravityUsageReader.ts";
1520

1621
/** A writable connection that stays open until the enclosing scope closes. */
1722
const openDatabase = Effect.fn("openDatabase")(function* (filename: string) {
@@ -335,6 +340,63 @@ describe("readAntigravityUsage", () => {
335340
}).pipe(Effect.provide(NodeServices.layer)),
336341
);
337342

343+
it.effect("serves a saved Antigravity cache after a restart without reading the database", () =>
344+
Effect.gen(function* () {
345+
const path = yield* Path.Path;
346+
const dir = yield* tempDir;
347+
const db = yield* openDatabase(path.join(dir, "session-1.db"));
348+
yield* db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)");
349+
yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [0, antigravityGeneration("r-1")]);
350+
// No response id, so its dedupe key is not an alias and is saved as is.
351+
yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [1, antigravityGeneration("")]);
352+
const cache = makeAntigravityUsageCache();
353+
const first = yield* readAntigravityUsage(dir, 0, cache);
354+
assert.isTrue(first.cacheChanged);
355+
const saved: unknown = JSON.parse(makeAntigravityUsageCacheWriter()(cache));
356+
357+
yield* db.exec("BEGIN EXCLUSIVE");
358+
const second = yield* readAntigravityUsage(dir, 0, decodeAntigravityUsageCache(saved));
359+
assert.deepStrictEqual(second.errors, []);
360+
assert.isFalse(second.cacheChanged);
361+
assert.deepStrictEqual(
362+
records(second).map((record) => record.dedupeKey),
363+
["antigravity:11:r-1", "antigravity:session-1:generation:1:0"],
364+
);
365+
assert.deepStrictEqual(records(second), records(first));
366+
yield* db.exec("ROLLBACK");
367+
}).pipe(Effect.provide(NodeServices.layer)),
368+
);
369+
370+
it.effect("drops a corrupt saved Antigravity database instead of serving part of it", () =>
371+
Effect.gen(function* () {
372+
const path = yield* Path.Path;
373+
const dir = yield* tempDir;
374+
yield* seedDatabase(path.join(dir, "session-1.db"), (db) =>
375+
Effect.gen(function* () {
376+
yield* db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)");
377+
yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [
378+
0,
379+
antigravityGeneration("r-1"),
380+
]);
381+
yield* db.run("INSERT INTO gen_metadata VALUES (?, ?)", [
382+
1,
383+
antigravityGeneration("r-2"),
384+
]);
385+
}),
386+
);
387+
const cache = makeAntigravityUsageCache();
388+
yield* readAntigravityUsage(dir, 0, cache);
389+
const saved = JSON.parse(makeAntigravityUsageCacheWriter()(cache)) as {
390+
files: Record<string, { c: unknown[][] }>;
391+
};
392+
const [entry] = Object.values(saved.files);
393+
entry!.c[1]![5] = -1;
394+
395+
assert.strictEqual(decodeAntigravityUsageCache(saved).size, 0);
396+
assert.strictEqual(decodeAntigravityUsageCache({ ...saved, version: 2 }).size, 0);
397+
}).pipe(Effect.provide(NodeServices.layer)),
398+
);
399+
338400
it.effect("rereads an Antigravity database rewritten with its size and mtime restored", () =>
339401
Effect.gen(function* () {
340402
const fileSystem = yield* FileSystem.FileSystem;

0 commit comments

Comments
 (0)