Skip to content

Commit 3e70f55

Browse files
authored
fix(dev): skip auto-seeding large databases (#25365)
1 parent 3ed0688 commit 3e70f55

4 files changed

Lines changed: 103 additions & 16 deletions

File tree

‎AGENTS.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ The most common defect in this repo is a change that works on the path you teste
9090

9191
An empty database is a bad test. Seed your worktree's `.t3` with a copy of real data instead of pointing at live state:
9292

93-
- The first `vp run dev` in a worktree does this for you when its database is missing. `--no-seed` or `T3CODE_DEV_SEED=0` skips it.
93+
- The first `vp run dev` in a worktree does this for you when its database is missing and the source is at most 500 MiB. Larger sources warn and dev starts with an empty database. `--no-seed` or `T3CODE_DEV_SEED=0` skips seeding. Manual migration still accepts larger sources and may use substantial disk space before pruning.
9494
- Run `vp run migrate-dev-db` with your dev server stopped to refresh. It rebuilds `<worktree>/.t3/userdata/statev2.sqlite` from a read-only snapshot of `~/.t3/userdata/statev2.sqlite`, the developer's real data. It keeps recent projects and their stopped threads, and drops scheduled tasks, pending work, and auth sessions, so your dev server never runs the developer's agents. Raise `--projects` and `--threads-per-project` for more data.
9595
- Refresh `statev2.sqlite`, not `state.sqlite`. The server copies the V1 `state.sqlite` only when `statev2.sqlite` is missing.
9696
- Bring `secrets` and `settings.json` only if the flow under test needs them.

‎docs/operations/development.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,10 @@ Add `--browser` to open a browser automatically.
6262
Linked worktrees default to their own `.t3/userdata`, even when `T3CODE_HOME` is set.
6363
The main checkout defaults to `~/.t3/dev`. An explicit `--home-dir` wins in both cases.
6464
The first start in a worktree seeds its empty database from a read-only copy of
65-
`~/.t3/userdata`; `--no-seed` or `T3CODE_DEV_SEED=0` skips that.
65+
`~/.t3/userdata/statev2.sqlite` if the source is at most 500 MiB. Larger sources warn
66+
and dev starts with an empty database. `--no-seed` or `T3CODE_DEV_SEED=0` skips seeding.
67+
To copy a larger source, run `vp run migrate-dev-db` with dev stopped. It may use
68+
substantial disk space before pruning.
6669
Never run a development server against the live `~/.t3/userdata`.
6770
See [test data](../../AGENTS.md#test-data) for copying a consistent database snapshot.
6871

‎scripts/dev-runner.test.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,12 @@ import * as NodePath from "node:path";
66
import * as NetService from "@t3tools/shared/Net";
77
import * as HostProcess from "@t3tools/shared/HostProcess";
88
import { assert, describe, it } from "@effect/vitest";
9+
import * as ByteSize from "effect/ByteSize";
910
import * as ConfigProvider from "effect/ConfigProvider";
1011
import * as Effect from "effect/Effect";
12+
import * as FileSystem from "effect/FileSystem";
1113
import * as Layer from "effect/Layer";
14+
import * as Logger from "effect/Logger";
1215
import * as Path from "effect/Path";
1316
import * as PlatformError from "effect/PlatformError";
1417
import * as Sink from "effect/Sink";
@@ -739,6 +742,80 @@ it.layer(NodeServices.layer)("dev-runner", (it) => {
739742
}).pipe(Effect.scoped),
740743
);
741744

745+
it.effect.each([
746+
{ sourceBytes: 500 * 1024 * 1024 - 1, shouldSeed: true },
747+
{ sourceBytes: 500 * 1024 * 1024, shouldSeed: true },
748+
{ sourceBytes: 500 * 1024 * 1024 + 1, shouldSeed: false },
749+
])(
750+
"checks source size before seeding an empty worktree: $sourceBytes bytes",
751+
({ sourceBytes, shouldSeed }) =>
752+
Effect.gen(function* () {
753+
const fs = yield* FileSystem.FileSystem;
754+
const path = yield* Path.Path;
755+
const root = yield* makeWorktree;
756+
const home = path.join(root, "home");
757+
const source = path.join(home, ".t3", "userdata", "statev2.sqlite");
758+
NodeFS.mkdirSync(path.dirname(source), { recursive: true });
759+
NodeFS.writeFileSync(source, "");
760+
const spawned: Array<ReadonlyArray<string>> = [];
761+
const warnings: Array<string> = [];
762+
const spawnerLayer = Layer.succeed(
763+
ChildProcessSpawner.ChildProcessSpawner,
764+
ChildProcessSpawner.make((command) => {
765+
spawned.push((command as unknown as { readonly args: ReadonlyArray<string> }).args);
766+
return Effect.succeed(mockProcess(0));
767+
}),
768+
);
769+
770+
yield* runDevRunnerWithInput({ ...devServerInput, t3Home: undefined }).pipe(
771+
Effect.provide([
772+
emptyConfigLayer,
773+
netServiceLayer,
774+
spawnerLayer,
775+
Logger.layer([
776+
Logger.make((options) => {
777+
if (options.logLevel === "Warn") warnings.push(String(options.message));
778+
}),
779+
]),
780+
]),
781+
Effect.provideService(FileSystem.FileSystem, {
782+
...fs,
783+
stat: (file) =>
784+
fs
785+
.stat(file)
786+
.pipe(
787+
Effect.map((info) =>
788+
file === source ? { ...info, size: ByteSize.bytes(sourceBytes) } : info,
789+
),
790+
),
791+
}),
792+
Effect.provideService(HostProcess.Platform, "linux"),
793+
Effect.provideService(HostProcess.WorkingDirectory, root),
794+
Effect.provideService(HostProcess.HomeDirectory, home),
795+
Effect.provideService(HostProcess.Environment, {}),
796+
);
797+
798+
assert.equal(
799+
spawned.some((args) => args.some((arg) => arg.endsWith("migrate-dev-db.ts"))),
800+
shouldSeed,
801+
);
802+
assert.deepStrictEqual(spawned.at(-1), [
803+
"run",
804+
"--filter=t3",
805+
"dev",
806+
...devServerInput.runArgs,
807+
]);
808+
if (shouldSeed) {
809+
assert.deepStrictEqual(warnings, []);
810+
} else {
811+
assert.lengthOf(warnings, 1);
812+
assert.include(warnings[0]!, source);
813+
assert.include(warnings[0]!, "skipping automatic database seeding");
814+
assert.include(warnings[0]!, "vp run migrate-dev-db");
815+
}
816+
}).pipe(Effect.scoped),
817+
);
818+
742819
it.effect("falls back to an ambient T3CODE_HOME outside a worktree", () =>
743820
Effect.gen(function* () {
744821
const path = yield* Path.Path;

‎scripts/dev-runner.ts‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,7 @@ const FETCH_BAD_PORTS = new Set([
4141
]);
4242
// Dev binds loopback; other interfaces cannot establish port availability.
4343
const DEV_PORT_PROBE_HOSTS = ["127.0.0.1", "::1"] as const;
44-
// A pruned migrate-dev-db copy is tens of MB; some worktrees hold a full copy
45-
// of the live database instead.
44+
// Migration prunes after copying, so large sources also need an automatic-seeding limit.
4645
const LARGE_DEV_DATABASE_BYTES = 500 * 1024 * 1024;
4746

4847
/** Only loopback and wildcard binds keep the loopback proxy target reachable. */
@@ -603,7 +602,7 @@ const runRepoScript = Effect.fn("dev-runner.runRepoScript")(function* (
603602
});
604603

605604
/**
606-
* The first start in a worktree seeds its empty database from the live install
605+
* The first start in a worktree seeds its empty database from a small live database
607606
* and warms the web dependency cache. Each step checks for its own output, so
608607
* later starts skip it.
609608
*/
@@ -634,17 +633,25 @@ const prepareFirstStart = Effect.fn("dev-runner.prepareFirstStart")(function* (i
634633
if (
635634
input.seed &&
636635
!(yield* exists(path.join(stateDir, "statev2.sqlite"))) &&
637-
!(yield* exists(path.join(stateDir, "state.sqlite"))) &&
638-
(yield* exists(source))
636+
!(yield* exists(path.join(stateDir, "state.sqlite")))
639637
) {
640-
yield* Effect.logInfo(
641-
`[dev-runner] seeding ${input.worktreeHome} from ${source} (read-only). --no-seed or T3CODE_DEV_SEED=0 skips this.`,
642-
);
643-
yield* runRepoScript(
644-
"apps/server/scripts/migrate-dev-db.ts",
645-
["--base-dir", input.worktreeHome, "--source", source],
646-
input.env,
647-
);
638+
const info = yield* fs.stat(source).pipe(Effect.option);
639+
if (Option.isSome(info)) {
640+
if (Number(info.value.size) > LARGE_DEV_DATABASE_BYTES) {
641+
yield* Effect.logWarning(
642+
`[dev-runner] ${source} exceeds 500 MiB; skipping automatic database seeding and starting with an empty database. Run \`vp run migrate-dev-db\` manually with dev stopped if needed; it may use substantial disk space before pruning.`,
643+
);
644+
} else {
645+
yield* Effect.logInfo(
646+
`[dev-runner] seeding ${input.worktreeHome} from ${source} (read-only). --no-seed or T3CODE_DEV_SEED=0 skips this.`,
647+
);
648+
yield* runRepoScript(
649+
"apps/server/scripts/migrate-dev-db.ts",
650+
["--base-dir", input.worktreeHome, "--source", source],
651+
input.env,
652+
);
653+
}
654+
}
648655
}
649656
}
650657

@@ -953,7 +960,7 @@ const devRunnerCli = Command.make("dev-runner", {
953960
),
954961
seed: Flag.Boolean("seed").pipe(
955962
Flag.withDescription(
956-
"Seed an empty worktree database from ~/.t3 on first start (read-only, via migrate-dev-db). On by default; --no-seed or T3CODE_DEV_SEED=0 skips it.",
963+
"Seed an empty worktree database from ~/.t3 on first start (read-only, via migrate-dev-db) if the source is at most 500 MiB. On by default; --no-seed or T3CODE_DEV_SEED=0 skips it.",
957964
),
958965
Flag.withFallbackConfig(optionalBooleanConfig("T3CODE_DEV_SEED")),
959966
),

0 commit comments

Comments
 (0)