Skip to content

Commit 2298adc

Browse files
author
anqiang.ex
committed
fix(core): append V1 messages written after the migration completed
V1→V2 import reached `completed` once and never looked at an already imported session again. A pre-upgrade V1 client that keeps running after the upgrade therefore kept writing to the legacy `message` table while V2 wrote to `session_message`, and the extra messages never reached the projection: the TUI showed a truncated session and `opencode session export` silently dropped them. Each imported session now records the newest legacy `time_created` it imported, and completion appends only legacy messages past that watermark. Absence from `session_message` alone is not enough: `RevertEvent.Committed` deletes rows from the revert boundary on, so a message the user reverted in V2 would otherwise be re-appended on the next start. The watermark is written in the same transaction as the rows and advanced past every message it considered, including ones the transform drops, so nothing is reconsidered on later starts. Appended rows take `seq` after the highest projected one, and the aggregate sequence moves to `max(current, last appended)`; the report warned that rewinding `event_sequence` corrupts the log. Totals are the appended messages' own amounts, added to the live row, because `transformed.session` is a V1-only recomputation that would replace V2's cost and tokens and clear `revert` and `time_compacting`. A database migrated before the watermark existed cannot tell a late message from a reverted one, so the first run adopts what is there and appends nothing; messages written from then on are picked up normally. Repairing what such a database already lost needs a separate path, and is not attempted here. Adds four database-level cases: the late append itself, a reverted message staying gone, appended totals not overwriting V2 state, and the watermark-less adoption.
1 parent f394d9d commit 2298adc

2 files changed

Lines changed: 333 additions & 3 deletions

File tree

‎packages/core/src/database/v1-migration.bun.ts‎

Lines changed: 162 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -537,7 +537,10 @@ export function run(options: Options = {}): Effect.Effect<RunResult, never, Data
537537
const db = (yield* Database.Service).db
538538
const global = yield* Global.Service
539539
const state = yield* readState(db)
540-
if (state?.phase === "completed") return { status: "completed" as const }
540+
if (state?.phase === "completed") {
541+
yield* appendLateMessages(db)
542+
return { status: "completed" as const }
543+
}
541544
if (!(yield* hasLegacySessions(db))) return { status: "completed" as const }
542545
const now = Date.now()
543546
yield* db.run(sql`
@@ -666,6 +669,9 @@ export function run(options: Options = {}): Effect.Effect<RunResult, never, Data
666669
set: { seq: transformed.watermark, owner_id: null },
667670
})
668671
.run()
672+
// Record how far this session's legacy messages reached, so a later start can tell a message
673+
// written afterwards from one V2 removed through a committed revert.
674+
yield* writeImportedThrough(tx, next.id, latest(sourceMessages))
669675
}),
670676
)
671677
.pipe(Effect.orDie)
@@ -699,6 +705,155 @@ export function run(options: Options = {}): Effect.Effect<RunResult, never, Data
699705
)
700706
}
701707

708+
/**
709+
* A pre-upgrade V1 client can keep appending to a session this process already imported. Completion was a
710+
* permanent latch, so those messages never reached `session_message` and export silently truncated the
711+
* session (#57000). Each imported session records how far its legacy messages reached, so only messages
712+
* written later are added: a message V2 removed through a committed revert stays removed.
713+
*/
714+
function appendLateMessages(db: Database.Interface["db"]) {
715+
return Effect.gen(function* () {
716+
if (!(yield* hasLegacyTable(db, "message"))) return
717+
const sessions = yield* db.all<{ id: string }>(sql`
718+
SELECT DISTINCT m.session_id AS id
719+
FROM message m
720+
JOIN session_v2 projected_session ON projected_session.id = m.session_id
721+
WHERE NOT EXISTS (SELECT 1 FROM session_message projected WHERE projected.id = m.id)
722+
`)
723+
yield* Effect.forEach(sessions, (session) => appendLateSession(db, session.id), { discard: true })
724+
})
725+
}
726+
727+
function appendLateSession(db: Database.Interface["db"], sessionID: string) {
728+
return Effect.gen(function* () {
729+
const session = yield* db
730+
.select()
731+
.from(SessionTable)
732+
.where(eq(SessionTable.id, SessionSchema.ID.make(sessionID)))
733+
.get()
734+
// A session that is not projected yet belongs to the initial pass, which imports it in full.
735+
if (!session) return
736+
const sourceMessages = yield* db.all<SourceMessage>(
737+
sql`SELECT id, session_id, time_created, time_updated, data FROM message WHERE session_id = ${sessionID}`,
738+
)
739+
const through = yield* readImportedThrough(db, sessionID)
740+
// A database migrated before this watermark existed cannot tell a late message from one reverted in V2,
741+
// so adopt what is already there and add nothing; messages written from now on are still picked up.
742+
if (through === undefined) {
743+
yield* writeImportedThrough(db, sessionID, latest(sourceMessages))
744+
return
745+
}
746+
const projected = yield* db.all<{ id: string; seq: number }>(
747+
sql`SELECT id, seq FROM session_message WHERE session_id = ${sessionID} ORDER BY seq`,
748+
)
749+
const known = new Set(projected.map((row) => row.id))
750+
const lateSources = sourceMessages.filter((row) => row.time_created > through && !known.has(row.id))
751+
if (lateSources.length === 0) return
752+
const late = new Set(lateSources.map((row) => row.id))
753+
const sourceParts = yield* db.all<SourcePart>(
754+
sql`SELECT id, message_id, session_id, time_created, time_updated, data FROM part WHERE session_id = ${sessionID}`,
755+
)
756+
const transformed = transformSession({ session, messages: sourceMessages, parts: sourceParts })
757+
// The legacy tool notice is synthesized rather than a V1 message, and already sits where the import put it.
758+
const messages = transformed.messages.filter((message) => late.has(message.id))
759+
const lateIDs = new Set(messages.map((message) => message.id))
760+
yield* Effect.forEach(
761+
transformed.warnings.filter((warning) => warning.messageID !== undefined && lateIDs.has(warning.messageID)),
762+
(warning) => Effect.logWarning("Skipped V1 migration row", warning),
763+
)
764+
// Totals are added as the late messages' own amounts. Writing `transformed.session` would replace the
765+
// live row's cost and tokens with V1-only sums and clear `revert` and `time_compacting` (#57000 review).
766+
const added = transformSession({
767+
session,
768+
messages: lateSources,
769+
parts: sourceParts.filter((row) => late.has(row.message_id)),
770+
}).session
771+
const first = (projected.at(-1)?.seq ?? -1) + 1
772+
const sequence = yield* db.get<{ seq: number }>(
773+
sql`SELECT seq FROM event_sequence WHERE aggregate_id = ${sessionID}`,
774+
)
775+
// Appending must never rewind the aggregate sequence: V2 may already be past the migrated watermark.
776+
const watermark = Math.max(sequence?.seq ?? -1, first + messages.length - 1)
777+
// Every message past the watermark counts as handled, including ones the transform drops, so a row that
778+
// can never be projected is not reconsidered on every start.
779+
const reached = latest(lateSources)
780+
yield* db
781+
.transaction((tx) =>
782+
Effect.gen(function* () {
783+
yield* Effect.forEach(messages, (message, index) =>
784+
tx
785+
.insert(SessionMessageTable)
786+
.values({
787+
id: SessionMessage.ID.make(message.id),
788+
session_id: SessionSchema.ID.make(message.session_id),
789+
type: message.type,
790+
seq: first + index,
791+
time_created: message.time_created,
792+
time_updated: message.time_updated,
793+
data: sql`${JSON.stringify(message.data)}`,
794+
})
795+
.run(),
796+
)
797+
yield* tx
798+
.update(SessionTable)
799+
.set({
800+
cost: sql`${SessionTable.cost} + ${added.cost}`,
801+
tokens_input: sql`${SessionTable.tokens_input} + ${added.tokens_input}`,
802+
tokens_output: sql`${SessionTable.tokens_output} + ${added.tokens_output}`,
803+
tokens_reasoning: sql`${SessionTable.tokens_reasoning} + ${added.tokens_reasoning}`,
804+
tokens_cache_read: sql`${SessionTable.tokens_cache_read} + ${added.tokens_cache_read}`,
805+
tokens_cache_write: sql`${SessionTable.tokens_cache_write} + ${added.tokens_cache_write}`,
806+
})
807+
.where(eq(SessionTable.id, session.id))
808+
.run()
809+
yield* tx
810+
.insert(EventSequenceTable)
811+
.values({ aggregate_id: session.id, seq: watermark })
812+
.onConflictDoUpdate({ target: EventSequenceTable.aggregate_id, set: { seq: watermark } })
813+
.run()
814+
yield* writeImportedThrough(tx, sessionID, reached)
815+
}),
816+
)
817+
.pipe(Effect.orDie)
818+
})
819+
}
820+
821+
type ThroughWriter = Pick<Database.Interface["db"], "insert">
822+
823+
/** The newest legacy `time_created` a session has already imported, absent before watermarks existed. */
824+
function readImportedThrough(db: Database.Interface["db"], sessionID: string) {
825+
return db
826+
.select({ value: KVTable.value })
827+
.from(KVTable)
828+
.where(eq(KVTable.key, importedThroughKey(sessionID)))
829+
.get()
830+
.pipe(
831+
Effect.map((row) => {
832+
const value = row?.value
833+
if (!value || typeof value !== "object" || !("through" in value)) return undefined
834+
return typeof value.through === "number" ? value.through : undefined
835+
}),
836+
Effect.orDie,
837+
)
838+
}
839+
840+
function writeImportedThrough(db: ThroughWriter, sessionID: string, through: number) {
841+
return db
842+
.insert(KVTable)
843+
.values({ key: importedThroughKey(sessionID), value: { through } })
844+
.onConflictDoUpdate({ target: KVTable.key, set: { value: { through }, time_updated: Date.now() } })
845+
.run()
846+
.pipe(Effect.orDie)
847+
}
848+
849+
function importedThroughKey(sessionID: string) {
850+
return `migration.v1-v2:through:${sessionID}`
851+
}
852+
853+
function latest(messages: ReadonlyArray<SourceMessage>) {
854+
return messages.reduce((newest, message) => Math.max(newest, message.time_created), 0)
855+
}
856+
702857
function nextPath(options: Options, data: string) {
703858
if (options.nextDatabasePath) return options.nextDatabasePath
704859
if (process.env.OPENCODE_DB === ":memory:") return undefined
@@ -1127,9 +1282,13 @@ function parseState(input: unknown): MigrationState | undefined {
11271282
if (typeof input.cursor === "string") return { phase: "sessions", cursor: input.cursor }
11281283
}
11291284

1130-
function hasLegacySessions(db: Database.Interface["db"]) {
1131-
return db.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'session'`).pipe(
1285+
function hasLegacyTable(db: Database.Interface["db"], table: "session" | "message") {
1286+
return db.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ${table}`).pipe(
11321287
Effect.map((row) => row !== undefined),
11331288
Effect.orDie,
11341289
)
11351290
}
1291+
1292+
function hasLegacySessions(db: Database.Interface["db"]) {
1293+
return hasLegacyTable(db, "session")
1294+
}

‎packages/core/test/v1-migration.test.ts‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1370,6 +1370,177 @@ describe("V1Migration database workflow", () => {
13701370
)
13711371
})
13721372

1373+
// A pre-upgrade V1 client can keep appending to a session V2 has already imported. The completion
1374+
// latch meant those messages never reached session_message, so export silently truncated the session.
1375+
test("appends legacy messages written after the import completed", async () => {
1376+
await database(
1377+
Effect.gen(function* () {
1378+
const { db } = yield* Database.Service
1379+
yield* db.run(
1380+
sql`INSERT INTO project (id, worktree, time_created, time_updated, sandboxes) VALUES ('global', '/tmp/test', 1, 2, '[]')`,
1381+
)
1382+
yield* db.run(
1383+
sql`INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) VALUES ('ses_test', 'global', 'test', '/tmp/test', 'Test', '1', 1, 2)`,
1384+
)
1385+
const insertLegacy = (message: V1Migration.SourceMessage) =>
1386+
db.run(
1387+
sql`INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (${message.id}, 'ses_test', ${message.time_created}, ${message.time_updated}, ${message.data})`,
1388+
)
1389+
1390+
const first = user("msg_000000000010aaaaaaaaaaaaaa", {}, 10)
1391+
const second = user("msg_000000000020aaaaaaaaaaaaaa", {}, 20)
1392+
yield* insertLegacy(first)
1393+
yield* insertLegacy(second)
1394+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1395+
expect(yield* db.all(sql`SELECT id, seq FROM session_message ORDER BY seq`)).toEqual([
1396+
{ id: first.id, seq: 0 },
1397+
{ id: second.id, seq: 1 },
1398+
])
1399+
1400+
// V2 keeps writing to the session after the import, and advances the aggregate sequence.
1401+
yield* db.run(
1402+
sql`INSERT INTO session_message (id, session_id, type, seq, time_created, time_updated, data) VALUES ('msg_from_v2', 'ses_test', 'user', 2, 30, 31, '{"text":"from v2","time":{"created":30}}')`,
1403+
)
1404+
yield* db.run(sql`UPDATE event_sequence SET seq = 5 WHERE aggregate_id = 'ses_test'`)
1405+
1406+
const third = user("msg_000000000030aaaaaaaaaaaaaa", {}, 40)
1407+
const fourth = user("msg_000000000040aaaaaaaaaaaaaa", {}, 50)
1408+
yield* insertLegacy(third)
1409+
yield* insertLegacy(fourth)
1410+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1411+
1412+
// The late messages append after what is already projected: nothing is deleted or renumbered.
1413+
expect(yield* db.all(sql`SELECT id, seq FROM session_message ORDER BY seq`)).toEqual([
1414+
{ id: first.id, seq: 0 },
1415+
{ id: second.id, seq: 1 },
1416+
{ id: "msg_from_v2", seq: 2 },
1417+
{ id: third.id, seq: 3 },
1418+
{ id: fourth.id, seq: 4 },
1419+
])
1420+
// Appending must not rewind the aggregate sequence, which would corrupt the event log.
1421+
expect(yield* db.get(sql`SELECT seq FROM event_sequence WHERE aggregate_id = 'ses_test'`)).toEqual({ seq: 5 })
1422+
1423+
// A third run finds nothing to append and leaves the projection alone.
1424+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1425+
expect((yield* db.get<{ value: number }>(sql`SELECT COUNT(*) AS value FROM session_message`))?.value).toBe(5)
1426+
}),
1427+
)
1428+
})
1429+
1430+
// V2 removes rows on a committed revert, so "absent from session_message" alone cannot mean "never
1431+
// imported". The per-session watermark keeps a reverted message from coming back.
1432+
test("does not re-append a legacy message a V2 revert removed", async () => {
1433+
await database(
1434+
Effect.gen(function* () {
1435+
const { db } = yield* Database.Service
1436+
yield* db.run(
1437+
sql`INSERT INTO project (id, worktree, time_created, time_updated, sandboxes) VALUES ('global', '/tmp/test', 1, 2, '[]')`,
1438+
)
1439+
yield* db.run(
1440+
sql`INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) VALUES ('ses_test', 'global', 'test', '/tmp/test', 'Test', '1', 1, 2)`,
1441+
)
1442+
const first = user("msg_000000000010aaaaaaaaaaaaaa", {}, 10)
1443+
const second = user("msg_000000000020aaaaaaaaaaaaaa", {}, 20)
1444+
for (const message of [first, second])
1445+
yield* db.run(
1446+
sql`INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (${message.id}, 'ses_test', ${message.time_created}, ${message.time_updated}, ${message.data})`,
1447+
)
1448+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1449+
1450+
// RevertEvent.Committed deletes the projection from the boundary on and clears the session's revert.
1451+
yield* db.run(sql`DELETE FROM session_message WHERE id = ${second.id}`)
1452+
1453+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1454+
expect((yield* db.all<{ id: string }>(sql`SELECT id FROM session_message`)).map((row) => row.id)).toEqual([first.id])
1455+
}),
1456+
)
1457+
})
1458+
1459+
// `transformed.session` is a V1-only recomputation, so writing it over the live row would replace V2's
1460+
// totals and clear revert and time_compacting. Only the appended messages' own amounts may be added.
1461+
test("adds appended totals to V2 session state instead of replacing it", async () => {
1462+
await database(
1463+
Effect.gen(function* () {
1464+
const { db } = yield* Database.Service
1465+
yield* db.run(
1466+
sql`INSERT INTO project (id, worktree, time_created, time_updated, sandboxes) VALUES ('global', '/tmp/test', 1, 2, '[]')`,
1467+
)
1468+
yield* db.run(
1469+
sql`INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) VALUES ('ses_test', 'global', 'test', '/tmp/test', 'Test', '1', 1, 2)`,
1470+
)
1471+
const first = user("msg_000000000010aaaaaaaaaaaaaa", {}, 10)
1472+
yield* db.run(
1473+
sql`INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (${first.id}, 'ses_test', ${first.time_created}, ${first.time_updated}, ${first.data})`,
1474+
)
1475+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1476+
1477+
// V2 continues the session: it owns the totals and carries live revert and compaction state.
1478+
yield* db.run(
1479+
sql`UPDATE session_v2 SET cost = 42, tokens_input = 7, tokens_output = 8, agent = 'v2-agent', revert = '{"messageID":"' || ${first.id} || '"}', time_compacting = 99 WHERE id = 'ses_test'`,
1480+
)
1481+
1482+
const prompt = user("msg_000000000030aaaaaaaaaaaaaa", {}, 30)
1483+
const reply = assistant("msg_000000000040aaaaaaaaaaaaaa", prompt.id, {}, 40)
1484+
for (const message of [prompt, reply])
1485+
yield* db.run(
1486+
sql`INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (${message.id}, 'ses_test', ${message.time_created}, ${message.time_updated}, ${message.data})`,
1487+
)
1488+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1489+
1490+
const row = yield* db.get(
1491+
sql`SELECT cost, tokens_input, tokens_output, agent, revert, time_compacting FROM session_v2 WHERE id = 'ses_test'`,
1492+
)
1493+
expect(row).toEqual({
1494+
cost: 43,
1495+
tokens_input: 9,
1496+
tokens_output: 11,
1497+
agent: "v2-agent",
1498+
revert: `{"messageID":"${first.id}"}`,
1499+
time_compacting: 99,
1500+
})
1501+
}),
1502+
)
1503+
})
1504+
1505+
// Databases migrated before watermarks existed cannot tell a late message from a reverted one, so the
1506+
// first run adopts what is there without appending, and later messages are then picked up normally.
1507+
test("adopts a completed database that has no watermark", async () => {
1508+
await database(
1509+
Effect.gen(function* () {
1510+
const { db } = yield* Database.Service
1511+
yield* db.run(
1512+
sql`INSERT INTO project (id, worktree, time_created, time_updated, sandboxes) VALUES ('global', '/tmp/test', 1, 2, '[]')`,
1513+
)
1514+
yield* db.run(
1515+
sql`INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) VALUES ('ses_test', 'global', 'test', '/tmp/test', 'Test', '1', 1, 2)`,
1516+
)
1517+
const first = user("msg_000000000010aaaaaaaaaaaaaa", {}, 10)
1518+
yield* db.run(
1519+
sql`INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (${first.id}, 'ses_test', ${first.time_created}, ${first.time_updated}, ${first.data})`,
1520+
)
1521+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1522+
yield* db.run(sql`DELETE FROM kv WHERE key = 'migration.v1-v2:through:ses_test'`)
1523+
1524+
const second = user("msg_000000000020aaaaaaaaaaaaaa", {}, 20)
1525+
yield* db.run(
1526+
sql`INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (${second.id}, 'ses_test', ${second.time_created}, ${second.time_updated}, ${second.data})`,
1527+
)
1528+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1529+
expect((yield* db.all<{ id: string }>(sql`SELECT id FROM session_message`)).map((row) => row.id)).toEqual([first.id])
1530+
1531+
const third = user("msg_000000000030aaaaaaaaaaaaaa", {}, 30)
1532+
yield* db.run(
1533+
sql`INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES (${third.id}, 'ses_test', ${third.time_created}, ${third.time_updated}, ${third.data})`,
1534+
)
1535+
expect(yield* V1Migration.run()).toEqual({ status: "completed" })
1536+
expect((yield* db.all<{ id: string }>(sql`SELECT id FROM session_message ORDER BY seq`)).map((row) => row.id)).toEqual([
1537+
first.id,
1538+
third.id,
1539+
])
1540+
}),
1541+
)
1542+
})
1543+
13731544
test("serializes concurrent callers and migrates each session once", async () => {
13741545
await database(
13751546
Effect.gen(function* () {

0 commit comments

Comments
 (0)