Skip to content

Commit 5e4b29d

Browse files
dzianisvengineer
andcommitted
fix(session): recover orphaned assistant messages on startup (#59)
When the server crashes or restarts while a session is actively executing tool calls, in-memory session state is lost but the database retains stale incomplete messages. This leaves sessions permanently stuck in a 'Thinking' state with no way to recover. Add Session.recover() that runs during InstanceBootstrap to: - find assistant messages with no time.completed - force running/pending tool parts to error status - mark orphaned messages as completed - emit bus events so connected frontends update Closes anomalyco#19023 Co-authored-by: engineer <[email protected]>
1 parent a2acf91 commit 5e4b29d

5 files changed

Lines changed: 163 additions & 16 deletions

File tree

‎bun.lock‎

Lines changed: 19 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/opencode/package.json‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,6 @@
9696
"@aws-sdk/credential-providers": "3.993.0",
9797
"@clack/prompts": "1.0.0-alpha.1",
9898
"@effect/platform-node": "4.0.0-beta.31",
99-
"@gitlab/gitlab-ai-provider": "3.6.0",
100-
"@gitlab/opencode-gitlab-auth": "1.3.3",
10199
"@hono/standard-validator": "0.1.5",
102100
"@hono/zod-validator": "catalog:",
103101
"@modelcontextprotocol/sdk": "1.27.1",
@@ -110,8 +108,8 @@
110108
"@opencode-ai/sdk": "workspace:*",
111109
"@opencode-ai/util": "workspace:*",
112110
"@openrouter/ai-sdk-provider": "1.5.4",
113-
"@opentui/core": "0.1.87",
114-
"@opentui/solid": "0.1.87",
111+
"@opentui/core": "0.1.90",
112+
"@opentui/solid": "0.1.90",
115113
"@parcel/watcher": "2.5.1",
116114
"@pierre/diffs": "catalog:",
117115
"@solid-primitives/event-bus": "1.1.2",
@@ -130,7 +128,7 @@
130128
"drizzle-orm": "catalog:",
131129
"effect": "catalog:",
132130
"fuzzysort": "3.1.0",
133-
"gitlab-ai-provider": "6.0.0",
131+
"gitlab-ai-provider": "5.2.2",
134132
"glob": "13.0.5",
135133
"google-auth-library": "10.5.0",
136134
"gray-matter": "4.0.3",
@@ -141,8 +139,7 @@
141139
"mime-types": "3.0.2",
142140
"minimatch": "10.0.3",
143141
"open": "10.1.2",
144-
"opencode-gitlab-auth": "2.0.1",
145-
"opencode-poe-auth": "0.0.1",
142+
"opencode-gitlab-auth": "2.0.0",
146143
"opentui-spinner": "0.0.6",
147144
"partial-json": "0.1.7",
148145
"remeda": "catalog:",

‎packages/opencode/src/project/bootstrap.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@ import { Command } from "../command"
1111
import { Instance } from "./instance"
1212
import { Log } from "@/util/log"
1313
import { ShareNext } from "@/share/share-next"
14+
import { Session } from "@/session"
1415

1516
export async function InstanceBootstrap() {
1617
Log.Default.info("bootstrapping", { directory: Instance.directory })
18+
await Session.recover()
1719
await Plugin.init()
1820
ShareNext.init()
1921
Format.init()

‎packages/opencode/src/session/index.ts‎

Lines changed: 137 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@ import { Config } from "../config/config"
99
import { Flag } from "../flag/flag"
1010
import { Installation } from "../installation"
1111

12-
import { Database, NotFoundError, eq, and, gte, isNull, desc, like, inArray, lt } from "../storage/db"
13-
import { SyncEvent } from "../sync"
12+
import { Database, NotFoundError, eq, and, or, gte, isNull, desc, like, inArray, lt, sql } from "../storage/db"
1413
import type { SQL } from "../storage/db"
1514
import { SessionTable } from "./session.sql"
1615
import { ProjectTable } from "../project/project.sql"
@@ -880,6 +879,142 @@ export namespace Session {
880879
(input) => runPromise((svc) => svc.updatePartDelta(input)),
881880
)
882881

882+
export const getUsage = fn(
883+
z.object({
884+
model: z.custom<Provider.Model>(),
885+
usage: z.custom<LanguageModelV2Usage>(),
886+
metadata: z.custom<ProviderMetadata>().optional(),
887+
}),
888+
(input) => {
889+
const safe = (value: number) => {
890+
if (!Number.isFinite(value)) return 0
891+
return value
892+
}
893+
const inputTokens = safe(input.usage.inputTokens ?? 0)
894+
const outputTokens = safe(input.usage.outputTokens ?? 0)
895+
const reasoningTokens = safe(input.usage.reasoningTokens ?? 0)
896+
897+
const cacheReadInputTokens = safe(input.usage.cachedInputTokens ?? 0)
898+
const cacheWriteInputTokens = safe(
899+
(input.metadata?.["anthropic"]?.["cacheCreationInputTokens"] ??
900+
// @ts-expect-error
901+
input.metadata?.["bedrock"]?.["usage"]?.["cacheWriteInputTokens"] ??
902+
// @ts-expect-error
903+
input.metadata?.["venice"]?.["usage"]?.["cacheCreationInputTokens"] ??
904+
0) as number,
905+
)
906+
907+
// OpenRouter provides inputTokens as the total count of input tokens (including cached).
908+
// AFAIK other providers (OpenRouter/OpenAI/Gemini etc.) do it the same way e.g. vercel/ai#8794 (comment)
909+
// Anthropic does it differently though - inputTokens doesn't include cached tokens.
910+
// It looks like OpenCode's cost calculation assumes all providers return inputTokens the same way Anthropic does (I'm guessing getUsage logic was originally implemented with anthropic), so it's causing incorrect cost calculation for OpenRouter and others.
911+
const excludesCachedTokens = !!(input.metadata?.["anthropic"] || input.metadata?.["bedrock"])
912+
const adjustedInputTokens = safe(
913+
excludesCachedTokens ? inputTokens : inputTokens - cacheReadInputTokens - cacheWriteInputTokens,
914+
)
915+
916+
const total = iife(() => {
917+
// Anthropic doesn't provide total_tokens, also ai sdk will vastly undercount if we
918+
// don't compute from components
919+
if (
920+
input.model.api.npm === "@ai-sdk/anthropic" ||
921+
input.model.api.npm === "@ai-sdk/amazon-bedrock" ||
922+
input.model.api.npm === "@ai-sdk/google-vertex/anthropic"
923+
) {
924+
return adjustedInputTokens + outputTokens + cacheReadInputTokens + cacheWriteInputTokens
925+
}
926+
return input.usage.totalTokens
927+
})
928+
929+
const tokens = {
930+
total,
931+
input: adjustedInputTokens,
932+
output: outputTokens,
933+
reasoning: reasoningTokens,
934+
cache: {
935+
write: cacheWriteInputTokens,
936+
read: cacheReadInputTokens,
937+
},
938+
}
939+
940+
const costInfo =
941+
input.model.cost?.experimentalOver200K && tokens.input + tokens.cache.read > 200_000
942+
? input.model.cost.experimentalOver200K
943+
: input.model.cost
944+
return {
945+
cost: safe(
946+
new Decimal(0)
947+
.add(new Decimal(tokens.input).mul(costInfo?.input ?? 0).div(1_000_000))
948+
.add(new Decimal(tokens.output).mul(costInfo?.output ?? 0).div(1_000_000))
949+
.add(new Decimal(tokens.cache.read).mul(costInfo?.cache?.read ?? 0).div(1_000_000))
950+
.add(new Decimal(tokens.cache.write).mul(costInfo?.cache?.write ?? 0).div(1_000_000))
951+
// TODO: update models.dev to have better pricing model, for now:
952+
// charge reasoning tokens at the same rate as output tokens
953+
.add(new Decimal(tokens.reasoning).mul(costInfo?.output ?? 0).div(1_000_000))
954+
.toNumber(),
955+
),
956+
tokens,
957+
}
958+
},
959+
)
960+
961+
/**
962+
* Recover orphaned assistant messages left incomplete after a server crash or restart.
963+
* Finds assistant messages with no `time.completed`, forces their non-terminal tool
964+
* parts to error status, marks the messages as completed, and emits bus events so
965+
* connected frontends update.
966+
*
967+
* @see https://github.com/anomalyco/opencode/issues/19023
968+
*/
969+
export async function recover() {
970+
const rows = Database.use((db) =>
971+
db
972+
.select()
973+
.from(MessageTable)
974+
.where(
975+
and(
976+
sql`json_extract(${MessageTable.data}, '$.role') = 'assistant'`,
977+
sql`json_extract(${MessageTable.data}, '$.time.completed') is null`,
978+
),
979+
)
980+
.all(),
981+
)
982+
if (rows.length === 0) return
983+
log.info("recovering orphaned assistant messages", { count: rows.length })
984+
const now = Date.now()
985+
for (const row of rows) {
986+
const msg = { ...row.data, id: row.id, sessionID: row.session_id } as MessageV2.Assistant
987+
// Fix non-terminal tool parts
988+
const parts = await MessageV2.parts(row.id)
989+
for (const part of parts) {
990+
if (part.type !== "tool") continue
991+
if (part.state.status === "completed" || part.state.status === "error") continue
992+
await updatePart({
993+
...part,
994+
state: {
995+
...part.state,
996+
status: "error",
997+
error: "Tool execution was interrupted by server restart",
998+
time: {
999+
start: part.state.status === "running" ? part.state.time.start : now,
1000+
end: now,
1001+
},
1002+
},
1003+
})
1004+
}
1005+
// Mark message completed
1006+
msg.time.completed = now
1007+
await updateMessage(msg)
1008+
log.info("recovered orphaned message", { messageID: row.id, sessionID: row.session_id })
1009+
}
1010+
}
1011+
1012+
export class BusyError extends Error {
1013+
constructor(public readonly sessionID: string) {
1014+
super(`Session ${sessionID} is busy`)
1015+
}
1016+
}
1017+
8831018
export const initialize = fn(
8841019
z.object({ sessionID: SessionID.zod, modelID: ModelID.zod, providerID: ProviderID.zod, messageID: MessageID.zod }),
8851020
(input) => runPromise((svc) => svc.initialize(input)),

‎packages/opencode/src/tool/registry.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import { Flag } from "@/flag/flag"
2727
import { Log } from "@/util/log"
2828
import { LspTool } from "./lsp"
2929
import { Truncate } from "./truncate"
30+
3031
import { ApplyPatchTool } from "./apply_patch"
3132
import { Glob } from "../util/glob"
3233
import { pathToFileURL } from "url"

0 commit comments

Comments
 (0)