Skip to content

Commit 53a8023

Browse files
committed
feat(core): keep MCP tools after a dropped connection and reconnect on demand
Behind the mcp_resilience flag. A server whose connection drops on its own (process exit, closed transport, expired session) keeps offering its last tools and shows the reason as its status error. The next request that needs it reconnects, sharing one attempt; after a failed attempt requests fail at once with the last error for 1 s, doubling to 30 s. Auth failures go to needs_auth and drop the tools. Each server's last tool list is cached on disk, keyed by a hash of its connection identity, and offered while it starts.
1 parent e42d622 commit 53a8023

6 files changed

Lines changed: 532 additions & 6 deletions

File tree

‎packages/client/src/promise/generated/types.ts‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/core/src/mcp/connection.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ export interface Events {
3232
readonly settled: Effect.Effect<void>
3333
/** The live connection closed on its own. Runs under the lock; the owner decides how to stop. */
3434
readonly closed: (reason: string) => Effect.Effect<void>
35+
/**
36+
* The server no longer knows this connection's session. Runs under the lock. Resolves true when the owner took over
37+
* reconnecting; otherwise the connection stops and starts again in place.
38+
*/
39+
readonly expired?: Effect.Effect<boolean>
3540
/** A catalog changed: the server said so, or the connection was torn down and every catalog emptied. */
3641
readonly changed: (kind: Kind) => Effect.Effect<void>
3742
/**
@@ -132,6 +137,7 @@ export function make(input: Input): Connection {
132137
Effect.gen(function* () {
133138
if (state.client !== client) return
134139
yield* Effect.logInfo("mcp session expired, reconnecting", { server: input.name })
140+
if (input.events.expired && (yield* input.events.expired)) return
135141
yield* stop
136142
yield* start
137143
}).pipe(input.lock)

‎packages/core/src/mcp/index.ts‎

Lines changed: 165 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { ephemeral } from "@opencode/schema/event"
66
import type { Session } from "@opencode/schema/session"
77
import type { Transport } from "@modelcontextprotocol/client"
88
import { createHash } from "node:crypto"
9+
import path from "node:path"
910
import { isDeepStrictEqual } from "node:util"
1011
import {
1112
Clock,
@@ -16,6 +17,7 @@ import {
1617
FiberSet,
1718
Latch,
1819
Layer,
20+
Option,
1921
Schema,
2022
Scope,
2123
Semaphore,
@@ -25,9 +27,14 @@ import {
2527
import { FetchHttpClient } from "effect/http"
2628
import { makeLocationNode } from "@opencode/util/effect/app-node"
2729
import { httpFetch } from "@opencode/util/effect/app-node-platform"
30+
import { FSUtil } from "@opencode/util/fs-util"
31+
import { Global } from "@opencode/util/global"
32+
import { Hash } from "@opencode/util/hash"
33+
import { Config } from "../config.js"
2834
import { Credential } from "../credential.js"
2935
import { Bus } from "../bus.js"
3036
import { Environment } from "../environment/index.js"
37+
import { Flag } from "../flag.js"
3138
import { Form } from "../form.js"
3239
import { Integration } from "../integration.js"
3340
import { KeyedMutex } from "../effect/keyed-mutex.js"
@@ -106,6 +113,12 @@ type ServerEntry = {
106113
// Set when a remote server is registered as an OAuth integration; the credential lives in the global store.
107114
integrationID?: Integration.ID
108115
registration?: State.Registration
116+
// Tools offered while the connection has none: the last list of a connection that dropped on its own, or the
117+
// disk-cached list of a server still starting. The next connect replaces it; a failure or an explicit stop drops it.
118+
retained?: ReadonlyArray<McpClient.Tool>
119+
// Set while the connection is down after dropping on its own. The next request that needs the server reconnects,
120+
// unless a reconnect failed and its backoff has not passed yet. An explicit lifecycle operation clears it.
121+
disconnected?: { failures: number; retryAt: number }
109122
}
110123

111124
// A client-hosted MCP server attached over a connection the client opened. Internal: never listed,
@@ -120,6 +133,38 @@ type AttachedEntry = {
120133
// How long a model call may wait for a server that is still connecting, counted from when it began.
121134
const STARTUP_GRACE_MS = 2_000
122135

136+
// After a failed reconnect, requests fail at once for 1 s, doubling per consecutive failure up to 30 s.
137+
const RECONNECT_BACKOFF_MS = 1_000
138+
const RECONNECT_MAX_BACKOFF_MS = 30_000
139+
const disconnectedError = (error: string) => `${error}. Its tools stay listed; it reconnects when next used.`
140+
141+
// Last listed tools per server, keyed by everything that changes the connection (see `McpPool.key`).
142+
const CachedTools = Schema.fromJsonString(
143+
Schema.Struct({
144+
tools: Schema.Array(
145+
Schema.Struct({
146+
name: Schema.String,
147+
title: Schema.optional(Schema.String),
148+
description: Schema.optional(Schema.String),
149+
inputSchema: Schema.StructWithRest(
150+
Schema.Struct({
151+
type: Schema.Literal("object"),
152+
properties: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
153+
required: Schema.optional(Schema.mutable(Schema.Array(Schema.String))),
154+
}),
155+
[Schema.Record(Schema.String, Schema.Unknown)],
156+
),
157+
outputSchema: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
158+
annotations: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
159+
execution: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
160+
_meta: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
161+
}),
162+
),
163+
}),
164+
)
165+
const decodeCachedTools = Schema.decodeUnknownOption(CachedTools)
166+
const encodeCachedTools = Schema.encodeSync(CachedTools)
167+
123168
// Bounds re-listing the tools of an attached connection that takes over the top of its stack.
124169
const FALLBACK_LIST_TIMEOUT = "5 seconds"
125170

@@ -208,6 +253,10 @@ export const layer = (options?: Options) =>
208253
const credentials = yield* Credential.Service
209254
const fetch = yield* FetchHttpClient.Fetch
210255
const pool = yield* McpPool.Service
256+
const configService = yield* Config.Service
257+
const global = yield* Global.Service
258+
const fsutil = yield* FSUtil.Service
259+
const resilient = Flag.enabled("mcp_resilience").pipe(Effect.provideService(Config.Service, configService))
211260
const root = yield* Effect.scope
212261
const fork = yield* FiberSet.makeRuntime<never, void, never>()
213262
// Server scopes close in parallel on shutdown, so one slow server's stop grace does not delay
@@ -282,11 +331,19 @@ export const layer = (options?: Options) =>
282331
if (!entry) return yield* new NotFoundError({ server: name })
283332
return entry
284333
})
334+
// A request that needs a server disconnected on its own reconnects it first. Concurrent requests share the
335+
// attempt through the server lock; within the backoff after a failed attempt they get the last error at once.
285336
const connected = Effect.fnUntraced(function* (name: ServerName) {
286337
const entry = yield* requireServer(name)
287338
yield* entry.startup.await
339+
if (entry.disconnected) yield* revive(entry).pipe(locks.withLock(name))
288340
return { entry, client: entry.connection.client }
289341
})
342+
const revive = Effect.fnUntraced(function* (entry: ServerEntry) {
343+
if (entry.connection.client || !entry.disconnected) return
344+
if ((yield* Clock.currentTimeMillis) < entry.disconnected.retryAt) return
345+
yield* entry.connection.start
346+
})
290347

291348
// The credential one start connects with: the integration's latest, or none. Read it once per start and
292349
// pass the result along, so anything keyed on it and the provider it builds agree on the account.
@@ -307,6 +364,62 @@ export const layer = (options?: Options) =>
307364
)
308365
})
309366

367+
// One file per connection identity, so a change to the command, URL, environment, headers or signed-in
368+
// account reads a different file. Secrets reach the name only as fingerprints inside a one-way hash.
369+
const toolsFile = Effect.fnUntraced(function* (entry: ServerEntry) {
370+
const credential = yield* selectCredential(entry)
371+
const { McpStdio } = yield* Effect.promise(() => import("./stdio.js"))
372+
const key = McpPool.key({
373+
config: entry.config,
374+
cwd: entry.config.type === "local" ? McpStdio.resolveCwd(location.directory, entry.config.cwd) : undefined,
375+
credential: credential?.id,
376+
clientInfo: options?.clientInfo,
377+
})
378+
return path.join(
379+
global.cache,
380+
"mcp",
381+
"tools",
382+
`${Hash.sha256(JSON.stringify([location.workspaceID, key]))}.json`,
383+
)
384+
})
385+
386+
const loadTools = Effect.fnUntraced(function* (name: ServerName, entry: ServerEntry) {
387+
if (entry.config.disabled || !(yield* resilient)) return
388+
const text = yield* fsutil
389+
.readFileStringSafe(yield* toolsFile(entry))
390+
.pipe(Effect.orElseSucceed(() => undefined))
391+
if (text === undefined) return
392+
entry.retained = Option.getOrUndefined(decodeCachedTools(text))?.tools
393+
if (entry.retained) yield* bus.publish(McpEvent.ToolsChanged, { server: name })
394+
})
395+
396+
const saveTools = (entry: ServerEntry) =>
397+
fork(
398+
Effect.gen(function* () {
399+
const tools = entry.connection.tools
400+
if (!tools || !(yield* resilient)) return
401+
yield* fsutil.writeWithDirs(yield* toolsFile(entry), encodeCachedTools({ tools }))
402+
}).pipe(Effect.catch((error) => Effect.logWarning("failed to cache MCP tools", { error: error.message }))),
403+
)
404+
405+
const dropRetained = Effect.fnUntraced(function* (name: ServerName, entry: ServerEntry) {
406+
if (!entry.retained) return
407+
entry.retained = undefined
408+
yield* bus.publish(McpEvent.ToolsChanged, { server: name })
409+
})
410+
411+
// A connection that went away on its own keeps offering its last tools and reports why it went; the next request
412+
// that needs it reconnects. Runs under the server lock.
413+
const drop = Effect.fnUntraced(function* (name: ServerName, entry: ServerEntry, reason: string) {
414+
// A shared connection has already emptied its catalogs, so fall back to the last list read from it.
415+
entry.retained = entry.connection.tools ?? entry.tagged?.source
416+
entry.disconnected = { failures: 0, retryAt: 0 }
417+
yield* entry.connection.stop
418+
entry.status = { status: "failed", error: disconnectedError(reason) }
419+
yield* Effect.logWarning("mcp connection dropped", { server: name, reason })
420+
yield* bus.publish(McpEvent.StatusChanged, { server: name })
421+
})
422+
310423
const elicitation: McpClient.ElicitationHandler = {
311424
create: (input: {
312425
readonly server: string
@@ -441,6 +554,9 @@ export const layer = (options?: Options) =>
441554
}),
442555
connected: Effect.gen(function* () {
443556
entry.status = { status: "connected" }
557+
entry.retained = undefined
558+
entry.disconnected = undefined
559+
saveTools(entry)
444560
yield* Effect.logInfo("mcp connected", { server: name, tools: entry.connection.tools?.length })
445561
// Announce every list a consumer may read, since each reads them only on these events. A
446562
// connection another Location already holds has loaded its prompts, and no later change
@@ -452,9 +568,25 @@ export const layer = (options?: Options) =>
452568
}),
453569
failed: (failure) =>
454570
Effect.gen(function* () {
571+
// A failed reconnect keeps the tools and backs off; an auth failure drops them like any other.
572+
const disconnected = entry.disconnected
573+
if (disconnected && failure.status === "failed") {
574+
disconnected.failures += 1
575+
disconnected.retryAt =
576+
(yield* Clock.currentTimeMillis) +
577+
Math.min(
578+
RECONNECT_BACKOFF_MS * 2 ** Math.min(disconnected.failures - 1, 5),
579+
RECONNECT_MAX_BACKOFF_MS,
580+
)
581+
entry.status = { status: "failed", error: disconnectedError(failure.error) }
582+
yield* Effect.logWarning("mcp reconnect failed", { server: name, error: failure.error })
583+
return yield* bus.publish(McpEvent.StatusChanged, { server: name })
584+
}
455585
entry.status = failure
586+
entry.disconnected = undefined
456587
yield* Effect.logWarning("mcp connect failed", { server: name, status: entry.status })
457588
yield* bus.publish(McpEvent.StatusChanged, { server: name })
589+
yield* dropRetained(name, entry)
458590
}),
459591
// Every start runs under the server lock, so the attempt read here is the one it began.
460592
settled: Effect.suspend(() => {
@@ -464,11 +596,22 @@ export const layer = (options?: Options) =>
464596
}),
465597
closed: (reason) =>
466598
Effect.gen(function* () {
599+
if (yield* resilient) return yield* drop(name, entry, reason)
467600
entry.status = { status: "failed", error: reason }
468601
yield* entry.connection.stop
469602
yield* bus.publish(McpEvent.StatusChanged, { server: name })
470603
}),
471-
changed: (kind) => bus.publish(changes[kind], { server: name }),
604+
// The request that found the session expired needs the server now, so it reconnects at once.
605+
expired: Effect.gen(function* () {
606+
if (!(yield* resilient)) return false
607+
yield* drop(name, entry, "MCP session expired")
608+
yield* revive(entry)
609+
return true
610+
}),
611+
changed: (kind) => {
612+
if (kind === "tools") saveTools(entry)
613+
return bus.publish(changes[kind], { server: name })
614+
},
472615
},
473616
}),
474617
}
@@ -478,7 +621,7 @@ export const layer = (options?: Options) =>
478621
// MCP.tools() runs every step and the tool registry compares tools by identity, so tag the
479622
// connection's list once per listing instead of on every read.
480623
const serverTools = (name: ServerName, entry: ServerEntry) => {
481-
const source = entry.connection.tools
624+
const source = entry.connection.tools ?? entry.retained
482625
if (!source) return []
483626
if (entry.tagged?.source !== source)
484627
entry.tagged = {
@@ -516,18 +659,21 @@ export const layer = (options?: Options) =>
516659
),
517660
})
518661

519-
const disposeServer = Effect.fnUntraced(function* (entry: ServerEntry) {
662+
const disposeServer = Effect.fnUntraced(function* (name: ServerName, entry: ServerEntry) {
663+
entry.disconnected = undefined
520664
yield* entry.connection.stop
665+
yield* dropRetained(name, entry)
521666
if (entry.registration) yield* entry.registration.dispose
522667
})
523668

524669
const replaceServer = Effect.fnUntraced(function* (name: ServerName, serverConfig: Mcp.ServerConfig) {
525670
const previous = entries.get(name)
526-
if (previous) yield* disposeServer(previous)
671+
if (previous) yield* disposeServer(name, previous)
527672
const entry = makeEntry(name, serverConfig, yield* Clock.currentTimeMillis)
528673
entries.set(name, entry)
529674
yield* Effect.gen(function* () {
530675
yield* register(name, entry)
676+
yield* loadTools(name, entry)
531677
if (serverConfig.disabled) {
532678
entry.status = { status: "disabled" }
533679
yield* bus.publish(McpEvent.StatusChanged, { server: name })
@@ -541,7 +687,7 @@ export const layer = (options?: Options) =>
541687
})
542688

543689
const removeServer = Effect.fnUntraced(function* (name: ServerName, entry: ServerEntry) {
544-
yield* disposeServer(entry)
690+
yield* disposeServer(name, entry)
545691
// Credentials are keyed by name + URL and intentionally survive removal for a later re-add.
546692
entries.delete(name)
547693
yield* bus.publish(McpEvent.StatusChanged, { server: name })
@@ -655,6 +801,11 @@ export const layer = (options?: Options) =>
655801
entries.set(name, makeEntry(name, server, started))
656802
}
657803
yield* Effect.forEach(entries, ([name, entry]) => register(name, entry), { discard: true })
804+
// Read before any connection starts, so the first step already offers the cached tools.
805+
yield* Effect.forEach(entries, ([name, entry]) => loadTools(name, entry), {
806+
concurrency: "unbounded",
807+
discard: true,
808+
})
658809

659810
// Initial connections stay asynchronous so one slow server does not block Location startup.
660811
for (const [name, entry] of entries) {
@@ -697,6 +848,7 @@ export const layer = (options?: Options) =>
697848
const entry = entries.get(name)
698849
if (!entry || entry.integrationID !== integrationID) return
699850
if (entry.status.status === "disabled") return
851+
entry.disconnected = undefined
700852
yield* entry.connection.stop
701853
yield* entry.connection.start
702854
}).pipe(locks.withLock(name))
@@ -759,6 +911,7 @@ export const layer = (options?: Options) =>
759911
yield* transition(
760912
Effect.gen(function* () {
761913
const entry = yield* requireServer(name)
914+
entry.disconnected = undefined
762915
yield* entry.connection.stop
763916
yield* entry.connection.start
764917
}).pipe(locks.withLock(name)),
@@ -769,9 +922,11 @@ export const layer = (options?: Options) =>
769922
yield* transition(
770923
Effect.gen(function* () {
771924
const entry = yield* requireServer(name)
925+
entry.disconnected = undefined
772926
yield* entry.connection.stop
773927
entry.status = { status: "disabled" }
774928
yield* bus.publish(McpEvent.StatusChanged, { server: name })
929+
yield* dropRetained(name, entry)
775930
}).pipe(locks.withLock(name)),
776931
)
777932
}),
@@ -800,7 +955,8 @@ export const layer = (options?: Options) =>
800955
const now = yield* Clock.currentTimeMillis
801956
yield* Effect.all(
802957
Array.from(entries.values()).flatMap((entry) => {
803-
if (entry.status.status !== "pending" || !entry.attempt) return []
958+
// A server offering retained or cached tools has something to show; the step need not wait for it.
959+
if (entry.status.status !== "pending" || !entry.attempt || entry.retained) return []
804960
const remaining = entry.attempt.started + STARTUP_GRACE_MS - now
805961
return remaining > 0 ? [entry.attempt.settled.await.pipe(Effect.raceFirst(Effect.sleep(remaining)))] : []
806962
}),
@@ -929,6 +1085,9 @@ export function configured(options?: Options) {
9291085
Form.node,
9301086
Integration.node,
9311087
Credential.node,
1088+
Config.node,
1089+
Global.node,
1090+
FSUtil.node,
9321091
httpFetch,
9331092
McpPool.node,
9341093
],

0 commit comments

Comments
 (0)