Skip to content

Commit 9a1eac6

Browse files
fix: detect stalled opencode sessions and restart the server
A wedged session executor (anomalyco/opencode#53747) left prompts hanging while /api/info stayed green; only a restart helped. - watchdog: a running session silent past watchdog.inactivityTimeoutMs is interrupted, the user gets a reply, and an opencode restart is requested - event stream: Ktor's silent SSE reconnect is disabled (it hid events lost while disconnected); reconnects are explicit, missed results are recovered from /api/session/active + /message, and connections without heartbeats are re-established without touching the session - supervision: OpenCodeServerManager restarts opencode on stall (with cooldown) and on unexpected exit (with backoff), waiting for it to be healthy - readiness reports DOWN while opencode is starting or restarting - every opencode transport failure maps to OpenCodeClientException and the listener always replies, so Slack thread locks are always released; lock TTLs derive from maxAgentSessionDurationSeconds Fixes #20 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 0edca39 commit 9a1eac6

18 files changed

Lines changed: 1150 additions & 168 deletions

‎src/main/kotlin/io/github/torvehammok/domain/OCServerAgent.kt‎

Lines changed: 205 additions & 122 deletions
Large diffs are not rendered by default.

‎src/main/kotlin/io/github/torvehammok/domain/OpenCodeProps.kt‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@ data class OpenCodeProps(
3434
""".trimIndent(),
3535
@param:JsonPropertyDescription("Maximum agent session duration in seconds")
3636
val maxAgentSessionDurationSeconds: Long = 180,
37+
@param:JsonPropertyDescription("Stalled session detection and opencode server supervision")
38+
val watchdog: OpenCodeWatchdogProps = OpenCodeWatchdogProps(),
3739
@param:JsonPropertyDescription("Configuration synchronization")
3840
val configSync: ConfigSyncProps = ConfigSyncProps()
39-
)
41+
)
42+
43+
/**
44+
* Thread lock TTL - must outlive the longest agent session, so a second message cannot start a parallel run.
45+
*/
46+
fun OpenCodeProps.lockTtlSeconds(): Long = maxAgentSessionDurationSeconds + 30
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package io.github.torvehammok.domain
2+
3+
import com.fasterxml.jackson.annotation.JsonPropertyDescription
4+
5+
data class OpenCodeWatchdogProps(
6+
@param:JsonPropertyDescription("Max time to wait for the event stream to connect, in milliseconds")
7+
val eventStreamConnectTimeoutMs: Long = 10_000,
8+
@param:JsonPropertyDescription("A running session without any event for this long is considered stalled, in milliseconds")
9+
val inactivityTimeoutMs: Long = 120_000,
10+
@param:JsonPropertyDescription("How often the watchdog checks session liveness, in milliseconds")
11+
val checkIntervalMs: Long = 5_000,
12+
@param:JsonPropertyDescription("Event stream without heartbeat for this long is considered dead and reconnected, in milliseconds")
13+
val heartbeatTimeoutMs: Long = 30_000,
14+
@param:JsonPropertyDescription("Max consecutive failed event stream reconnection attempts")
15+
val maxReconnectAttempts: Int = 5,
16+
@param:JsonPropertyDescription("Initial event stream reconnection backoff, doubled on each attempt, in milliseconds")
17+
val reconnectBackoffMs: Long = 500,
18+
@param:JsonPropertyDescription("Max event stream reconnection backoff, in milliseconds")
19+
val reconnectBackoffMaxMs: Long = 5_000,
20+
@param:JsonPropertyDescription("Min time between opencode server restarts caused by stalled sessions, in milliseconds")
21+
val restartCooldownMs: Long = 30_000,
22+
@param:JsonPropertyDescription("Max time for a (re)started opencode server to become healthy, in milliseconds")
23+
val startupTimeoutMs: Long = 30_000,
24+
@param:JsonPropertyDescription("Max backoff between consecutive opencode server restarts, in milliseconds")
25+
val restartBackoffMaxMs: Long = 60_000,
26+
@param:JsonPropertyDescription("Restart backoff resets once the opencode server runs this long, in milliseconds")
27+
val backoffResetAfterMs: Long = 300_000
28+
)
Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
package io.github.torvehammok.domain
2+
3+
import io.github.torvehammok.infra.opencode.OpenCodeClientException
4+
import io.github.torvehammok.infra.opencode.model.OCEvent
5+
import kotlinx.coroutines.CompletableDeferred
6+
import kotlinx.coroutines.Job
7+
import org.slf4j.LoggerFactory
8+
import org.slf4j.MDC
9+
import java.math.BigDecimal
10+
import java.math.MathContext
11+
import java.math.RoundingMode
12+
import java.util.concurrent.ConcurrentHashMap
13+
14+
private val log = LoggerFactory.getLogger(SessionTurn::class.java)
15+
16+
internal val COST_CONTEXT = MathContext(4, RoundingMode.HALF_UP)
17+
18+
internal sealed interface TurnOutcome {
19+
data class Succeeded(val text: String) : TurnOutcome
20+
data class Failed(val reason: String) : TurnOutcome
21+
data class Stalled(val silentForMs: Long) : TurnOutcome
22+
}
23+
24+
/**
25+
* State of a single prompt round-trip on an opencode session.
26+
*/
27+
internal class SessionTurn(val sessionId: String) {
28+
29+
var cost: BigDecimal = BigDecimal(0, COST_CONTEXT)
30+
var toolInvocations: Int = 0
31+
32+
val connected = CompletableDeferred<Unit>()
33+
val outcome = CompletableDeferred<TurnOutcome>()
34+
35+
@Volatile
36+
var promptSentAt: Long? = null
37+
38+
@Volatile
39+
var lastProgressAt: Long = System.currentTimeMillis()
40+
41+
@Volatile
42+
var lastHeartbeatAt: Long = System.currentTimeMillis()
43+
44+
@Volatile
45+
var heartbeatsSeen: Boolean = false
46+
47+
@Volatile
48+
var executionStarted: Boolean = false
49+
50+
@Volatile
51+
var lastEventType: String? = null
52+
53+
/** current event stream subscription - cancelled by the watchdog to force a reconnect */
54+
@Volatile
55+
var subscription: Job? = null
56+
57+
private var lastText: String? = null
58+
59+
// `session.tool.called` carries the input but not the name - it is only on `session.tool.input.started`
60+
private val toolNames = mutableMapOf<String, String>()
61+
62+
// tool calls started but not finished yet - logged when the session stalls
63+
private val inFlightTools = ConcurrentHashMap<String, String>()
64+
65+
fun inFlightTools(): Collection<String> = inFlightTools.values
66+
67+
fun onConnected() {
68+
val now = System.currentTimeMillis()
69+
lastHeartbeatAt = now
70+
// events published while disconnected are lost - give the session a fresh inactivity window
71+
lastProgressAt = now
72+
connected.complete(Unit)
73+
}
74+
75+
fun onHeartbeat() {
76+
heartbeatsSeen = true
77+
lastHeartbeatAt = System.currentTimeMillis()
78+
}
79+
80+
fun complete(result: TurnOutcome) {
81+
outcome.complete(result)
82+
}
83+
84+
fun fail(reason: String) {
85+
connected.completeExceptionally(OpenCodeClientException(reason))
86+
outcome.complete(TurnOutcome.Failed(reason))
87+
}
88+
89+
fun lastText(): String? = lastText
90+
91+
fun handle(event: OCEvent) {
92+
MDC.put("sessionId", sessionId)
93+
lastProgressAt = System.currentTimeMillis()
94+
lastEventType = event.type
95+
96+
val data = event.data
97+
98+
when (event.type) {
99+
"session.execution.started" -> executionStarted = true
100+
101+
"session.tool.input.started" -> {
102+
log.info("opencode> tool={}", data?.name)
103+
104+
val callId = data?.id
105+
val name = data?.name
106+
if (callId != null && name != null) {
107+
toolNames[callId] = name
108+
inFlightTools[callId] = name
109+
}
110+
}
111+
112+
"session.tool.called" -> {
113+
toolInvocations++
114+
115+
if (toolNames.remove(data?.id) == CODE_MODE_TOOL) {
116+
val code = data?.input?.get("code") as? String ?: ""
117+
val calls = extractCodeModeCalls(code)
118+
calls.forEach { log.info("opencode> tool={} (via {})", it, CODE_MODE_TOOL) }
119+
120+
if (calls.isNotEmpty()) {
121+
data?.id?.let { inFlightTools[it] = "$CODE_MODE_TOOL(${calls.joinToString()})" }
122+
}
123+
}
124+
}
125+
126+
"session.tool.success" -> data?.id?.let { inFlightTools.remove(it) }
127+
128+
"session.tool.failed" -> {
129+
data?.id?.let { inFlightTools.remove(it) }
130+
log.info("opencode> tool failed: {}", data?.error?.message)
131+
}
132+
133+
"session.text.ended" -> {
134+
log.info("opencode> text={}", data?.text)
135+
lastText = data?.text
136+
}
137+
138+
"session.usage.updated" -> {
139+
// running total of the whole session
140+
if (data?.cost != null) {
141+
cost = BigDecimal(data.cost, COST_CONTEXT)
142+
}
143+
}
144+
145+
"session.step.failed" -> log.warn("opencode> step failed: {}", data?.error?.message)
146+
147+
"session.execution.succeeded" -> {
148+
log.info("Session {} finished successfully", sessionId)
149+
complete(TurnOutcome.Succeeded(lastText ?: ""))
150+
}
151+
152+
"session.execution.failed" -> complete(TurnOutcome.Failed("Session execution failed: ${data?.error?.message}"))
153+
154+
"session.execution.interrupted" -> complete(TurnOutcome.Failed("Session execution interrupted: ${data?.reason}"))
155+
156+
else -> log.trace("opencode> {}", event)
157+
}
158+
}
159+
}

‎src/main/kotlin/io/github/torvehammok/infra/httpclient/KtorHttpClientFactory.kt‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import tools.jackson.databind.DeserializationFeature
1515
import tools.jackson.databind.SerializationFeature
1616
import tools.jackson.databind.cfg.DateTimeFeature
1717
import tools.jackson.module.kotlin.KotlinModule
18-
import kotlin.time.Duration.Companion.seconds
1918

2019
@Singleton
2120
class KtorHttpClientFactory(private val meterRegistry: MeterRegistry) {
@@ -28,8 +27,8 @@ class KtorHttpClientFactory(private val meterRegistry: MeterRegistry) {
2827
expectSuccess = false
2928

3029
install(SSE) {
31-
maxReconnectionAttempts = 5
32-
reconnectionTime = 3.seconds
30+
// reconnects are handled by callers - a silent reconnect would hide events lost while disconnected
31+
maxReconnectionAttempts = 0
3332
}
3433

3534
install(ContentNegotiation) {

‎src/main/kotlin/io/github/torvehammok/infra/opencode/OpenCodeClient.kt‎

Lines changed: 65 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import io.ktor.client.call.*
99
import io.ktor.client.plugins.sse.*
1010
import io.ktor.client.request.*
1111
import io.ktor.http.*
12+
import kotlinx.coroutines.CancellationException
1213
import kotlinx.coroutines.currentCoroutineContext
1314
import kotlinx.coroutines.ensureActive
1415
import kotlinx.coroutines.flow.cancellable
@@ -31,26 +32,34 @@ class OpenCodeClient(
3132

3233
/**
3334
* Subscribes to the global (volatile) event bus. [onConnected] is invoked once the server
34-
* acknowledges the subscription - events published before that moment are not delivered.
35+
* acknowledges the subscription - events published before that moment (or while disconnected) are not delivered.
36+
* [onHeartbeat] is invoked for server heartbeats (SSE comments), which prove the connection is alive.
37+
*
38+
* Returns when the server closes the stream - it does not mean the session has finished.
3539
*/
3640
suspend fun subscribeSessionEvents(
3741
sessionId: String,
3842
onConnected: () -> Unit = {},
43+
onHeartbeat: () -> Unit = {},
3944
onEvent: (event: OCEvent) -> Unit
40-
) {
45+
) = withClientErrors("subscribe to opencode events") {
4146
httpClient.sse(
4247
urlString = "/api/event",
4348
request = {
4449
pathPattern("/api/event")
4550
opencodeAuth(opencodeProps)
4651
header(HttpHeaders.Accept, ContentType.Text.EventStream)
4752
},
53+
showCommentEvents = true
4854
) {
4955
incoming.cancellable().collect { serverEvent ->
5056
currentCoroutineContext().ensureActive()
5157

5258
val data = serverEvent.data
5359
if (data.isNullOrBlank()) {
60+
if (serverEvent.comments != null) {
61+
onHeartbeat()
62+
}
5463
return@collect
5564
}
5665

@@ -65,7 +74,7 @@ class OpenCodeClient(
6574
}
6675
}
6776

68-
suspend fun createSession(agent: String, modelId: String): String {
77+
suspend fun createSession(agent: String, modelId: String): String = withClientErrors("open new session") {
6978
val response = httpClient.post {
7079
pathPattern("/api/session")
7180
url("/api/session")
@@ -84,10 +93,10 @@ class OpenCodeClient(
8493

8594
val body = response.body<OCData<OCSession>>()
8695

87-
return body.data.id
96+
body.data.id
8897
}
8998

90-
suspend fun prompt(sessionId: String, prompt: String) {
99+
suspend fun prompt(sessionId: String, prompt: String) = withClientErrors("prompt session") {
91100
val response = httpClient.post {
92101
url("/api/session/$sessionId/prompt")
93102
pathPattern("/api/session/{sessionId}/prompt")
@@ -101,7 +110,7 @@ class OpenCodeClient(
101110
}
102111
}
103112

104-
suspend fun interrupt(sessionId: String) {
113+
suspend fun interrupt(sessionId: String) = withClientErrors("interrupt session") {
105114
val response = httpClient.post {
106115
url("/api/session/$sessionId/interrupt")
107116
pathPattern("/api/session/{sessionId}/interrupt")
@@ -113,6 +122,40 @@ class OpenCodeClient(
113122
}
114123
}
115124

125+
/**
126+
* Ids of sessions with an execution currently running on the server - absent sessions are idle.
127+
*/
128+
suspend fun fetchActiveSessionIds(): Set<String> = withClientErrors("fetch active sessions") {
129+
val response = httpClient.get {
130+
url("/api/session/active")
131+
pathPattern("/api/session/active")
132+
opencodeAuth(opencodeProps)
133+
}
134+
135+
if (!response.status.isSuccess()) {
136+
throw OpenCodeClientException("Failed to fetch active sessions")
137+
}
138+
139+
response.body<OCData<Map<String, Any?>>>().data.keys
140+
}
141+
142+
/**
143+
* Session messages, newest first.
144+
*/
145+
suspend fun fetchMessages(sessionId: String): List<OCSessionMessage> = withClientErrors("fetch session messages") {
146+
val response = httpClient.get {
147+
url("/api/session/$sessionId/message")
148+
pathPattern("/api/session/{sessionId}/message")
149+
opencodeAuth(opencodeProps)
150+
}
151+
152+
if (!response.status.isSuccess()) {
153+
throw OpenCodeClientException("Failed to fetch session messages")
154+
}
155+
156+
response.body<OCData<List<OCSessionMessage>>>().data
157+
}
158+
116159
suspend fun fetchHealthcheckStatus(): HttpStatusCode {
117160
val response = httpClient.get {
118161
pathPattern("/api/info")
@@ -126,6 +169,22 @@ class OpenCodeClient(
126169

127170
}
128171

172+
/**
173+
* Transport failures (timeouts, refused connections, broken streams) surface as [OpenCodeClientException],
174+
* so callers handle every opencode failure in one place.
175+
*/
176+
private suspend fun <T> withClientErrors(action: String, block: suspend () -> T): T {
177+
try {
178+
return block()
179+
} catch (ex: OpenCodeClientException) {
180+
throw ex
181+
} catch (ex: CancellationException) {
182+
throw ex
183+
} catch (ex: Exception) {
184+
throw OpenCodeClientException("Failed to $action: ${ex.message}", ex)
185+
}
186+
}
187+
129188
private fun toModelRef(modelId: String): OCModelRef {
130189
val split = modelId.split("/", limit = 2)
131190
require(split.size == 2) { "Model id must be in format <provider>/<model>, got: $modelId" }

0 commit comments

Comments
 (0)