Skip to content

Commit 7b776f3

Browse files
committed
fix(opencode): run one session loop at a time across processes
Two opencode processes on one project could each run the loop for the same session, because SessionRunState keeps its runners in a per-process map. Each loop's reload saw the other's unfinished assistant message, the exit check failed, and they took turns answering one user message. A per-session EffectFlock lock now wraps the work of ensureRunning and startShell. A process that can't take it marks the session busy and retries with an interruptible jittered sleep, warning with the holder's pid after 60 s. The holder refreshes the lock's heartbeat every 20 s. runLoop checks it still owns the lock before every iteration; if another process took it over (the holder stalled past the 60 s staleness window), the loop fails with SessionRunState.LeaseLostError instead of returning the last assistant message, so callers such as the task tool report an error rather than an empty completion. A stalled holder's in-flight turn may still finish, so this is best-effort, not fencing. Breaking a stale lock logs the previous holder's pid. EffectFlock gains tryAcquire (one attempt, release registered atomically), Held.verify and holder. It also fixes the heartbeat of acquire: the repeated utimes call was built once, so every tick re-applied the acquisition time and a lock held past 60 s looked abandoned. opencode run: a waiting run whose prompt the other process answered used to print nothing, because /event only carries the local process's events. After the prompt returns, run now renders the reply's messages through the same part renderer as the event loop, skipping parts it already printed, so text and JSON output match a local run. It also prints one error record per rejected prompt instead of reporting the same failure from both the session event and the HTTP rejection. Fixes anomalyco#49367
1 parent 7cb044e commit 7b776f3

12 files changed

Lines changed: 1269 additions & 118 deletions

File tree

‎packages/core/src/util/effect-flock.ts‎

Lines changed: 93 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,17 @@ export namespace EffectFlock {
7474

7575
export interface Interface {
7676
readonly acquire: (key: string, dir?: string) => Effect.Effect<void, LockError, Scope.Scope>
77+
readonly tryAcquire: (key: string, dir?: string) => Effect.Effect<Option.Option<Held>, never, Scope.Scope>
78+
readonly holder: (key: string, dir?: string) => Effect.Effect<Option.Option<Holder>>
7779
readonly withLock: {
7880
(key: string, dir?: string): <A, E, R>(body: Effect.Effect<A, E, R>) => Effect.Effect<A, E | LockError, R>
7981
<A, E, R>(body: Effect.Effect<A, E, R>, key: string, dir?: string): Effect.Effect<A, E | LockError, R>
8082
}
8183
}
8284

85+
export type Holder = { readonly pid: number; readonly hostname: string; readonly createdAt: string }
86+
export type Held = { readonly verify: Effect.Effect<boolean> }
87+
8388
export class Service extends Context.Service<Service, Interface>()("EffectFlock") {}
8489

8590
// ---------------------------------------------------------------------------
@@ -196,8 +201,19 @@ export namespace EffectFlock {
196201
// We own the breaker — double-check staleness, nuke, recreate
197202
const recreated = yield* Effect.gen(function* () {
198203
if (!(yield* isStale(lockDir, heartbeatPath, metaPath))) return false
204+
const previous = yield* fs.readFileString(metaPath).pipe(
205+
Effect.map((raw) => Schema.decodeUnknownOption(LockMetaJson)(raw)),
206+
Effect.orElseSucceed(() => Option.none()),
207+
)
199208
yield* forceRemove(lockDir)
200-
return yield* atomicMkdir(lockDir)
209+
const recreated = yield* atomicMkdir(lockDir)
210+
if (recreated) {
211+
yield* Effect.logWarning("EffectFlock stale lock broken", {
212+
lockDir,
213+
pid: Option.isSome(previous) ? previous.value.pid : "unknown",
214+
})
215+
}
216+
return recreated
201217
}).pipe(Effect.ensuring(forceRemove(breakerPath)))
202218

203219
if (!recreated) return yield* new NotAcquired()
@@ -248,21 +264,86 @@ export namespace EffectFlock {
248264
yield* forceRemove(handle.lockDir)
249265
})
250266

251-
// -- build service --
267+
const verify = (handle: Handle) =>
268+
fs.readFileString(handle.metaPath).pipe(
269+
Effect.map((raw) =>
270+
Option.match(Schema.decodeUnknownOption(LockMetaJson)(raw), {
271+
onNone: () => false,
272+
onSome: (meta) => meta.token === handle.token,
273+
}),
274+
),
275+
Effect.orElseSucceed(() => false),
276+
)
252277

253-
const acquire = Effect.fn("EffectFlock.acquire")(function* (key: string, dir?: string) {
278+
const releaseLease = (handle: Handle) =>
279+
Effect.uninterruptible(
280+
Effect.gen(function* () {
281+
const breakerPath = handle.lockDir + ".breaker"
282+
const claimed = yield* fs.makeDirectory(breakerPath, { mode: 0o700 }).pipe(
283+
Effect.as(true),
284+
Effect.catchIf(
285+
(e) => e.reason._tag === "AlreadyExists",
286+
() => Effect.succeed(false),
287+
),
288+
Effect.orDie,
289+
)
290+
if (!claimed) {
291+
yield* Effect.logWarning("EffectFlock lease lost before release; preserving current lock", {
292+
lockDir: handle.lockDir,
293+
})
294+
return
295+
}
296+
yield* Effect.gen(function* () {
297+
if (yield* verify(handle)) return yield* forceRemove(handle.lockDir)
298+
yield* Effect.logWarning("EffectFlock lease lost before release; preserving current lock", {
299+
lockDir: handle.lockDir,
300+
})
301+
}).pipe(Effect.ensuring(forceRemove(breakerPath)))
302+
}),
303+
)
304+
305+
const tryAcquire = Effect.fn("EffectFlock.tryAcquire")(function* (key: string, dir?: string) {
254306
const lockDir = dir ?? lockRoot
255307
yield* ensureDir(lockDir)
308+
// acquireRelease runs the single attempt uninterruptibly and registers the release in
309+
// the same step, so an interrupt can't leave an acquired lock without its release
310+
const acquired = yield* Effect.acquireRelease(
311+
tryAcquireLockDir(path.join(lockDir, Hash.fast(key) + ".lock"), key).pipe(Effect.option),
312+
(result) => (Option.isSome(result) ? releaseLease(result.value) : Effect.void),
313+
)
314+
if (Option.isNone(acquired)) return Option.none()
315+
yield* Effect.suspend(() => fs.utimes(acquired.value.heartbeatPath, new Date(), new Date())).pipe(
316+
Effect.ignore,
317+
Effect.repeat(Schedule.spaced(HEARTBEAT_MS)),
318+
Effect.forkScoped,
319+
)
320+
return Option.some({ verify: verify(acquired.value) })
321+
})
256322

257-
const lockfile = path.join(lockDir, Hash.fast(key) + ".lock")
258-
259-
// acquireRelease: acquire is uninterruptible, release is guaranteed
260-
const handle = yield* Effect.acquireRelease(acquireHandle(lockfile, key), (handle) => release(handle))
323+
const holder = Effect.fn("EffectFlock.holder")(function* (key: string, dir?: string) {
324+
const metaPath = path.join(dir ?? lockRoot, Hash.fast(key) + ".lock", "meta.json")
325+
return yield* fs.readFileString(metaPath).pipe(
326+
Effect.map((raw) =>
327+
Schema.decodeUnknownOption(LockMetaJson)(raw).pipe(
328+
Option.map((meta) => ({ pid: meta.pid, hostname: meta.hostname, createdAt: meta.createdAt })),
329+
),
330+
),
331+
Effect.orElseSucceed(() => Option.none<Holder>()),
332+
)
333+
})
261334

262-
// Heartbeat fiber — scoped, so it's interrupted before release runs
263-
yield* fs
264-
.utimes(handle.heartbeatPath, new Date(), new Date())
265-
.pipe(Effect.ignore, Effect.repeat(Schedule.spaced(HEARTBEAT_MS)), Effect.forkScoped)
335+
const acquire = Effect.fn("EffectFlock.acquire")(function* (key: string, dir?: string) {
336+
const lockDir = dir ?? lockRoot
337+
yield* ensureDir(lockDir)
338+
const handle = yield* Effect.acquireRelease(
339+
acquireHandle(path.join(lockDir, Hash.fast(key) + ".lock"), key),
340+
(handle) => release(handle),
341+
)
342+
yield* Effect.suspend(() => fs.utimes(handle.heartbeatPath, new Date(), new Date())).pipe(
343+
Effect.ignore,
344+
Effect.repeat(Schedule.spaced(HEARTBEAT_MS)),
345+
Effect.forkScoped,
346+
)
266347
})
267348

268349
const withLock: Interface["withLock"] = Function.dual(
@@ -276,7 +357,7 @@ export namespace EffectFlock {
276357
),
277358
)
278359

279-
return Service.of({ acquire, withLock })
360+
return Service.of({ acquire, tryAcquire, holder, withLock })
280361
}),
281362
)
282363

‎packages/core/test/util/effect-flock.test.ts‎

Lines changed: 146 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { spawn } from "child_process"
33
import fs from "fs/promises"
44
import path from "path"
55
import os from "os"
6-
import { Cause, Effect, Exit } from "effect"
6+
import { Cause, Deferred, Effect, Exit, Fiber } from "effect"
77
import { testEffect } from "../lib/effect"
88
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
99
import { LayerNode } from "@opencode-ai/core/effect/layer-node"
@@ -119,6 +119,151 @@ const testLayer = AppNodeBuilder.build(EffectFlock.node, [[Global.node, testGlob
119119
describe("util.effect-flock", () => {
120120
const it = testEffect(testLayer)
121121

122+
it.live(
123+
"refreshes held lease and regular lock heartbeats across their interval",
124+
Effect.gen(function* () {
125+
const flock = yield* EffectFlock.Service
126+
const tmp = yield* Effect.promise(() => fs.mkdtemp(path.join(os.tmpdir(), "eflock-heartbeat-")))
127+
yield* Effect.addFinalizer(() => Effect.promise(() => fs.rm(tmp, { recursive: true, force: true })))
128+
const dir = path.join(tmp, "locks")
129+
yield* Effect.scoped(
130+
Effect.gen(function* () {
131+
const lease = yield* flock.tryAcquire("lease", dir)
132+
expect(lease._tag).toBe("Some")
133+
yield* flock.acquire("regular", dir)
134+
const old = new Date(Date.now() - 120_000)
135+
yield* Effect.promise(async () => {
136+
for (const key of ["lease", "regular"]) {
137+
const target = lock(dir, key)
138+
await fs.utimes(path.join(target, "heartbeat"), old, old)
139+
await fs.utimes(path.join(target, "meta.json"), old, old)
140+
await fs.utimes(target, old, old)
141+
}
142+
})
143+
yield* Effect.sleep("21 seconds")
144+
for (const key of ["lease", "regular"]) {
145+
const mtime = yield* Effect.promise(() => fs.stat(path.join(lock(dir, key), "heartbeat")))
146+
expect(Date.now() - mtime.mtimeMs).toBeLessThan(5_000)
147+
expect((yield* flock.tryAcquire(key, dir))._tag).toBe("None")
148+
}
149+
}),
150+
)
151+
}),
152+
35_000,
153+
)
154+
155+
it.live(
156+
"tryAcquire exposes ownership and preserves a takeover",
157+
Effect.gen(function* () {
158+
const flock = yield* EffectFlock.Service
159+
const tmp = yield* Effect.promise(() => fs.mkdtemp(path.join(os.tmpdir(), "eflock-lease-")))
160+
const dir = path.join(tmp, "locks")
161+
const key = "eflock:lease"
162+
const lockDir = lock(dir, key)
163+
yield* Effect.scoped(
164+
Effect.gen(function* () {
165+
const first = yield* flock.tryAcquire(key, dir)
166+
expect(first._tag).toBe("Some")
167+
expect((yield* flock.tryAcquire(key, dir))._tag).toBe("None")
168+
const holder = yield* flock.holder(key, dir)
169+
expect(holder._tag).toBe("Some")
170+
if (holder._tag === "Some") expect(holder.value.pid).toBe(process.pid)
171+
if (first._tag === "None") return false
172+
expect(yield* first.value.verify).toBe(true)
173+
yield* Effect.promise(async () => {
174+
const old = new Date(Date.now() - 120_000)
175+
await fs.utimes(path.join(lockDir, "heartbeat"), old, old)
176+
await fs.utimes(path.join(lockDir, "meta.json"), old, old)
177+
})
178+
const second = yield* flock.tryAcquire(key, dir)
179+
expect(second._tag).toBe("Some")
180+
expect(yield* first.value.verify).toBe(false)
181+
if (second._tag === "None") return
182+
expect(yield* second.value.verify).toBe(true)
183+
}),
184+
)
185+
expect(yield* Effect.promise(() => exists(lockDir))).toBe(false)
186+
expect((yield* flock.holder(key, dir))._tag).toBe("None")
187+
yield* Effect.promise(() => fs.rm(tmp, { recursive: true, force: true }))
188+
}),
189+
)
190+
191+
it.live(
192+
"releasing a stale holder preserves the new owner's lock",
193+
Effect.gen(function* () {
194+
const flock = yield* EffectFlock.Service
195+
const tmp = yield* Effect.promise(() => fs.mkdtemp(path.join(os.tmpdir(), "eflock-takeover-")))
196+
yield* Effect.addFinalizer(() => Effect.promise(() => fs.rm(tmp, { recursive: true, force: true })))
197+
const dir = path.join(tmp, "locks")
198+
const key = "eflock:takeover-release"
199+
const lockDir = lock(dir, key)
200+
const ready = yield* Deferred.make<EffectFlock.Held>()
201+
const finish = yield* Deferred.make<void>()
202+
const owner = yield* Effect.gen(function* () {
203+
yield* Effect.scoped(
204+
Effect.gen(function* () {
205+
const result = yield* flock.tryAcquire(key, dir)
206+
if (result._tag === "None") return
207+
yield* Deferred.succeed(ready, result.value)
208+
yield* Deferred.await(finish)
209+
}),
210+
)
211+
}).pipe(Effect.forkScoped)
212+
const original = yield* Deferred.await(ready)
213+
yield* Effect.sleep("100 millis")
214+
yield* Effect.promise(async () => {
215+
const old = new Date(Date.now() - 120_000)
216+
await fs.utimes(path.join(lockDir, "heartbeat"), old, old)
217+
await fs.utimes(path.join(lockDir, "meta.json"), old, old)
218+
await fs.utimes(lockDir, old, old)
219+
})
220+
const current = yield* flock.tryAcquire(key, dir)
221+
expect(current._tag).toBe("Some")
222+
expect(yield* original.verify).toBe(false)
223+
if (current._tag === "None") return
224+
yield* Deferred.succeed(finish, undefined)
225+
yield* Fiber.join(owner)
226+
expect(yield* Effect.promise(() => exists(lockDir))).toBe(true)
227+
expect(yield* current.value.verify).toBe(true)
228+
}),
229+
)
230+
231+
it.live(
232+
"an interrupted retry loop leaves the released key available",
233+
Effect.gen(function* () {
234+
const flock = yield* EffectFlock.Service
235+
const tmp = yield* Effect.promise(() => fs.mkdtemp(path.join(os.tmpdir(), "eflock-wait-")))
236+
const dir = path.join(tmp, "locks")
237+
const key = "eflock:wait-interrupt"
238+
const lockDir = lock(dir, key)
239+
yield* Effect.scoped(
240+
Effect.gen(function* () {
241+
const holder = yield* flock.tryAcquire(key, dir)
242+
expect(holder._tag).toBe("Some")
243+
let waiterAcquired = false
244+
const waiter = yield* Effect.gen(function* () {
245+
while (true) {
246+
const result = yield* flock.tryAcquire(key, dir)
247+
if (result._tag === "Some") {
248+
waiterAcquired = true
249+
return
250+
}
251+
yield* Effect.sleep("50 millis")
252+
}
253+
}).pipe(Effect.forkScoped)
254+
yield* Effect.sleep("10 millis")
255+
yield* Fiber.interrupt(waiter)
256+
expect(waiterAcquired).toBe(false)
257+
expect(yield* Effect.promise(() => exists(lockDir))).toBe(true)
258+
}),
259+
)
260+
expect(yield* Effect.promise(() => exists(lockDir))).toBe(false)
261+
const fresh = yield* Effect.scoped(flock.tryAcquire(key, dir))
262+
expect(fresh._tag).toBe("Some")
263+
yield* Effect.promise(() => fs.rm(tmp, { recursive: true, force: true }))
264+
}),
265+
)
266+
122267
it.live(
123268
"acquire and release via scoped Effect",
124269
Effect.gen(function* () {
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
export function createRunErrorDeduper() {
2+
let sessionErrorEmitted = false
3+
const requestErrors = new Set<string>()
4+
return (error: unknown, source: "session" | "request") => {
5+
const key = errorKey(error)
6+
if (source === "session") {
7+
if (key && requestErrors.has(key)) return true
8+
sessionErrorEmitted = true
9+
return false
10+
}
11+
if (key) requestErrors.add(key)
12+
return sessionErrorEmitted
13+
}
14+
}
15+
16+
function errorKey(error: unknown) {
17+
if (!error || typeof error !== "object" || !("name" in error) || !("data" in error)) return
18+
const data = error.data
19+
if (!data || typeof data !== "object" || !("message" in data)) return
20+
return JSON.stringify([String(error.name), String(data.message)])
21+
}

0 commit comments

Comments
 (0)