Skip to content

Commit cf2deba

Browse files
author
Lauris
committed
fix(codemode): fail a program that stalls instead of holding the turn
A Code Mode program has no timers, so a promise settles only through a tool or extension call or the program's own steps. A program awaiting a promise that nothing can settle, such as `await new Promise(r => r)`, waited forever, and with no execute timeout the session held its turn until inactivity eviction interrupted it an hour later. The interpreter now counts its steps and the host calls in flight, including their hooks. A new optional `stallMs` limit fails the program with a `Stalled` diagnostic once a full window passes with no step and no call running. OpenCode's execute tool sets it to 30 seconds and tells the model not to use execute to wait or poll. Closes anomalyco#54340
1 parent 7b3d4ce commit cf2deba

10 files changed

Lines changed: 232 additions & 14 deletions

File tree

‎packages/codemode/interpreter-support.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ ultimate source of truth. Upstream test262 files run verbatim from `test/test262
3333
- [x] The global `search(...)` built-in: synchronous tool discovery that counts as an admitted tool call and is
3434
shadowable by program declarations like other globals.
3535
- [x] Cooperative timeout, an optional total tool-call limit, output bounding, and unrestricted tool-call concurrency.
36+
- [x] Optional stall detection (`stallMs`): a program body that takes no step for the whole window while no tool or
37+
extension call is running fails with `Stalled`. Without timers, only those calls or the program's own steps can
38+
settle a promise, so such a program is awaiting one that never settles, as in `await new Promise(() => {})`.
3639
- [x] The timeout fires between interpreter steps, so one built-in is bounded in what it may build: strings up to
3740
2^24 characters (`repeat`, `pad*`, `concat`, `join`, `+`, template literals, `JSON.stringify`), arrays up to
3841
10,000,000 elements (`Array(n)`, `length =`, `Array.from`, `split`, `matchAll`, `concat`, `flat`; below the JS

‎packages/codemode/src/codemode.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ export type ExecutionLimits = {
2525
* No default: absent means no timeout.
2626
*/
2727
readonly timeoutMs?: number
28+
/**
29+
* Milliseconds the program may wait with no step taken and no tool or extension call running before it fails as
30+
* stalled. Such a program is awaiting a promise nothing can settle. No default: absent means no stall detection.
31+
*/
32+
readonly stallMs?: number
2833
/** Maximum number of tool calls admitted by the runtime. No default: absent means unlimited. */
2934
readonly maxToolCalls?: number
3035
/**
@@ -36,6 +41,7 @@ export type ExecutionLimits = {
3641

3742
export type ResolvedExecutionLimits = {
3843
readonly timeoutMs: number | undefined
44+
readonly stallMs: number | undefined
3945
readonly maxToolCalls: number | undefined
4046
readonly maxOutputBytes: number | undefined
4147
}
@@ -74,6 +80,7 @@ export const DiagnosticKind = Schema.Literals([
7480
"InvalidDataValue",
7581
"ToolCallLimitExceeded",
7682
"TimeoutExceeded",
83+
"Stalled",
7784
"ToolFailure",
7885
"ExecutionFailure",
7986
"Truncated",
@@ -132,6 +139,7 @@ const validateLimit = (name: keyof ExecutionLimits, value: number | undefined, m
132139

133140
const resolveExecutionLimits = (limits?: ExecutionLimits): ResolvedExecutionLimits => ({
134141
timeoutMs: validateLimit("timeoutMs", limits?.timeoutMs, 1),
142+
stallMs: validateLimit("stallMs", limits?.stallMs, 1),
135143
maxToolCalls: validateLimit("maxToolCalls", limits?.maxToolCalls, 0),
136144
maxOutputBytes: validateLimit("maxOutputBytes", limits?.maxOutputBytes, 0),
137145
})
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import { Effect } from "effect"
2+
3+
/**
4+
* Promises settle only through program steps or host calls, so a program that takes no step while no host call is
5+
* running is waiting on something that can never settle, such as `await new Promise(() => {})`.
6+
*/
7+
export class Activity {
8+
/** Counts program steps and host calls starting or ending; the hot path only increments it. */
9+
steps = 0
10+
private running = 0
11+
12+
track<A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> {
13+
return Effect.acquireUseRelease(
14+
Effect.sync(() => {
15+
this.steps++
16+
this.running++
17+
}),
18+
() => effect,
19+
// Ending counts too: a call can settle just before a check, ahead of the step that consumes its result.
20+
() =>
21+
Effect.sync(() => {
22+
this.steps++
23+
this.running--
24+
}),
25+
)
26+
}
27+
28+
/** Completes once a full `idleMs` window passes with no step and no host call running. */
29+
stalled(idleMs: number): Effect.Effect<void> {
30+
const self = this
31+
return Effect.gen(function* () {
32+
let steps = self.steps
33+
while (true) {
34+
yield* Effect.sleep(idleMs)
35+
if (self.running === 0 && self.steps === steps) return
36+
steps = self.steps
37+
}
38+
})
39+
}
40+
}

‎packages/codemode/src/interpreter/execute.ts‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import type { Value } from "./objects.js"
88
import { createBuiltins } from "./intrinsics.js"
99
import { Pending } from "./promises.js"
1010
import { Interpreter } from "./interpreter.js"
11+
import { Activity } from "./activity.js"
1112

1213
export const executeProgram = <R>(
1314
code: string,
@@ -29,6 +30,7 @@ export const executeProgram = <R>(
2930
const builtins = createBuiltins()
3031
const tools = ToolRuntime.make(prepared, limits.maxToolCalls, hooks)
3132
const logs: Array<string> = []
33+
const activity = new Activity()
3234
const logged = () => (logs.length > 0 ? { logs: [...logs] } : {})
3335
// Set only after copy-out so timeouts cannot report invalid values as completed.
3436
let returned: { value: DataValue; pending: Pending<R> } | undefined
@@ -39,7 +41,7 @@ export const executeProgram = <R>(
3941
Effect.gen(function* () {
4042
const program = parseProgram(code)
4143
const pending = new Pending<R>(scope, builtins.Promise)
42-
const ctx = new Interpreter<R>({ tools, pending, builtins, logs, globals })
44+
const ctx = new Interpreter<R>({ tools, pending, builtins, logs, activity, globals })
4345
const result = (yield* toBoundary(ctx, yield* ctx.run(program))) ?? null
4446
returned = { value: result, pending }
4547
const warnings = yield* pending.interrupt()
@@ -53,11 +55,35 @@ export const executeProgram = <R>(
5355
}),
5456
(scope, exit) => Scope.close(scope, exit),
5557
)
58+
const stallMs = limits.stallMs
59+
// Only the program body can stall; once it returns, settling background work is bounded by the timeout.
60+
// Cleanup runs through tracked calls, so the guard below is a backstop for cleanup that is slow while untracked.
61+
const watched =
62+
stallMs === undefined
63+
? base
64+
: Effect.raceFirst(
65+
base,
66+
activity.stalled(stallMs).pipe(
67+
Effect.flatMap(() => (returned === undefined ? Effect.void : Effect.never)),
68+
Effect.map(
69+
() =>
70+
({
71+
ok: false,
72+
error: {
73+
kind: "Stalled",
74+
message: `Execution stalled: nothing ran for ${stallMs}ms while the program awaited a promise that no running call can settle. Timers are unavailable, so a promise settles only through a tool or extension call, or by calling its resolve or reject.`,
75+
},
76+
...logged(),
77+
toolCalls: tools.calls,
78+
}) satisfies Result,
79+
),
80+
),
81+
)
5682
const timeoutMs = limits.timeoutMs
5783
const operation =
5884
timeoutMs === undefined
59-
? base
60-
: base.pipe(
85+
? watched
86+
: watched.pipe(
6187
Effect.timeoutOrElse({
6288
duration: timeoutMs,
6389
orElse: () =>

‎packages/codemode/src/interpreter/extensions.ts‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -140,14 +140,18 @@ export const extensionGlobals = <R>(
140140
fn<R>(builtins, value.name, value.length, (_, values) => {
141141
const args = values.map((item, index) => toHost(item, `Argument ${index + 1} to ${label}`))
142142
const hooks = ctx.tools.hooks
143+
// Track the hooks too: a slow `extension.before` is a call in flight, as `tool.before` is.
143144
const settle = (run: Effect.Effect<unknown, unknown, R>) =>
144-
(describe === undefined
145-
? run
146-
: hooked(describe(args), hooks["extension.before"], hooks["extension.after"], run)
147-
).pipe(
148-
Effect.mapError((reason) => new Throw(fromHost(reason, label))),
149-
Effect.map((settled) => fromHost(settled, label)),
150-
)
145+
ctx.activity
146+
.track(
147+
describe === undefined
148+
? run
149+
: hooked(describe(args), hooks["extension.before"], hooks["extension.after"], run),
150+
)
151+
.pipe(
152+
Effect.mapError((reason) => new Throw(fromHost(reason, label))),
153+
Effect.map((settled) => fromHost(settled, label)),
154+
)
151155
let result: unknown
152156
try {
153157
result = value.apply(undefined, args)

‎packages/codemode/src/interpreter/interpreter.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@ import { type Hint, preserveConsumerError, toPrimitive } from "./callback.js"
9999
import { Pending, resolvePromise, resolvePromiseValue } from "./promises.js"
100100
import { describeValue, isOpaque, rejectCircularInsertion, typeofValue } from "./references.js"
101101
import { ScopeStack } from "./scope.js"
102+
import { Activity } from "./activity.js"
102103
import { constructRegExp } from "../stdlib/regexp.js"
103104
import { enumerableSource } from "../stdlib/object.js"
104105
import { compoundOperators } from "../stdlib/value.js"
@@ -316,6 +317,7 @@ export class Interpreter<R> {
316317
readonly pending: Pending<R>
317318
readonly builtins: Builtins
318319
readonly logs: Array<string>
320+
readonly activity: Activity
319321
/** Template objects by site: a tag sees the same `strings` array every time its literal is evaluated, as in JS. */
320322
readonly templates = new WeakMap<TaggedTemplateExpression, Arr>()
321323
private readonly root: Frame<R>
@@ -325,12 +327,14 @@ export class Interpreter<R> {
325327
readonly pending: Pending<R>
326328
readonly builtins: Builtins
327329
readonly logs?: Array<string>
330+
readonly activity?: Activity
328331
readonly globals?: (ctx: Interpreter<R>) => ReadonlyArray<readonly [string, Value]>
329332
}) {
330333
this.tools = options.tools
331334
this.pending = options.pending
332335
this.builtins = options.builtins
333336
this.logs = options.logs ?? []
337+
this.activity = options.activity ?? new Activity()
334338
// Program code has no receiver: top-level `this` is undefined, as in a module.
335339
const globalScope = new Map<string, Binding>([["this", { mutable: false, value: undefined }]])
336340
// Calling back into the program never reads frame state, so any frame serves; the root is always alive.
@@ -368,7 +372,7 @@ export class Interpreter<R> {
368372
const ctx = this
369373
return Effect.gen(function* () {
370374
const json = yield* Effect.forEach(args, (arg) => toBoundary(ctx, arg))
371-
return fromJson(ctx, yield* run(json))
375+
return fromJson(ctx, yield* ctx.activity.track(run(json)))
372376
})
373377
}
374378
}
@@ -434,6 +438,7 @@ class Frame<R> {
434438
}
435439

436440
private evaluateStatement(node: Statement | ModuleDeclaration): Effect.Effect<StatementResult, unknown, R> {
441+
this.ctx.activity.steps++
437442
switch (node.type) {
438443
case "ExpressionStatement":
439444
return Effect.as(this.evaluateExpression(node.expression), { kind: "none" })
@@ -1289,6 +1294,7 @@ class Frame<R> {
12891294
}
12901295

12911296
private evaluateExpression(node: Expression): Effect.Effect<Value, unknown, R> {
1297+
this.ctx.activity.steps++
12921298
switch (node.type) {
12931299
case "Literal": {
12941300
const regex = node.regex

‎packages/codemode/test/codemode.test.ts‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1280,3 +1280,109 @@ describe("CodeMode public contract", () => {
12801280
expect(elapsedMs).toBeLessThan(3_000)
12811281
})
12821282
})
1283+
1284+
describe("CodeMode stall detection", () => {
1285+
const sleepy = (ms: number) =>
1286+
Tool.make({
1287+
description: "Resolve after a delay",
1288+
input: Schema.Struct({}),
1289+
output: Schema.String,
1290+
execute: () => Effect.as(Effect.sleep(ms), "done"),
1291+
})
1292+
const later = Extension.make({
1293+
name: "later",
1294+
globals: { later: () => new Promise((resolve) => setTimeout(() => resolve("late"), 150)) },
1295+
})
1296+
const stalls = (code: string, options: Omit<CodeMode.Options, "limits"> = {}) =>
1297+
Effect.runPromise(CodeMode.make({ ...options, limits: { stallMs: 50 } }).execute(code))
1298+
1299+
test("fails a program awaiting a promise nothing can settle", async () => {
1300+
for (const code of [
1301+
"await new Promise((resolve) => resolve); return 1",
1302+
"await new Promise(() => {}); return 1",
1303+
"const { promise } = Promise.withResolvers(); return await promise",
1304+
]) {
1305+
const startedAt = Date.now()
1306+
const result = await stalls(code)
1307+
expect(result).toMatchObject({ ok: false, error: { kind: "Stalled" } })
1308+
expect(Date.now() - startedAt).toBeLessThan(2_000)
1309+
}
1310+
})
1311+
1312+
test("keeps the calls and logs made before the stall", async () => {
1313+
const result = await stalls('console.log("waiting"); await tools.host.wait({}); await new Promise(() => {})', {
1314+
tools: { host: { wait: sleepy(1) } },
1315+
})
1316+
expect(result).toMatchObject({
1317+
ok: false,
1318+
error: { kind: "Stalled" },
1319+
logs: ["waiting"],
1320+
toolCalls: [{ name: "host.wait" }],
1321+
})
1322+
})
1323+
1324+
test("waits for tool and extension calls that outlast the stall window", async () => {
1325+
expect(await stalls("return await tools.host.wait({})", { tools: { host: { wait: sleepy(150) } } })).toMatchObject({
1326+
ok: true,
1327+
value: "done",
1328+
})
1329+
expect(await stalls("return await later()", { extensions: [later] })).toMatchObject({ ok: true, value: "late" })
1330+
})
1331+
1332+
test("counts slow before hooks as calls in flight", async () => {
1333+
const slowBefore = { "tool.before": () => Effect.sleep(150), "extension.before": () => Effect.sleep(150) }
1334+
expect(
1335+
await stalls("return await tools.host.wait({})", { tools: { host: { wait: sleepy(1) } }, hooks: slowBefore }),
1336+
).toMatchObject({ ok: true, value: "done" })
1337+
expect(await stalls("return await later()", { extensions: [later], hooks: slowBefore })).toMatchObject({
1338+
ok: true,
1339+
value: "late",
1340+
})
1341+
})
1342+
1343+
test("reports whichever of the stall and the timeout comes first", async () => {
1344+
const run = (code: string, limits: CodeMode.ExecutionLimits) =>
1345+
Effect.runPromise(CodeMode.make({ limits }).execute(code))
1346+
expect(await run("await new Promise(() => {})", { stallMs: 50, timeoutMs: 1_000 })).toMatchObject({
1347+
ok: false,
1348+
error: { kind: "Stalled" },
1349+
})
1350+
expect(await run("await new Promise(() => {})", { stallMs: 1_000, timeoutMs: 50 })).toMatchObject({
1351+
ok: false,
1352+
error: { kind: "TimeoutExceeded" },
1353+
})
1354+
})
1355+
1356+
test("reports slow cleanup after return as a timeout warning, not a stall", async () => {
1357+
// Interrupting the un-awaited call at return runs its `tool.after`, which outlasts both windows. The hook
1358+
// counts as a call in flight, so the stall watcher stays quiet and the timeout ends the cleanup.
1359+
const result = await Effect.runPromise(
1360+
CodeMode.make({
1361+
tools: { host: { wait: sleepy(10_000) } },
1362+
hooks: { "tool.after": () => Effect.sleep(400) },
1363+
limits: { stallMs: 50, timeoutMs: 200 },
1364+
}).execute("tools.host.wait({}); return 5"),
1365+
)
1366+
expect(result).toMatchObject({ ok: true, value: 5, warnings: [{ kind: "TimeoutExceeded" }] })
1367+
})
1368+
1369+
test("never stalls a program that keeps running", async () => {
1370+
const result = await stalls("let total = 0; for (let i = 0; i < 300000; i += 1) total += i; return total")
1371+
expect(result).toMatchObject({ ok: true, value: 44999850000 })
1372+
})
1373+
1374+
test("does not count an un-awaited promise left behind at return", async () => {
1375+
expect(await stalls("new Promise(() => {}); return 5")).toMatchObject({ ok: true, value: 5 })
1376+
})
1377+
1378+
test("is off unless the host sets stallMs", async () => {
1379+
const result = await Effect.runPromise(
1380+
CodeMode.make({ limits: { timeoutMs: 200 } }).execute("await new Promise(() => {})"),
1381+
)
1382+
expect(result).toMatchObject({ ok: false, error: { kind: "TimeoutExceeded" } })
1383+
})
1384+
1385+
test("rejects an invalid stallMs", () => {
1386+
expect(() => CodeMode.make({ limits: { stallMs: 0 } })).toThrow(RangeError)
1387+
})
1388+
})

‎packages/codemode/test/test262/run.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ type Frontmatter = {
2929
negative?: { phase: "parse" | "resolution" | "runtime"; type: string }
3030
}
3131

32-
const limits = { timeoutMs: 5000, maxToolCalls: undefined, maxOutputBytes: undefined }
32+
const limits = { timeoutMs: 5000, stallMs: undefined, maxToolCalls: undefined, maxOutputBytes: undefined }
3333
const prepared = ToolRuntime.prepare<never>({})
3434

3535
export const run = async (file: string): Promise<Outcome> => {

‎packages/core/src/codemode/tool.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@ export type Inventory = {
5858
readonly namespaces?: ReadonlyMap<string, ToolNamespace>
5959
}
6060

61+
// A program that takes no step while no call is running can never resume; fail it instead of holding the turn.
62+
const STALL_MS = 30_000
63+
6164
// Invariant model-facing guidance; the changing tool catalog is delivered through Instructions.
6265
const description = [
6366
"Run JavaScript in a confined Code Mode runtime to script tool calls and HTTP requests and compose their results.",
@@ -66,6 +69,7 @@ const description = [
6669
'Call tools through `tools` using only exact paths and signatures from the catalog. Do not infer or normalize tool names; preserve bracket notation such as `tools.<namespace>["tool-name"](input)`.',
6770
"Prefer an explicit `return`; if omitted, the final top-level expression becomes the result.",
6871
"Await every call whose completion matters; pending calls are interrupted when execution ends. Run independent calls concurrently with `Promise.allSettled`.",
72+
"Do not use it to wait or poll: without timers it cannot sleep, and a program awaiting a promise that nothing can settle fails as stalled.",
6973
].join("\n")
7074

7175
export const create = (
@@ -236,7 +240,12 @@ function runtime(
236240
})
237241
}
238242
const tools = renderTools(root)
239-
return CodeMode.make<typeof tools>({ tools, extensions: [CodeModeWeb.extension], hooks })
243+
return CodeMode.make<typeof tools>({
244+
tools,
245+
extensions: [CodeModeWeb.extension],
246+
hooks,
247+
limits: { stallMs: STALL_MS },
248+
})
240249
}
241250

242251
function getNode<T>(root: Node<T>, path: string) {

0 commit comments

Comments
 (0)