Skip to content

Commit 49af9d7

Browse files
frankeknfrankekn
committed
fix: abort generic no-progress tool loops
1 parent c9c9c3f commit 49af9d7

5 files changed

Lines changed: 53 additions & 34 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ Docs: https://docs.openclaw.ai
7070
- Gateway: dedupe concurrent `send`, `poll`, and `message.action` requests while delivery is still in flight, preventing duplicate outbound work for the same idempotency key. (#68341) Thanks @thesomewhatyou.
7171
- Cron: keep main-session `systemEvent` heartbeat wakes on their bound session route for both direct and queued wake paths by dropping inherited explicit heartbeat destinations when forcing `target: "last"`. Fixes #73900. Thanks @richardmqq.
7272
- Telegram: honor forced document delivery for video media so `--force-document` sends MP4s as documents instead of typed videos. Fixes #80389. (#80405) Thanks @jbetala7.
73+
- Agents: abort generic repeated no-progress tool loops at the critical threshold when identical calls keep returning identical outcomes. Thanks @frankekn.
7374
- Gateway: clear speculative node wake state when APNs registration is missing, preventing unregistered or mistyped node IDs from retaining wake throttle entries. Fixes #68847. (#68848) Thanks @Feelw00.
7475
- Auto-reply: keep late follow-up queue drain finalizers from deleting a replacement queue registered after `/stop`, preventing immediate follow-up messages from being orphaned. Fixes #68838. (#68839) Thanks @Feelw00.
7576
- Feishu: make manual App ID/App Secret setup the default channel-binding path while keeping QR scan-to-create as an optional best-effort flow, and document the manual fallback for domestic Feishu mobile clients that do not react to the QR code. Fixes #80591. Thanks @wei-wei-zhao.

‎docs/tools/loop-detection.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,10 @@ Per-agent override (optional):
7777
| `enabled` | `false` | Master switch for the rolling-history detectors. Setting `false` also disables the post-compaction guard. |
7878
| `historySize` | `30` | Number of recent tool calls kept for analysis. |
7979
| `warningThreshold` | `10` | Threshold before a pattern is classified as warning-only. |
80-
| `criticalThreshold` | `20` | Threshold for blocking repetitive loop patterns. |
80+
| `criticalThreshold` | `20` | Threshold for blocking repetitive no-progress loop patterns. |
8181
| `unknownToolThreshold` | `10` | Block repeated calls to the same unavailable tool after this many misses. |
8282
| `globalCircuitBreakerThreshold` | `30` | Global no-progress breaker threshold across all detectors. |
83-
| `detectors.genericRepeat` | `true` | Detects repeated same-tool + same-params patterns. |
83+
| `detectors.genericRepeat` | `true` | Warns on repeated same-tool + same-params patterns and blocks when the same calls also return identical outcomes. |
8484
| `detectors.knownPollNoProgress` | `true` | Detects known polling-like patterns with no state change. |
8585
| `detectors.pingPong` | `true` | Detects alternating ping-pong patterns. |
8686
| `postCompactionGuard.windowSize` | `3` | Number of post-compaction tool calls during which the guard stays armed and the count of identical triples that aborts the run. |

‎src/agents/pi-tools.before-tool-call.e2e.test.ts‎

Lines changed: 20 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import {
1515
runBeforeToolCallHook,
1616
wrapToolWithBeforeToolCallHook,
1717
} from "./pi-tools.before-tool-call.js";
18-
import { CRITICAL_THRESHOLD, GLOBAL_CIRCUIT_BREAKER_THRESHOLD } from "./tool-loop-detection.js";
18+
import { CRITICAL_THRESHOLD } from "./tool-loop-detection.js";
1919
import type { AnyAgentTool } from "./tools/common.js";
2020
import { callGatewayTool } from "./tools/gateway.js";
2121

@@ -277,28 +277,23 @@ describe("before_tool_call loop detection behavior", () => {
277277
}
278278
});
279279

280-
it("keeps generic repeated calls warn-only below global breaker", async () => {
280+
it("keeps generic repeated calls unblocked below critical threshold", async () => {
281281
const { tool, params } = createGenericReadRepeatFixture();
282282

283-
for (let i = 0; i < CRITICAL_THRESHOLD + 5; i += 1) {
283+
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
284284
await expectUnblockedToolExecution(tool, `read-${i}`, params);
285285
}
286286
});
287287

288-
it("blocks generic repeated no-progress calls at global breaker threshold", async () => {
288+
it("blocks generic repeated no-progress calls at critical threshold", async () => {
289289
const { tool, params } = createGenericReadRepeatFixture();
290290

291-
for (let i = 0; i < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; i += 1) {
291+
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
292292
await expectUnblockedToolExecution(tool, `read-${i}`, params);
293293
}
294294

295-
const result = await tool.execute(
296-
`read-${GLOBAL_CIRCUIT_BREAKER_THRESHOLD}`,
297-
params,
298-
undefined,
299-
undefined,
300-
);
301-
expectToolLoopBlockedResult(result, "global circuit breaker");
295+
const result = await tool.execute(`read-${CRITICAL_THRESHOLD}`, params, undefined, undefined);
296+
expectToolLoopBlockedResult(result, "identical outcomes");
302297
});
303298

304299
it("does not carry loop history across run ids", async () => {
@@ -316,27 +311,27 @@ describe("before_tool_call loop detection behavior", () => {
316311
runId: "heartbeat-2",
317312
});
318313

319-
for (let i = 0; i < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; i += 1) {
314+
for (let i = 0; i < CRITICAL_THRESHOLD; i += 1) {
320315
await expectUnblockedToolExecution(firstRunTool, `old-run-${i}`, params);
321316
}
322317

323318
await expectUnblockedToolExecution(secondRunTool, "new-run-0", params);
324319
});
325320

326-
it("coalesces repeated generic warning events into threshold buckets", async () => {
327-
await withToolLoopEvents(
328-
async (emitted) => {
329-
const { tool, params } = createGenericReadRepeatFixture();
321+
it("escalates generic repeat diagnostics from warning to critical", async () => {
322+
await withToolLoopEvents(async (emitted) => {
323+
const { tool, params } = createGenericReadRepeatFixture();
330324

331-
for (let i = 0; i < 21; i += 1) {
332-
await tool.execute(`read-bucket-${i}`, params, undefined, undefined);
333-
}
325+
for (let i = 0; i < 21; i += 1) {
326+
await tool.execute(`read-bucket-${i}`, params, undefined, undefined);
327+
}
334328

335-
const genericWarns = emitted.filter((evt) => evt.detector === "generic_repeat");
336-
expect(genericWarns.map((evt) => evt.count)).toEqual([10, 20]);
337-
},
338-
(evt) => evt.level === "warning",
339-
);
329+
const genericEvents = emitted.filter((evt) => evt.detector === "generic_repeat");
330+
expect(genericEvents.map((evt) => [evt.level, evt.count])).toEqual([
331+
["warning", 10],
332+
["critical", 20],
333+
]);
334+
});
340335
});
341336

342337
it("emits structured warning diagnostic events for ping-pong loops", async () => {

‎src/agents/tool-loop-detection.test.ts‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -388,7 +388,7 @@ describe("tool-loop-detection", () => {
388388
}
389389
});
390390

391-
it("keeps generic loops warn-only below global breaker threshold", () => {
391+
it("blocks generic no-progress loops at critical threshold", () => {
392392
const fixture = createReadNoProgressFixture();
393393
const loopResult = detectLoopAfterRepeatedCalls({
394394
toolName: fixture.toolName,
@@ -398,7 +398,9 @@ describe("tool-loop-detection", () => {
398398
});
399399
expect(loopResult.stuck).toBe(true);
400400
if (loopResult.stuck) {
401-
expect(loopResult.level).toBe("warning");
401+
expect(loopResult.level).toBe("critical");
402+
expect(loopResult.detector).toBe("generic_repeat");
403+
expect(loopResult.message).toContain("identical outcomes");
402404
}
403405
});
404406

@@ -524,6 +526,10 @@ describe("tool-loop-detection", () => {
524526
toolParams: fixture.params,
525527
result: fixture.result,
526528
count: GLOBAL_CIRCUIT_BREAKER_THRESHOLD,
529+
config: {
530+
enabled: true,
531+
detectors: { genericRepeat: false, knownPollNoProgress: true, pingPong: true },
532+
},
527533
});
528534
expect(loopResult.stuck).toBe(true);
529535
if (loopResult.stuck) {
@@ -537,7 +543,7 @@ describe("tool-loop-detection", () => {
537543
const state = createState();
538544
const params = { command: "grafana-api.sh datasources" };
539545

540-
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
546+
for (let index = 0; index < CRITICAL_THRESHOLD; index += 1) {
541547
recordSuccessfulCall(
542548
state,
543549
"exec",
@@ -560,15 +566,15 @@ describe("tool-loop-detection", () => {
560566
expect(loopResult.stuck).toBe(true);
561567
if (loopResult.stuck) {
562568
expect(loopResult.level).toBe("critical");
563-
expect(loopResult.detector).toBe("global_circuit_breaker");
569+
expect(loopResult.detector).toBe("generic_repeat");
564570
}
565571
});
566572

567573
it("blocks repeated running exec calls despite volatile session details and text", () => {
568574
const state = createState();
569575
const params = { command: "tail -f /var/log/app.log", yieldMs: 1000 };
570576

571-
for (let index = 0; index < GLOBAL_CIRCUIT_BREAKER_THRESHOLD; index += 1) {
577+
for (let index = 0; index < CRITICAL_THRESHOLD; index += 1) {
572578
recordSuccessfulCall(
573579
state,
574580
"exec",
@@ -597,7 +603,7 @@ describe("tool-loop-detection", () => {
597603
expect(loopResult.stuck).toBe(true);
598604
if (loopResult.stuck) {
599605
expect(loopResult.level).toBe("critical");
600-
expect(loopResult.detector).toBe("global_circuit_breaker");
606+
expect(loopResult.detector).toBe("generic_repeat");
601607
}
602608
});
603609

‎src/agents/tool-loop-detection.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -609,11 +609,28 @@ export function detectToolCallLoop(
609609
};
610610
}
611611

612-
// Generic detector: warn-only for repeated identical calls.
612+
// Generic detector: warn on repeated identical calls, then block only after
613+
// outcomes prove the calls are not making progress.
613614
const recentCount = history.filter(
614615
(h) => h.toolName === toolName && h.argsHash === currentHash,
615616
).length;
616617

618+
if (
619+
!knownPollTool &&
620+
resolvedConfig.detectors.genericRepeat &&
621+
noProgressStreak >= resolvedConfig.criticalThreshold
622+
) {
623+
log.error(`Critical generic loop detected: ${toolName} repeated ${noProgressStreak} times`);
624+
return {
625+
stuck: true,
626+
level: "critical",
627+
detector: "generic_repeat",
628+
count: noProgressStreak,
629+
message: `CRITICAL: Called ${toolName} with identical arguments and identical outcomes ${noProgressStreak} times. Session execution blocked to prevent runaway loops.`,
630+
warningKey: `generic:${toolName}:${currentHash}:${noProgress.latestResultHash ?? "none"}`,
631+
};
632+
}
633+
617634
if (
618635
!knownPollTool &&
619636
resolvedConfig.detectors.genericRepeat &&

0 commit comments

Comments
 (0)