test(ai): live-probe overflow classification and output clamping

steipete · steipete · commit 60405efce9bf · 2026-07-17T03:34:22.000-07:00
diff --git a/packages/ai/src/providers/anthropic.live.test.ts b/packages/ai/src/providers/anthropic.live.test.ts
@@ -1,6 +1,8 @@
 import { describe, expect, it } from "vitest";
 import type { Context, Model } from "../types.js";
-import { streamSimpleAnthropic } from "./anthropic.js";
+import { isContextOverflow } from "../utils/overflow.js";
+import { streamAnthropic, streamSimpleAnthropic } from "./anthropic.js";
+import { clampMaxTokensToContext, estimateContextInputTokens } from "./simple-options.js";
 
 const apiKey = process.env.ANTHROPIC_API_KEY?.trim() ?? "";
 const live = process.env.OPENCLAW_LIVE_TEST === "1" && apiKey.length > 0;
@@ -83,4 +85,66 @@ describeLive("Anthropic provider live", () => {
     },
     timeoutMs,
   );
+
+  it(
+    "classifies the provider's current context-overflow error",
+    async () => {
+      const result = await streamAnthropic(
+        model,
+        {
+          messages: [
+            {
+              role: "user",
+              content: "x ".repeat(Math.ceil(model.contextWindow * 1.1)),
+              timestamp: 0,
+            },
+          ],
+        },
+        { apiKey, maxTokens: 1, maxRetries: 0 },
+      ).result();
+
+      expect(result.stopReason).toBe("error");
+      expect(isContextOverflow(result)).toBe(true);
+    },
+    timeoutMs,
+  );
+
+  it(
+    "clamps an excessive output request to the remaining context",
+    async () => {
+      let sentMaxTokens: number | undefined;
+      const contextWithoutSystem: Context = {
+        messages: [{ role: "user", content: "Reply briefly with ok.", timestamp: 0 }],
+      };
+      const promptUnit = "x ";
+      const baseEstimate = estimateContextInputTokens(contextWithoutSystem);
+      const unitEstimate =
+        estimateContextInputTokens({ ...contextWithoutSystem, systemPrompt: promptUnit }) -
+        baseEstimate;
+      const targetEstimate = model.contextWindow - Math.floor(model.maxTokens / 2);
+      const context: Context = {
+        ...contextWithoutSystem,
+        systemPrompt: promptUnit.repeat(Math.ceil((targetEstimate - baseEstimate) / unitEstimate)),
+      };
+      const requestedMaxTokens = model.maxTokens * 100;
+      expect(estimateContextInputTokens(context)).toBeGreaterThanOrEqual(targetEstimate);
+      const expectedMaxTokens = clampMaxTokensToContext(model, context, requestedMaxTokens);
+      expect(expectedMaxTokens).toBeLessThan(model.maxTokens);
+
+      const result = await streamSimpleAnthropic(model, context, {
+        apiKey,
+        maxTokens: requestedMaxTokens,
+        maxRetries: 0,
+        reasoning: "off",
+        onPayload: (payload) => {
+          sentMaxTokens = (payload as { max_tokens?: number }).max_tokens;
+        },
+      }).result();
+
+      expect(sentMaxTokens).toBe(expectedMaxTokens);
+      expect(result.errorMessage).toBeUndefined();
+      expect(["stop", "length"]).toContain(result.stopReason);
+    },
+    timeoutMs,
+  );
 });
diff --git a/packages/ai/src/providers/openai-responses.live.test.ts b/packages/ai/src/providers/openai-responses.live.test.ts
@@ -1,6 +1,7 @@
 import { Type } from "typebox";
 import { describe, expect, it } from "vitest";
 import type { Context, Model } from "../types.js";
+import { isContextOverflow } from "../utils/overflow.js";
 import { streamOpenAIResponses } from "./openai-responses.js";
 
 // Live coverage for the Responses stream state machine: real streams interleave
@@ -12,6 +13,8 @@ const describeLive = LIVE && OPENAI_KEY ? describe : describe.skip;
 
 const LIVE_MODEL_ID = process.env.OPENCLAW_LIVE_RESPONSES_MODEL || "gpt-5.6-luna";
 const LIVE_TIMEOUT_MS = 120_000;
+const OVERFLOW_MODEL_ID = "gpt-4o-mini";
+const OVERFLOW_CONTEXT_WINDOW = 128_000;
 
 function liveModel(overrides: Partial<Model<"openai-responses">> = {}) {
   return {
@@ -113,4 +116,34 @@ describeLive("OpenAI Responses live", () => {
     },
     LIVE_TIMEOUT_MS,
   );
+
+  it(
+    "classifies the provider's current context-overflow error",
+    async () => {
+      const overflowModel = liveModel({
+        id: OVERFLOW_MODEL_ID,
+        name: OVERFLOW_MODEL_ID,
+        contextWindow: OVERFLOW_CONTEXT_WINDOW,
+        maxTokens: 16_384,
+        reasoning: false,
+      });
+      const result = await streamOpenAIResponses(
+        overflowModel,
+        {
+          messages: [
+            {
+              role: "user",
+              content: "x ".repeat(Math.ceil(overflowModel.contextWindow * 1.1)),
+              timestamp: 0,
+            },
+          ],
+        },
+        { apiKey: OPENAI_KEY, maxTokens: 16, maxRetries: 0 },
+      ).result();
+
+      expect(result.stopReason).toBe("error");
+      expect(isContextOverflow(result)).toBe(true);
+    },
+    LIVE_TIMEOUT_MS,
+  );
 });
