|
| 1 | +import { afterEach, describe, expect, test } from "bun:test"; |
| 2 | +import { createKiroAdapter } from "../../../src/adapters/kiro"; |
| 3 | +import { KIRO_COMPLETION_RETRY_MESSAGE, KIRO_COMPLETION_TOOL_NAME } from "../../../src/adapters/kiro-constants"; |
| 4 | +import { resetKiroThrottleStateForTests } from "../../../src/adapters/kiro-retry"; |
| 5 | +import { encodeMessage } from "../../../src/lib/eventstream-decoder"; |
| 6 | +import { createTranslatorBudget } from "../../../src/lib/translator-budget"; |
| 7 | +import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; |
| 8 | + |
| 9 | +// claude-sonnet-5 answers a trivial request in plain text with END_TURN instead of calling the |
| 10 | +// private completion tool, so the bounded completion retry fires even though the held text WAS the |
| 11 | +// answer. The retry used to open with "Continue the existing task"; the model read that as the |
| 12 | +// user's next message and called the completion tool with "There's no active task in this |
| 13 | +// conversation for me to continue", superseding the correct "OK". The frames below are the shape |
| 14 | +// captured from the live upstream for that turn. |
| 15 | + |
| 16 | +const provider: OcxProviderConfig = { |
| 17 | + adapter: "kiro", baseUrl: "https://runtime.us-east-1.kiro.dev", apiKey: "ksk_test", |
| 18 | +}; |
| 19 | +// A Claude Code Messages turn reduced to what matters here: tools (so completion is `required`), |
| 20 | +// an effort without a native Kiro field for this model, and a one-word answer. |
| 21 | +const parsed: OcxParsedRequest = { |
| 22 | + modelId: "claude-sonnet-5", stream: true, options: { reasoning: "high" }, |
| 23 | + context: { |
| 24 | + messages: [{ role: "user", content: "Reply with exactly: OK" }], |
| 25 | + tools: [ |
| 26 | + { name: "Read", description: "Read a file", parameters: { type: "object", properties: { file_path: { type: "string" } } } }, |
| 27 | + { name: "Bash", description: "Run a command", parameters: { type: "object", properties: { command: { type: "string" } } } }, |
| 28 | + ], |
| 29 | + }, |
| 30 | +}; |
| 31 | +const frame = (type: string, payload: object) => encodeMessage( |
| 32 | + { ":message-type": "event", ":event-type": type }, |
| 33 | + new TextEncoder().encode(JSON.stringify(payload)), |
| 34 | +); |
| 35 | +const tail = [ |
| 36 | + frame("contextUsageEvent", { contextUsagePercentage: 0.33 }), |
| 37 | + frame("meteringEvent", { unit: "credit", unitPlural: "credits", usage: 0.0077 }), |
| 38 | +]; |
| 39 | +// First attempt: the answer as ordinary text, then a native END_TURN. |
| 40 | +const plainTextAnswer = [ |
| 41 | + frame("initial-response", { conversationId: "" }), |
| 42 | + frame("assistantResponseEvent", { content: "O" }), |
| 43 | + frame("assistantResponseEvent", { content: "K" }), |
| 44 | + frame("metadataEvent", { stopReason: "END_TURN" }), |
| 45 | + ...tail, |
| 46 | +]; |
| 47 | +// Retry: the same answer resubmitted through the completion tool, streamed in pieces. |
| 48 | +const completionAnswer = [ |
| 49 | + frame("initial-response", { conversationId: "" }), |
| 50 | + frame("toolUseEvent", { name: KIRO_COMPLETION_TOOL_NAME, toolUseId: "toolu-1" }), |
| 51 | + frame("toolUseEvent", { name: KIRO_COMPLETION_TOOL_NAME, toolUseId: "toolu-1", input: "{\"answer\":" }), |
| 52 | + frame("toolUseEvent", { name: KIRO_COMPLETION_TOOL_NAME, toolUseId: "toolu-1", input: " \"OK\"}" }), |
| 53 | + frame("toolUseEvent", { name: KIRO_COMPLETION_TOOL_NAME, toolUseId: "toolu-1", stop: true }), |
| 54 | + frame("metadataEvent", { stopReason: "TOOL_USE" }), |
| 55 | + ...tail, |
| 56 | +]; |
| 57 | +function response(frames: Uint8Array[]): Response { |
| 58 | + return new Response(new ReadableStream<Uint8Array>({ |
| 59 | + start(controller) { |
| 60 | + for (const value of frames) controller.enqueue(value); |
| 61 | + controller.close(); |
| 62 | + }, |
| 63 | + })); |
| 64 | +} |
| 65 | +afterEach(resetKiroThrottleStateForTests); |
| 66 | + |
| 67 | +async function run(first: Uint8Array[], retry: Uint8Array[]) { |
| 68 | + const adapter = createKiroAdapter(provider); |
| 69 | + const budget = createTranslatorBudget(); |
| 70 | + const sends: number[] = []; |
| 71 | + const events: AdapterEvent[] = []; |
| 72 | + const bodies: string[] = []; |
| 73 | + try { |
| 74 | + const request = await adapter.buildRequest(structuredClone(parsed)); |
| 75 | + const upstream = await adapter.fetchResponse!(request, { |
| 76 | + executor: (async (_url, init) => { |
| 77 | + bodies.push(String(init?.body)); |
| 78 | + return response(bodies.length === 1 ? first : retry); |
| 79 | + }) as typeof fetch, |
| 80 | + onPhysicalSend: send => { sends.push(send.ordinal); }, |
| 81 | + }); |
| 82 | + for await (const event of adapter.parseStream(upstream, budget)) events.push(event); |
| 83 | + expect(budget.snapshot().currentBytes).toBe(0); |
| 84 | + const retryState = bodies[1] ? JSON.parse(bodies[1]).conversationState : undefined; |
| 85 | + return { events, sends, bodies, retryState }; |
| 86 | + } finally { |
| 87 | + budget.dispose(); |
| 88 | + } |
| 89 | +} |
| 90 | + |
| 91 | +describe("Kiro completion retry after a plain-text answer (claude-sonnet-5)", () => { |
| 92 | + test("the retry states the turn's state instead of presuming an open task", () => { |
| 93 | + // The phrase the model answered with "no active task ... to continue". |
| 94 | + expect(KIRO_COMPLETION_RETRY_MESSAGE).not.toContain("Continue the existing task"); |
| 95 | + expect(KIRO_COMPLETION_RETRY_MESSAGE).toContain("no final answer has reached the user yet"); |
| 96 | + expect(KIRO_COMPLETION_RETRY_MESSAGE) |
| 97 | + .toContain(`If your previous reply already was the complete final answer, call ${KIRO_COMPLETION_TOOL_NAME} now with that same answer, unchanged.`); |
| 98 | + // The progress-prose path the retry exists for (#531) keeps its instruction. |
| 99 | + expect(KIRO_COMPLETION_RETRY_MESSAGE).toContain("next real tool call now"); |
| 100 | + }); |
| 101 | + |
| 102 | + test("a plain-text END_TURN answer is replayed and resubmitted through the completion tool", async () => { |
| 103 | + const { events, sends, bodies, retryState } = await run(plainTextAnswer, completionAnswer); |
| 104 | + expect(bodies).toHaveLength(2); |
| 105 | + expect(sends).toEqual([1, 2]); |
| 106 | + // The held answer is replayed as the model's own previous reply, and the retry instruction is |
| 107 | + // the whole following user turn, so "your previous reply" points at exactly that text. |
| 108 | + expect(retryState.history.at(-1)).toEqual({ assistantResponseMessage: { content: "OK" } }); |
| 109 | + expect(retryState.currentMessage.userInputMessage.content).toBe(KIRO_COMPLETION_RETRY_MESSAGE); |
| 110 | + expect(retryState.currentMessage.userInputMessage.userInputMessageContext.tools |
| 111 | + .some((tool: { toolSpecification: { name: string } }) => tool.toolSpecification.name === KIRO_COMPLETION_TOOL_NAME)) |
| 112 | + .toBe(true); |
| 113 | + // One final answer: the resubmitted completion supersedes the held plain-text copy. |
| 114 | + expect(events.filter(event => event.type === "text_delta")).toEqual([ |
| 115 | + { type: "text_delta", text: "OK", phase: "final_answer" }, |
| 116 | + ]); |
| 117 | + expect(events.at(-1)).toMatchObject({ type: "done", endTurn: true }); |
| 118 | + }); |
| 119 | + |
| 120 | + test("a retry without replayable text appends the same instruction to the original request", async () => { |
| 121 | + // Reasoning only and no stop reason: nothing to replay, so there is no previous reply and the |
| 122 | + // conditional sentence is vacuous rather than false. |
| 123 | + const reasoningOnly = [ |
| 124 | + frame("initial-response", { conversationId: "" }), |
| 125 | + frame("reasoningContentEvent", { text: "The user wants the word OK." }), |
| 126 | + ...tail, |
| 127 | + ]; |
| 128 | + const { retryState, events } = await run(reasoningOnly, completionAnswer); |
| 129 | + expect(retryState.history ?? []).toEqual([]); |
| 130 | + const content: string = retryState.currentMessage.userInputMessage.content; |
| 131 | + expect(content).toContain("Reply with exactly: OK"); |
| 132 | + expect(content.endsWith(KIRO_COMPLETION_RETRY_MESSAGE)).toBe(true); |
| 133 | + expect(events.filter(event => event.type === "text_delta")).toEqual([ |
| 134 | + { type: "text_delta", text: "OK", phase: "final_answer" }, |
| 135 | + ]); |
| 136 | + }); |
| 137 | +}); |
0 commit comments