Skip to content

Commit d2f3dff

Browse files
committed
fix(devin): replay the assistant's reasoning instead of dropping it
The adapter asserted that Cognition has no reasoning-replay field and dropped every assistant thinking block, so a reasoning model restarted its chain on each turn of a tool loop and paid for it again. The field exists. Two independent clients of the same service write it on the assistant prompt: #11 thinking, #12 signature, #18 signature_type. The response side was half-missing too. delta_thinking (#9) was decoded but delta_signature (#10) was not, so even once the prompt could carry a signature there was never one to carry: the replay would always have been unsigned. #10 now surfaces as a thinking_signature event. Reasoning still stays out of the replayed assistant TEXT, which is what the original comment was right about — folding chain-of-thought into the visible content makes the model treat it as something it said to the user. It rides its own field instead. A turn that produced only reasoning is now replayed rather than skipped, since dropping it is what forces the next turn to re-derive the same chain.
1 parent 92e21bc commit d2f3dff

3 files changed

Lines changed: 157 additions & 6 deletions

File tree

‎src/adapters/devin.ts‎

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -160,15 +160,44 @@ function assistantToolCalls(message: OcxAssistantMessage): Array<{ id: string; n
160160

161161
function assistantText(message: OcxAssistantMessage): string {
162162
return message.content
163-
// Thinking stays out of the replayed content. Cognition has no reasoning
164-
// replay field, and folding chain-of-thought into assistant text sends it
165-
// back as visible prior output - which the model then treats as something
166-
// it said to the user.
163+
// Thinking stays out of the replayed TEXT: folding chain-of-thought into
164+
// assistant text sends it back as visible prior output, which the model
165+
// then treats as something it said to the user. It is replayed in its own
166+
// field instead — see assistantThinking below.
167167
.map((part) => (part.type === "text" ? part.text : ""))
168168
.filter(Boolean)
169169
.join("\n");
170170
}
171171

172+
/**
173+
* The assistant turn's own reasoning, for replay in ChatMessagePrompt #11.
174+
*
175+
* This adapter previously asserted that Cognition has no reasoning-replay
176+
* field and dropped the thinking outright, so a reasoning model restarted its
177+
* chain on every turn of a tool loop. The field exists: two independent
178+
* clients of the same service write #11 thinking with #12 signature and #18
179+
* signature_type on the assistant prompt.
180+
*
181+
* The signature attests the thinking it was produced with, so a block without
182+
* one contributes its text and nothing else rather than borrowing a neighbour's.
183+
*/
184+
function assistantThinking(
185+
message: OcxAssistantMessage,
186+
): { thinking?: string; signature?: string } {
187+
const blocks = message.content.filter(
188+
(part): part is Extract<typeof part, { type: "thinking" }> => part.type === "thinking",
189+
);
190+
if (blocks.length === 0) return {};
191+
const thinking = blocks.map(b => b.thinking).filter(Boolean).join("\n");
192+
// Only one signature can ride the prompt, so take the last block that has
193+
// one: that is the block the turn actually ended on.
194+
const signature = blocks.filter(b => b.signature).at(-1)?.signature;
195+
return {
196+
...(thinking ? { thinking } : {}),
197+
...(signature ? { signature } : {}),
198+
};
199+
}
200+
172201
export function mapOcxMessagesToDevin(parsed: OcxParsedRequest): ChatHistoryItem[] {
173202
const items: ChatHistoryItem[] = [];
174203
// Cognition is not an OpenAI host, and this adapter does advertise a real
@@ -203,11 +232,15 @@ function mapOneMessage(message: OcxMessage): ChatHistoryItem | undefined {
203232
if (message.role === "assistant") {
204233
const toolCalls = assistantToolCalls(message);
205234
const text = assistantText(message);
206-
if (!text && toolCalls.length === 0) return undefined;
235+
const reasoning = assistantThinking(message);
236+
// A turn that produced only reasoning is still worth replaying: dropping it
237+
// is what makes the next turn re-derive the same chain.
238+
if (!text && toolCalls.length === 0 && !reasoning.thinking) return undefined;
207239
return {
208240
role: "assistant",
209241
content: text || "",
210242
...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
243+
...reasoning,
211244
};
212245
}
213246
if (message.role === "toolResult") {
@@ -335,6 +368,12 @@ export function createDevinAdapter(
335368
if (event.text) emit({ type: "thinking_delta", thinking: event.text });
336369
continue;
337370
}
371+
if (event.kind === "reasoning_signature") {
372+
// Carried back out so the next turn can replay it in the prompt's
373+
// signature field; an unsigned replay is what the service ignores.
374+
emit({ type: "thinking_signature", signature: event.signature });
375+
continue;
376+
}
338377
if (event.kind === "tool_call_start") {
339378
closeOpenTool();
340379
openToolId = event.id;

‎src/adapters/devin/cloud-direct/chat.ts‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,9 @@ export function allocateCascadeId(): string {
120120
* #4 num_tokens: int (rough estimate)
121121
* #5 safe_for_code_telemetry: bool (1 = ok to log)
122122
* #10 images: repeated ImageData (multimodal)
123+
* #11 thinking: string (assistant reasoning, replayed)
124+
* #12 signature: string (opaque attestation for #11)
125+
* #18 signature_type: string
123126
* }
124127
*
125128
* ImageData (exa.codeium_common_pb.ImageData) {
@@ -153,7 +156,13 @@ function encodeChatToolCall(tc: { id: string; name: string; arguments: string })
153156
function encodeChatMessagePrompt(
154157
content: ContentPart[],
155158
source: number,
156-
opts?: { toolCallId?: string; toolCalls?: Array<{ id: string; name: string; arguments: string }> },
159+
opts?: {
160+
toolCallId?: string;
161+
toolCalls?: Array<{ id: string; name: string; arguments: string }>;
162+
thinking?: string;
163+
signature?: string;
164+
signatureType?: string;
165+
},
157166
): Buffer {
158167
const textParts = content.filter((p): p is { type: 'text'; text: string } => p.type === 'text');
159168
const imageParts = content.filter((p): p is { type: 'image'; mimeType: string; base64Data: string; caption?: string } => p.type === 'image');
@@ -178,6 +187,14 @@ function encodeChatMessagePrompt(
178187
for (const img of imageParts) {
179188
parts.push(encodeMessage(10, encodeImageData(img)));
180189
}
190+
// Reasoning replay. This adapter used to assert that Cognition has no
191+
// reasoning-replay field and drop the assistant's own thinking, so a
192+
// reasoning model restarted its chain on every turn of a tool loop. Two
193+
// independent clients of the same service write it here: #11 thinking,
194+
// #12 signature, #18 signature_type on the assistant prompt.
195+
if (opts?.thinking) parts.push(encodeString(11, opts.thinking));
196+
if (opts?.signature) parts.push(encodeString(12, opts.signature));
197+
if (opts?.signatureType) parts.push(encodeString(18, opts.signatureType));
181198
return Buffer.concat(parts);
182199
}
183200

@@ -342,6 +359,15 @@ export interface ChatHistoryItem {
342359
* each ChatToolCall has #1 id, #2 name, #3 arguments_json).
343360
*/
344361
tool_calls?: Array<{ id: string; name: string; arguments: string }>;
362+
/**
363+
* For `role: 'assistant'` only — the model's own reasoning from that turn,
364+
* replayed so a reasoning model does not restart its chain on the next one.
365+
* Encoded as ChatMessagePrompt #11 with its #12 signature and #18
366+
* signature_type.
367+
*/
368+
thinking?: string;
369+
signature?: string;
370+
signature_type?: string;
345371
}
346372

347373
/**
@@ -399,6 +425,12 @@ export interface ToolDef {
399425
export type CloudChatEvent =
400426
| { kind: 'text'; text: string }
401427
| { kind: 'reasoning'; text: string }
428+
/**
429+
* `delta_signature` (#10) — the opaque attestation for the reasoning this
430+
* turn produced. Without decoding it there is nothing to put in the prompt's
431+
* #12 on the next turn, so the replay would always be unsigned.
432+
*/
433+
| { kind: 'reasoning_signature'; signature: string }
402434
| { kind: 'tool_call_start'; id: string; name: string }
403435
| {
404436
kind: 'tool_call_args';
@@ -592,6 +624,9 @@ function buildGetChatMessageRequest(args: BuildArgs): Buffer {
592624
{
593625
toolCallId: m.role === 'tool' ? m.tool_call_id : undefined,
594626
toolCalls: m.role === 'assistant' ? m.tool_calls : undefined,
627+
thinking: m.role === 'assistant' ? m.thinking : undefined,
628+
signature: m.role === 'assistant' ? m.signature : undefined,
629+
signatureType: m.role === 'assistant' ? m.signature_type : undefined,
595630
},
596631
),
597632
),
@@ -716,6 +751,9 @@ export function* decodeChatFrame(proto: Buffer): Generator<CloudChatEvent> {
716751
// block instead of inline with the answer.
717752
const s = (f.value as Buffer).toString('utf8');
718753
if (s) yield { kind: 'reasoning', text: s };
754+
} else if (f.num === 10 && f.wire === 2 && Buffer.isBuffer(f.value)) {
755+
const s = (f.value as Buffer).toString('utf8');
756+
if (s) yield { kind: 'reasoning_signature', signature: s };
719757
} else if (f.num === 6 && f.wire === 2 && Buffer.isBuffer(f.value)) {
720758
let id: string | undefined;
721759
let name: string | undefined;

‎tests/providers/devin-hardening.test.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { describe, expect, test } from "bun:test";
22
import { normalizeDevinModelId } from "../../src/adapters/devin";
3+
import { mapOcxMessagesToDevin } from "../../src/adapters/devin";
34
import { parseDevinAuthPaste, refreshDevinToken } from "../../src/oauth/devin";
45
import { DEVIN_DEFAULT_API_SERVER, resolveDevinApiBaseUrl, validateDevinApiBaseUrl } from "../../src/oauth/devin/api-base";
56
import { registerUser } from "../../src/oauth/devin/register-user";
@@ -444,3 +445,76 @@ describe("devin status classification across the newly reachable trailer codes",
444445
expect(cls(504)).toEqual({ status: 504, retryable: true });
445446
});
446447
});
448+
449+
describe("devin reasoning replay", () => {
450+
const parsedWith = (messages: unknown[]) => ({
451+
context: { messages, tools: undefined, systemPrompt: undefined },
452+
options: { toolChoice: undefined },
453+
}) as never;
454+
function uvarint(value: number): number[] {
455+
const out: number[] = [];
456+
let v = value;
457+
do { const b = v & 0x7f; v = Math.floor(v / 128); out.push(v > 0 ? b | 0x80 : b); } while (v > 0);
458+
return out;
459+
}
460+
function lenDelim(num: number, payload: Buffer): Buffer {
461+
return Buffer.concat([Buffer.from([...uvarint((num << 3) | 2), ...uvarint(payload.length)]), payload]);
462+
}
463+
function fieldsOf(buf: Buffer): Record<number, Buffer[]> {
464+
const out: Record<number, Buffer[]> = {};
465+
for (const f of iterFields(buf)) {
466+
if (Buffer.isBuffer(f.value)) (out[f.num] ??= []).push(f.value);
467+
}
468+
return out;
469+
}
470+
471+
test("an assistant turn's thinking and signature ride the prompt instead of being dropped", () => {
472+
// The adapter used to assert this field did not exist and drop the chain,
473+
// so a reasoning model re-derived it on every turn of a tool loop.
474+
const history = mapOcxMessagesToDevin(parsedWith([
475+
{ role: "user", content: [{ type: "text", text: "hi" }] },
476+
{
477+
role: "assistant",
478+
content: [
479+
{ type: "thinking", thinking: "step one", signature: "sig-abc" },
480+
{ type: "text", text: "answer" },
481+
],
482+
},
483+
]));
484+
const assistant = history.find(m => m.role === "assistant");
485+
expect(assistant?.thinking).toBe("step one");
486+
expect(assistant?.signature).toBe("sig-abc");
487+
// Reasoning must not leak into the visible text.
488+
expect(assistant?.content).toBe("answer");
489+
});
490+
491+
test("a turn that produced only reasoning is still replayed", () => {
492+
const history = mapOcxMessagesToDevin(parsedWith([
493+
{ role: "user", content: [{ type: "text", text: "hi" }] },
494+
{ role: "assistant", content: [{ type: "thinking", thinking: "only thought" }] },
495+
]));
496+
expect(history.find(m => m.role === "assistant")?.thinking).toBe("only thought");
497+
});
498+
499+
test("the encoded prompt carries thinking at #11 and its signature at #12", () => {
500+
const req = buildGetChatMessageRequestForTests({
501+
apiKey: "devin-session-token$x",
502+
modelUid: "swe-2",
503+
messages: [
504+
{ role: "user", content: "hi" },
505+
{ role: "assistant", content: "answer", thinking: "step one", signature: "sig-abc" },
506+
],
507+
cascadeId: "c",
508+
} as never);
509+
const prompts = fieldsOf(req)[3] ?? [];
510+
const assistantPrompt = prompts.map(fieldsOf).find(p => p[11]);
511+
expect(assistantPrompt?.[11]?.[0]?.toString("utf8")).toBe("step one");
512+
expect(assistantPrompt?.[12]?.[0]?.toString("utf8")).toBe("sig-abc");
513+
});
514+
515+
test("the response signature is decoded so there is something to replay", () => {
516+
const frame = lenDelim(10, Buffer.from("sig-from-cloud", "utf8"));
517+
const events = [...decodeChatFrame(frame)];
518+
expect(events).toEqual([{ kind: "reasoning_signature", signature: "sig-from-cloud" }]);
519+
});
520+
});

0 commit comments

Comments
 (0)