Skip to content

Commit b01f166

Browse files
lidge-junlonefisher
andcommitted
test(devin): independently verify system blocklist rewriting
Decode request field #2 so a sanitized tool description cannot mask a system-prompt regression. Cover flexible whitespace and literal tool results, and align sanitizer comments with instruction-only scope. Co-authored-by: Terry Yu <[email protected]>
1 parent 24b94d2 commit b01f166

2 files changed

Lines changed: 14 additions & 7 deletions

File tree

‎src/adapters/devin/cloud-direct/chat.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -628,8 +628,6 @@ const MAX_TOOL_DESC_LEN = 6998;
628628
* for "whether to allow", verified live to clear the filter while preserving
629629
* the instruction's meaning.
630630
*
631-
* Scope note: the first entries only ever appeared in tool descriptions, but
632-
* this one lives in request #2 (the system prompt) and could equally appear
633631
* Scope note: the sanitizer only ever touches *instruction surfaces* —
634632
* tool descriptions and the #2 system prompt, where a meaning-preserving
635633
* reword loses nothing. It deliberately does NOT touch data fields:
@@ -669,7 +667,7 @@ export function sanitizeToolDescriptionForCognitionForTests(description: string)
669667
return sanitizeTextForCognition(description);
670668
}
671669

672-
/** Text sanitizer applied to every string the adapter puts on the wire. */
670+
/** Test-only: exercise the sanitizer used for system text and tool descriptions. */
673671
export function sanitizeTextForCognitionForTests(text: string): string {
674672
return sanitizeTextForCognition(text);
675673
}

‎tests/providers/devin-adapter.test.ts‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { createDevinAdapter, mapDevinToolCallStartForTests, mapOcxMessagesToDevi
66
import { buildGetChatMessageRequestForTests, sanitizeTextForCognitionForTests, sanitizeToolDescriptionForCognitionForTests } from "../../src/adapters/devin/cloud-direct/chat";
77
import { DEVIN_MODEL_CONTEXT_WINDOWS, DEVIN_STATIC_MODELS, collapseDevinModelUid } from "../../src/adapters/devin/live-models";
88
import { parseCatalogBuffer } from "../../src/adapters/devin/cloud-direct/catalog";
9-
import { encodeMessage, encodeString, encodeVarintField } from "../../src/adapters/devin/cloud-direct/wire";
9+
import { encodeMessage, encodeString, encodeVarintField, iterFields } from "../../src/adapters/devin/cloud-direct/wire";
1010
import { DEPRECATED_OAUTH_PROVIDER_ALIASES, OAUTH_PROVIDERS, resolveRefreshPolicy } from "../../src/oauth";
1111
import { DEVIN_DEFAULT_API_SERVER } from "../../src/oauth/devin";
1212
import { saveCredential } from "../../src/oauth/store";
@@ -304,7 +304,7 @@ describe("devin adapter", () => {
304304
expect(sanitizeToolDescriptionForCognitionForTests(nearMiss)).toBe(nearMiss);
305305
});
306306

307-
test("rewrites the Codex escalation-instruction phrase in every wire field", () => {
307+
test("rewrites the Codex escalation phrase on instruction surfaces and preserves data", () => {
308308
// This trigger lives in Codex's <permissions instructions> boilerplate,
309309
// which Codex sends as system prompt — not in a tool description. The
310310
// clause was isolated live: the full sentence is refused while every
@@ -318,6 +318,9 @@ describe("devin adapter", () => {
318318
// Flexible whitespace/case, same as the other Codex entries.
319319
expect(sanitizeTextForCognitionForTests(trigger.toUpperCase()))
320320
.toContain("whether to allow the action");
321+
expect(sanitizeTextForCognitionForTests(trigger.replaceAll(" ", "\n "))).toBe(
322+
"asking the user whether to allow the action in the `justification` parameter",
323+
);
321324

322325
// The phrase is rewritten on instruction surfaces — #2 (system prompt)
323326
// and tool descriptions — but data fields must stay byte-exact: a user
@@ -342,17 +345,23 @@ describe("devin adapter", () => {
342345
thinking: `the prompt says: ${trigger}`,
343346
tool_calls: [{ id: "c1", name: "apply_patch", arguments: `{"patch":"${trigger}"}` }],
344347
},
348+
{ role: "tool", tool_call_id: "c1", content: `file bytes: ${trigger}` },
345349
{ role: "user", content: "hello" },
346350
],
347351
tools: [{ name: "codex_escalation", description: trigger, parameters: { type: "object" } }],
348352
});
349-
// Instruction surfaces rewritten...
350-
expect(req.includes(Buffer.from("asking the user whether to allow the action", "utf8"))).toBe(true);
353+
// Decode #2 independently: a rewritten tool description cannot mask an
354+
// unsanitized system prompt in a whole-buffer substring assertion.
355+
const systemField = [...iterFields(req)].find((field) => field.num === 2);
356+
expect(systemField?.value).toEqual(Buffer.from(
357+
`<permissions instructions>\n- Include a short question asking the user whether to allow the action in the \`justification\` parameter. e.g. "Do it?"\n</permissions>`,
358+
));
351359
// ...but data fields preserve the literal bytes (user text, thinking,
352360
// tool-call arguments all still carry the verbatim trigger).
353361
expect(req.includes(Buffer.from(`here is a quote: ${trigger}`, "utf8"))).toBe(true);
354362
expect(req.includes(Buffer.from(`the prompt says: ${trigger}`, "utf8"))).toBe(true);
355363
expect(req.includes(Buffer.from(`{"patch":"${trigger}"}`, "utf8"))).toBe(true);
364+
expect(req.includes(Buffer.from(`file bytes: ${trigger}`, "utf8"))).toBe(true);
356365

357366
// A request where the ONLY trigger carrier is a tool description must
358367
// come out clean — pins encodeToolDef specifically.

0 commit comments

Comments
 (0)