Skip to content

Commit c64b853

Browse files
HQ1995claude
andcommitted
fix(devin): keep one trajectory per named conversation so prompt caching hits
GetChatMessage stamped a fresh CortexTrajectoryReference id (#15) on every request. Cognition routes a trajectory to the replica that holds its prompt cache, so every turn of one conversation landed on a cold replica: SWE-2 reported no cache reads even for byte-identical repeats. A conversation the client names (client thread, session_id or x-session-affinity) now keeps one trajectory across the request-scoped adapters the proxy builds per HTTP request. Unnamed turns still mint one per request, and a live trajectory is never lent to an overlapping turn, because the server holds or drops concurrent turns on one id. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 2e3acab commit c64b853

3 files changed

Lines changed: 108 additions & 4 deletions

File tree

‎src/adapters/devin.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import type { AdapterEvent, OcxAssistantMessage, OcxContentPart, OcxMessage, Ocx
1010
import { namespacedToolName } from "../types";
1111
import type { IncomingMeta, ProviderAdapter } from "./base";
1212
import { streamChatEventsWithResetRetry, devinStatedResetWaitMs, allocateCascadeId, CloudChatError, type ChatHistoryItem, type ToolDef } from "./devin/cloud-direct";
13-
import type { CloudChatEvent, ContentPart } from "./devin/cloud-direct/chat";
13+
import { devinCacheIdentity, type CloudChatEvent, type ContentPart } from "./devin/cloud-direct/chat";
1414
import { getCachedCatalog, type CacheEntry, type ModelCatalogEntry } from "./devin/cloud-direct/catalog";
1515
import { collapseDevinModelUid, devinFamiliesOf, devinFamilyBaseId, selectDevinFamilyMember, type DevinVariantRequest } from "./devin/live-models";
1616
import { buildNonOpenAIToolCatalogNudgeForTools } from "./tool-catalog-nudge";
@@ -656,6 +656,32 @@ export function mapDevinToolCallStartForTests(
656656
return mapDevinToolCallStart(id, name, buildDevinReturnedToolNameMap(tools));
657657
}
658658

659+
/**
660+
* Cognition routes a CortexTrajectoryReference to the replica that holds its
661+
* prompt cache. A conversation the client names keeps one trajectory, which
662+
* outlives the request-scoped adapter a proxy builds per HTTP request; an
663+
* unnamed turn still mints one per request. A live trajectory is never lent
664+
* to an overlapping turn: the server holds or drops concurrent turns on one id.
665+
*/
666+
const TRAJECTORY_ID_MAX = 256;
667+
const trajectoryIds = new Map<string, string>();
668+
const liveTrajectories = new Set<string>();
669+
670+
function claimTrajectory(apiKey: string, host: string, conversation: string | null | undefined): string | undefined {
671+
if (!conversation) return undefined;
672+
const key = devinCacheIdentity(apiKey, `${host}\x1f${conversation}`);
673+
let id = trajectoryIds.get(key);
674+
if (!id) {
675+
const oldest = trajectoryIds.size >= TRAJECTORY_ID_MAX ? trajectoryIds.keys().next().value : undefined;
676+
if (oldest !== undefined) trajectoryIds.delete(oldest);
677+
id = crypto.randomUUID();
678+
trajectoryIds.set(key, id);
679+
}
680+
if (liveTrajectories.has(id)) return undefined;
681+
liveTrajectories.add(id);
682+
return id;
683+
}
684+
659685
export function createDevinAdapter(
660686
provider: OcxProviderConfig,
661687
context: { providerId?: string } = {},
@@ -744,6 +770,9 @@ export function createDevinAdapter(
744770
let messages: ChatHistoryItem[] = [];
745771
let tools: ToolDef[] | undefined;
746772

773+
const trajectoryId = claimTrajectory(apiKey, host, parsed._clientThreadId
774+
|| incoming.headers.get("session_id") || incoming.headers.get("x-session-affinity"));
775+
747776
const closeOpenTool = () => {
748777
if (!openToolId) return;
749778
emit({ type: "tool_call_end" });
@@ -780,6 +809,7 @@ export function createDevinAdapter(
780809
messages,
781810
tools,
782811
cascadeId,
812+
trajectoryId,
783813
completionOpts: {
784814
...(maxOutputTokens !== undefined ? { maxOutputTokens } : {}),
785815
...(typeof parsed.options.temperature === "number" ? { temperature: parsed.options.temperature } : {}),
@@ -961,6 +991,8 @@ export function createDevinAdapter(
961991
...(error instanceof CloudChatError && error.code ? { code: error.code } : {}),
962992
...(usage ? { usage } : {}),
963993
});
994+
} finally {
995+
if (trajectoryId) liveTrajectories.delete(trajectoryId);
964996
}
965997
},
966998
};

‎src/adapters/devin/cloud-direct/chat.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -536,6 +536,8 @@ export type CloudChatEvent =
536536

537537
interface BuildArgs {
538538
apiKey: string;
539+
/** CortexTrajectoryReference #15 id; minted per request when absent. */
540+
trajectoryId?: string;
539541
userJwt?: string;
540542
modelUid: string;
541543
messages: ChatHistoryItem[];
@@ -779,9 +781,10 @@ function buildGetChatMessageRequest(args: BuildArgs): Buffer {
779781
// native client and CLIProxyAPIPlus, which places it outside its tools gate.
780782
encodeMessage(13, encodeVarintField(1, PROMPT_CACHE_EPHEMERAL)),
781783
// #15 CortexTrajectoryReference: { id, 1, 4 }. Present on every verified
782-
// request.
784+
// request. Cognition routes a trajectory to the replica holding its prompt
785+
// cache, so a fresh id on every turn of one conversation is a cold miss.
783786
encodeMessage(15, Buffer.concat([
784-
encodeString(1, crypto.randomUUID()),
787+
encodeString(1, args.trajectoryId ?? crypto.randomUUID()),
785788
encodeVarintField(2, 1),
786789
encodeVarintField(3, 4),
787790
])),
@@ -1105,6 +1108,8 @@ export interface CloudChatRequest {
11051108
tools?: ToolDef[];
11061109
/** Cascade ID — reuse across turns of the same conversation. */
11071110
cascadeId?: string;
1111+
/** Trajectory ID (#15) — reuse across sequential turns of one conversation. */
1112+
trajectoryId?: string;
11081113
/** Optional sampling overrides. */
11091114
completionOpts?: BuildArgs['completionOpts'];
11101115
/** Override request_type (default = 5, CASCADE). */
@@ -1255,6 +1260,7 @@ export async function* streamChatEvents(req: CloudChatRequest): AsyncGenerator<C
12551260
tools: req.tools,
12561261
cascadeId: sessionIds.cascadeId,
12571262
sessionId: sessionIds.sessionId,
1263+
trajectoryId: req.trajectoryId,
12581264
requestId: BigInt(Date.now()),
12591265
triggerId: crypto.randomUUID(),
12601266
requestType: req.requestType,

‎tests/providers/devin-prompt-cache.test.ts‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,21 @@ describe("prompt cache options on the wire", () => {
5454
});
5555
});
5656

57+
describe("trajectory reference on the wire", () => {
58+
function trajectoryOf(buf: Buffer): string {
59+
const reference = [...iterFields(buf)].find(field => field.num === 15)!.value as Buffer;
60+
return ([...iterFields(reference)].find(field => field.num === 1)!.value as Buffer).toString();
61+
}
62+
63+
test("a supplied trajectory id is sent as #15.1", () => {
64+
expect(trajectoryOf(buildRequest({ trajectoryId: "trajectory-1" }))).toBe("trajectory-1");
65+
});
66+
67+
test("without one, every request mints its own", () => {
68+
expect(trajectoryOf(buildRequest())).not.toBe(trajectoryOf(buildRequest()));
69+
});
70+
});
71+
5772
describe("devin cache identity", () => {
5873
test("the raw credential never becomes the cache key", () => {
5974
const apiKey = "devin-secret-token-value";
@@ -128,14 +143,15 @@ describe("one catalog read serves the cached chat path", () => {
128143
provider: Partial<OcxProviderConfig> = {},
129144
options: OcxParsedRequest["options"] = {},
130145
signal?: AbortSignal,
146+
headers = new Headers(),
131147
): Promise<AdapterEvent[]> {
132148
const adapter = createDevinAdapter({ ...provider, adapter: "devin", apiKey, baseUrl: host });
133149
const events: AdapterEvent[] = [];
134150
await adapter.runTurn!({
135151
modelId, stream: true,
136152
context: { messages: [{ role: "user", content: "hi", timestamp: 1 }] },
137153
options: { maxOutputTokens: 64, ...options },
138-
}, { headers: new Headers(), translatorBudget: createTranslatorBudget(), abortSignal: signal },
154+
}, { headers, translatorBudget: createTranslatorBudget(), abortSignal: signal },
139155
event => { events.push(event); });
140156
return events;
141157
}
@@ -226,6 +242,56 @@ describe("one catalog read serves the cached chat path", () => {
226242
expect(requests).toHaveLength(0);
227243
});
228244

245+
function sentTrajectories(): string[] {
246+
return requests.map(request => {
247+
const reference = fields(request).get(15)!.value as Buffer;
248+
return (fields(reference).get(1)!.value as Buffer).toString();
249+
});
250+
}
251+
const conversation = (id: string) => new Headers({ session_id: id });
252+
253+
test("a named conversation keeps one trajectory across request-scoped adapters", async () => {
254+
// Cognition routes a trajectory to the replica holding its prompt cache; a
255+
// proxy builds a fresh adapter for every HTTP request.
256+
seed([{ uid: "swe-2-high", window: 262_000 }]);
257+
await run("swe-2-high", {}, {}, undefined, conversation("conversation-a"));
258+
await run("swe-2-high", {}, {}, undefined, new Headers({ "x-session-affinity": "conversation-a" }));
259+
await run("swe-2-high", {}, {}, undefined, conversation("conversation-b"));
260+
const [first, second, other] = sentTrajectories();
261+
expect(second).toBe(first);
262+
expect(other).not.toBe(first);
263+
});
264+
265+
test("unnamed turns still mint a trajectory per request", async () => {
266+
seed([{ uid: "swe-2-high", window: 262_000 }]);
267+
await run();
268+
await run();
269+
const [first, second] = sentTrajectories();
270+
expect(second).not.toBe(first);
271+
});
272+
273+
test("an overlapping turn never shares the live trajectory of its conversation", async () => {
274+
seed([{ uid: "swe-2-high", window: 262_000 }]);
275+
const recorded = globalThis.fetch;
276+
let release!: () => void;
277+
const held = new Promise<void>(resolve => { release = resolve; });
278+
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
279+
const response = await recorded(input, init);
280+
if (String(input).endsWith("/GetChatMessage") && requests.length === 1) await held;
281+
return response;
282+
}) as typeof fetch;
283+
const first = run("swe-2-high", {}, {}, undefined, conversation("conversation-c"));
284+
while (requests.length < 1) await Bun.sleep(1);
285+
await run("swe-2-high", {}, {}, undefined, conversation("conversation-c"));
286+
release();
287+
await first;
288+
await run("swe-2-high", {}, {}, undefined, conversation("conversation-c"));
289+
const [live, overlapping, later] = sentTrajectories();
290+
expect(overlapping).not.toBe(live);
291+
// Released once the turn ends, so the conversation keeps its trajectory.
292+
expect(later).toBe(live);
293+
});
294+
229295
test("an already cancelled turn never fetches metadata or sends inference", async () => {
230296
const controller = new AbortController();
231297
controller.abort();

0 commit comments

Comments
 (0)