Skip to content

Commit 503a2df

Browse files
committed
feat(model): add preserveInterruptedResponse option for KV cache consistency
When enabled on a model, interrupted responses are saved with their partial content (marked with [interrupted]) instead of being discarded. This preserves the token sequence for local inference backends with KV cache (like llama.cpp), avoiding expensive context reprocessing on the next request. Configuration example: ```json { "provider": { "my-local": { "models": { "my-model": { "preserveInterruptedResponse": true } } } } } ``` Closes anomalyco#19081 Developed with assistance from Claude Opus 4.5
1 parent f736116 commit 503a2df

6 files changed

Lines changed: 60 additions & 2 deletions

File tree

‎packages/opencode/src/plugin/codex.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,7 @@ export async function CodexAuthPlugin(input: PluginInput): Promise<Hooks> {
393393
input: { text: true, audio: false, image: true, video: false, pdf: false },
394394
output: { text: true, audio: false, image: false, video: false, pdf: false },
395395
interleaved: false,
396+
preserveInterruptedResponse: false,
396397
},
397398
cost: { input: 0, output: 0, cache: { read: 0, write: 0 } },
398399
limit: { context: 400_000, input: 272_000, output: 128_000 },

‎packages/opencode/src/provider/models.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@ export namespace ModelsDev {
3434
.strict(),
3535
])
3636
.optional(),
37+
preserveInterruptedResponse: z
38+
.boolean()
39+
.optional()
40+
.describe(
41+
"When enabled, interrupted responses are saved with their partial content instead of being discarded. " +
42+
"This preserves the token sequence for local inference backends with KV cache, avoiding expensive reprocessing.",
43+
),
3744
cost: z
3845
.object({
3946
input: z.number(),

‎packages/opencode/src/provider/provider.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,7 @@ export namespace Provider {
643643
pdf: false,
644644
},
645645
interleaved: false,
646+
preserveInterruptedResponse: false,
646647
},
647648
release_date: "",
648649
variants: {},
@@ -807,6 +808,7 @@ export namespace Provider {
807808
field: z.enum(["reasoning_content", "reasoning_details"]),
808809
}),
809810
]),
811+
preserveInterruptedResponse: z.boolean(),
810812
}),
811813
cost: z.object({
812814
input: z.number(),
@@ -914,6 +916,7 @@ export namespace Provider {
914916
pdf: model.modalities?.output?.includes("pdf") ?? false,
915917
},
916918
interleaved: model.interleaved ?? false,
919+
preserveInterruptedResponse: model.preserveInterruptedResponse ?? false,
917920
},
918921
release_date: model.release_date,
919922
variants: {},
@@ -1034,6 +1037,10 @@ export namespace Provider {
10341037
pdf: model.modalities?.output?.includes("pdf") ?? existingModel?.capabilities.output.pdf ?? false,
10351038
},
10361039
interleaved: model.interleaved ?? false,
1040+
preserveInterruptedResponse:
1041+
model.preserveInterruptedResponse ??
1042+
existingModel?.capabilities.preserveInterruptedResponse ??
1043+
false,
10371044
},
10381045
cost: {
10391046
input: model?.cost?.input ?? existingModel?.cost?.input ?? 0,

‎packages/opencode/src/session/processor.ts‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,10 @@ export namespace SessionProcessor {
4848
needsCompaction = false
4949
const shouldBreak = (await Config.get()).experimental?.continue_loop_on_deny !== true
5050
while (true) {
51+
// Moved outside try block so they're accessible in catch for preserveInterruptedResponse
52+
let currentText: MessageV2.TextPart | undefined
53+
let reasoningMap: Record<string, MessageV2.ReasoningPart> = {}
5154
try {
52-
let currentText: MessageV2.TextPart | undefined
53-
let reasoningMap: Record<string, MessageV2.ReasoningPart> = {}
5455
const stream = await LLM.stream(streamInput)
5556

5657
for await (const value of stream.fullStream) {
@@ -384,6 +385,34 @@ export namespace SessionProcessor {
384385
})
385386
await SessionStatus.set(input.sessionID, { type: "idle" })
386387
}
388+
389+
// Preserve partial response content when interrupted and option is enabled
390+
// This maintains token sequence consistency for local inference backends with KV cache
391+
if (input.abort.aborted && input.model.capabilities.preserveInterruptedResponse) {
392+
log.info("preserving interrupted response", { sessionID: input.sessionID })
393+
394+
// Finalize any partial text with interrupt marker
395+
if (currentText && currentText.text) {
396+
currentText.text = currentText.text.trimEnd() + "\n\n[interrupted]"
397+
currentText.time = {
398+
start: currentText.time?.start ?? Date.now(),
399+
end: Date.now(),
400+
}
401+
await Session.updatePart(currentText)
402+
}
403+
404+
// Finalize any pending reasoning parts with interrupt marker
405+
for (const [id, part] of Object.entries(reasoningMap)) {
406+
if (part.text) {
407+
part.text = part.text.trimEnd() + "\n\n[interrupted]"
408+
part.time = {
409+
start: part.time?.start ?? Date.now(),
410+
end: Date.now(),
411+
}
412+
await Session.updatePart(part)
413+
}
414+
}
415+
}
387416
}
388417
if (snapshot) {
389418
const patch = await Snapshot.patch(snapshot)

‎packages/opencode/test/provider/transform.test.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ describe("ProviderTransform.options - setCacheKey", () => {
2424
input: { text: true, audio: false, image: true, video: false, pdf: true },
2525
output: { text: true, audio: false, image: false, video: false, pdf: false },
2626
interleaved: false,
27+
preserveInterruptedResponse: false,
2728
},
2829
cost: {
2930
input: 0.003,
@@ -125,6 +126,7 @@ describe("ProviderTransform.options - google thinkingConfig gating", () => {
125126
input: { text: true, audio: false, image: true, video: false, pdf: true },
126127
output: { text: true, audio: false, image: false, video: false, pdf: false },
127128
interleaved: false,
129+
preserveInterruptedResponse: false,
128130
},
129131
cost: {
130132
input: 0.001,
@@ -191,6 +193,7 @@ describe("ProviderTransform.options - gpt-5 textVerbosity", () => {
191193
input: { text: true, audio: false, image: true, video: false, pdf: false },
192194
output: { text: true, audio: false, image: false, video: false, pdf: false },
193195
interleaved: false,
196+
preserveInterruptedResponse: false,
194197
},
195198
cost: { input: 0.03, output: 0.06, cache: { read: 0.001, write: 0.002 } },
196199
limit: { context: 128000, output: 4096 },
@@ -263,6 +266,7 @@ describe("ProviderTransform.options - gateway", () => {
263266
input: { text: true, audio: false, image: true, video: false, pdf: true },
264267
output: { text: true, audio: false, image: false, video: false, pdf: false },
265268
interleaved: false,
269+
preserveInterruptedResponse: false,
266270
},
267271
cost: {
268272
input: 0.001,
@@ -309,6 +313,7 @@ describe("ProviderTransform.providerOptions", () => {
309313
input: { text: true, audio: false, image: true, video: false, pdf: false },
310314
output: { text: true, audio: false, image: false, video: false, pdf: false },
311315
interleaved: false,
316+
preserveInterruptedResponse: false,
312317
},
313318
cost: {
314319
input: 0.001,
@@ -825,6 +830,7 @@ describe("ProviderTransform.message - DeepSeek reasoning content", () => {
825830
interleaved: {
826831
field: "reasoning_content",
827832
},
833+
preserveInterruptedResponse: false,
828834
},
829835
cost: {
830836
input: 0.001,
@@ -885,6 +891,7 @@ describe("ProviderTransform.message - DeepSeek reasoning content", () => {
885891
input: { text: true, audio: false, image: true, video: false, pdf: false },
886892
output: { text: true, audio: false, image: false, video: false, pdf: false },
887893
interleaved: false,
894+
preserveInterruptedResponse: false,
888895
},
889896
cost: {
890897
input: 0.03,
@@ -929,6 +936,7 @@ describe("ProviderTransform.message - empty image handling", () => {
929936
input: { text: true, audio: false, image: true, video: false, pdf: true },
930937
output: { text: true, audio: false, image: false, video: false, pdf: false },
931938
interleaved: false,
939+
preserveInterruptedResponse: false,
932940
},
933941
cost: {
934942
input: 0.003,
@@ -1032,6 +1040,7 @@ describe("ProviderTransform.message - anthropic empty content filtering", () =>
10321040
input: { text: true, audio: false, image: true, video: false, pdf: true },
10331041
output: { text: true, audio: false, image: false, video: false, pdf: false },
10341042
interleaved: false,
1043+
preserveInterruptedResponse: false,
10351044
},
10361045
cost: {
10371046
input: 0.003,
@@ -1239,6 +1248,7 @@ describe("ProviderTransform.message - strip openai metadata when store=false", (
12391248
input: { text: true, audio: false, image: true, video: false, pdf: false },
12401249
output: { text: true, audio: false, image: false, video: false, pdf: false },
12411250
interleaved: false,
1251+
preserveInterruptedResponse: false,
12421252
},
12431253
cost: { input: 0.03, output: 0.06, cache: { read: 0.001, write: 0.002 } },
12441254
limit: { context: 128000, output: 4096 },
@@ -1531,6 +1541,7 @@ describe("ProviderTransform.message - providerOptions key remapping", () => {
15311541
input: { text: true, audio: false, image: true, video: false, pdf: true },
15321542
output: { text: true, audio: false, image: false, video: false, pdf: false },
15331543
interleaved: false,
1544+
preserveInterruptedResponse: false,
15341545
},
15351546
cost: { input: 0.001, output: 0.002, cache: { read: 0.0001, write: 0.0002 } },
15361547
limit: { context: 128000, output: 8192 },
@@ -1685,6 +1696,7 @@ describe("ProviderTransform.message - cache control on gateway", () => {
16851696
input: { text: true, audio: false, image: true, video: false, pdf: true },
16861697
output: { text: true, audio: false, image: false, video: false, pdf: false },
16871698
interleaved: false,
1699+
preserveInterruptedResponse: false,
16881700
},
16891701
cost: { input: 0.001, output: 0.002, cache: { read: 0.0001, write: 0.0002 } },
16901702
limit: { context: 200_000, output: 8192 },
@@ -1783,6 +1795,7 @@ describe("ProviderTransform.variants", () => {
17831795
input: { text: true, audio: false, image: true, video: false, pdf: false },
17841796
output: { text: true, audio: false, image: false, video: false, pdf: false },
17851797
interleaved: false,
1798+
preserveInterruptedResponse: false,
17861799
},
17871800
cost: {
17881801
input: 0.001,

‎packages/opencode/test/session/message-v2.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ const model: Provider.Model = {
3737
pdf: false,
3838
},
3939
interleaved: false,
40+
preserveInterruptedResponse: false,
4041
},
4142
cost: {
4243
input: 0,

0 commit comments

Comments
 (0)