Skip to content

Commit 678a4a8

Browse files
fix(core): keep using the provider's count when plugins shape the request
Built-in tool plugins register compaction hooks, so skipping the count when any hook existed meant it was never used. A hook that grows the request is now caught by the provider's rejection and the request is resent as text. Leave only as much room for the summary as the request can generate. Co-authored-by: Kit Langton <[email protected]>
1 parent 315a6b7 commit 678a4a8

2 files changed

Lines changed: 53 additions & 27 deletions

File tree

‎packages/core/src/session/compaction.ts‎

Lines changed: 16 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import { Bus } from "../bus.js"
2424
import { Database } from "../database/database.js"
2525
import { llmClient } from "../effect/app-node-platform.js"
2626
import { Model } from "../model.js"
27-
import { PluginHooks } from "../plugin/hooks.js"
2827
import { State } from "../state.js"
2928
import { Token } from "../util/token.js"
3029
import type { SessionContext } from "./context.js"
@@ -253,7 +252,6 @@ export const layer = Layer.effect(
253252
const models = yield* SessionRunnerModel.Service
254253
const db = (yield* Database.Service).db
255254
const requests = yield* SessionModelRequest.Service
256-
const hooks = yield* PluginHooks.Service
257255

258256
const scope = yield* Scope.Scope
259257
const precomputed = new Map<SessionContext.Loaded["session"]["id"], Precomputed>()
@@ -469,13 +467,11 @@ export const layer = Layer.effect(
469467
const headings = SUMMARY_TEMPLATE.split("\n").filter((line) => line.startsWith("##"))
470468
const filled = (text: string) => text.split("\n").some((line) => headings.includes(line.trim()))
471469
const prepared = yield* prepare(context, messages, budget)
472-
// The characters in a conversation overstate its size when the provider drops old reasoning, so the last
473-
// response's count sizes the unchanged request instead. A compaction hook can change the request, and an
474-
// overflow just proved the count low, so both keep the character estimate.
475-
const measured =
476-
trigger.reason === "overflow" || (yield* hooks.has("session", "compaction", context.model.ref.providerID))
477-
? undefined
478-
: measuredPrompt({ ...context, messages })
470+
// Size the unchanged request with the number that decided to compact: the last response's count plus an
471+
// estimate of what came after it. Characters overstate a conversation whose old reasoning the provider drops.
472+
// A hook can still make this request larger than that count; the provider then rejects it and it is resent as
473+
// text. An overflow already proved the count low, so it keeps the character estimate.
474+
const measured = trigger.reason === "overflow" ? undefined : measuredPrompt({ ...context, messages })
479475

480476
// Hooks saw the request without the summary prompt, so it is appended here. A reply that ignores the
481477
// template gets one reminder before it counts as a failure.
@@ -512,7 +508,10 @@ export const layer = Layer.effect(
512508
// which still reuses the cache for that shorter prefix, and as text otherwise. Dropping exchanges to make room
513509
// for the tail would leave them in neither the summary nor the tail. Where 15% of the window is under
514510
// `RESERVE_MIN`, the ceiling equals `summaryRoom`, so automatic compaction always takes this fallback.
515-
const fit = trigger.reason === "overflow" ? budget : Math.max(budget, summaryRoom(context.model.limit))
511+
const fit =
512+
trigger.reason === "overflow"
513+
? budget
514+
: Math.max(budget, summaryRoom(context.model.limit, prepared.request.generation?.maxTokens))
516515
const result = yield* deliver(
517516
trigger,
518517
prepared,
@@ -942,16 +941,7 @@ export const layer = Layer.effect(
942941
export const node = makeLocationNode({
943942
service: Service,
944943
layer,
945-
deps: [
946-
Agent.node,
947-
Bus.node,
948-
Database.node,
949-
llmClient,
950-
Model.node,
951-
PluginHooks.node,
952-
SessionModelRequest.node,
953-
SessionRunnerModel.node,
954-
],
944+
deps: [Agent.node, Bus.node, Database.node, llmClient, Model.node, SessionModelRequest.node, SessionRunnerModel.node],
955945
})
956946

957947
/**
@@ -1150,9 +1140,12 @@ const calculateCeiling = (limit: SessionContext.Loaded["model"]["limit"], buffer
11501140
return window - Math.max(Math.floor(window * RESERVE_SHARE), window >= 2 * RESERVE_MIN ? RESERVE_MIN : 0)
11511141
}
11521142

1153-
/** The largest summary request that leaves `RESERVE_MIN` of the window for the summary. Zero for an unknown window. */
1154-
const summaryRoom = (limit: SessionContext.Loaded["model"]["limit"]) =>
1155-
Math.max(0, (limit.input || limit.context) - RESERVE_MIN)
1143+
/**
1144+
* The largest summary request that leaves `RESERVE_MIN` of the window for the summary, or less when the request
1145+
* cannot generate that much. Zero for an unknown window.
1146+
*/
1147+
const summaryRoom = (limit: SessionContext.Loaded["model"]["limit"], output = RESERVE_MIN) =>
1148+
Math.max(0, (limit.input || limit.context) - Math.min(RESERVE_MIN, output))
11561149

11571150
/**
11581151
* Another provider's count may describe a native window this model cannot replay. History then holds the original

‎packages/core/test/session-compaction.test.ts‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1108,20 +1108,53 @@ it.effect("summary measured recovery shrinks from the rejected measured size", (
11081108
}),
11091109
)
11101110

1111-
it.effect("summary sizes by characters when a compaction hook can change the request", () =>
1111+
it.effect("summary cache prefix survives hooks that shape every request alike", () =>
11121112
Effect.gen(function* () {
11131113
const fixture = yield* summaryFixture([
11141114
textUser("Earlier question"),
11151115
oldReasoning(),
11161116
textUser("Latest question"),
11171117
measured("Latest answer", summaryInput),
11181118
])
1119+
// Built-in tool plugins register one hook for both normal and compaction requests.
11191120
const hooks = yield* PluginHooks.Service
1120-
yield* hooks.register("session", "compaction", () => Effect.void)
1121+
const describe = (event: { tools: Record<string, { description: string }> }) =>
1122+
Effect.sync(() => {
1123+
if (event.tools.read) event.tools.read.description = "Read files from the project"
1124+
})
1125+
yield* hooks.register("session", "context", describe)
1126+
yield* hooks.register("session", "compaction", describe)
11211127
expect(yield* fixture.run()).toEqual({ status: "completed" })
11221128
expect(requests).toHaveLength(1)
1123-
// The old reasoning counts at full length, so the request no longer fits and is flattened.
1124-
expect(requests[0].tools).toEqual([])
1129+
expect(requests[0].tools.map((tool) => tool.description)).toEqual(["Read files from the project"])
1130+
}),
1131+
)
1132+
1133+
it.effect("summary resends as text when a hook makes the request larger than the measured count", () =>
1134+
Effect.gen(function* () {
1135+
const fixture = yield* summaryFixture([
1136+
textUser("Earlier question"),
1137+
oldReasoning(),
1138+
textUser("Latest question"),
1139+
measured("Latest answer", summaryInput),
1140+
])
1141+
const hooks = yield* PluginHooks.Service
1142+
yield* hooks.register("session", "compaction", (event) =>
1143+
Effect.sync(() => {
1144+
event.system.push({ type: "text", text: "Plugin guidance" })
1145+
}),
1146+
)
1147+
// The provider rejects the unchanged request, which the measured count said would fit.
1148+
reject = (request) => request.tools.length > 0
1149+
yield* Effect.addFinalizer(() =>
1150+
Effect.sync(() => {
1151+
reject = undefined
1152+
}),
1153+
)
1154+
expect(yield* fixture.run()).toEqual({ status: "completed" })
1155+
expect(requests).toHaveLength(2)
1156+
expect(requests[0].tools).toEqual(fixture.context.tools.definitions)
1157+
expect(requests[1].tools).toEqual([])
11251158
}),
11261159
)
11271160

0 commit comments

Comments
 (0)