Skip to content

Commit 41b33fe

Browse files
fix(processor): coalesce streaming deltas to cut per-token pipeline load
1 parent fdb3778 commit 41b33fe

1 file changed

Lines changed: 36 additions & 2 deletions

File tree

‎packages/opencode/src/session/processor.ts‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,38 @@ const layer = Layer.effect(
9090
const plugin = yield* Plugin.Service
9191
const summary = yield* SessionSummary.Service
9292
const scope = yield* Scope.Scope
93+
// Streaming deltas are non-durable and fire per token; accumulate them per part field and
94+
// flush on a short cadence (or before a part's final update) to cut per-token pipeline work.
95+
const DELTA_FLUSH_MS = 40
96+
type DeltaInput = Parameters<typeof session.updatePartDelta>[0]
97+
const pendingDeltas = new Map<string, DeltaInput>()
98+
99+
const flushDelta = (key: string) =>
100+
Effect.suspend(() => {
101+
const buffer = pendingDeltas.get(key)
102+
if (!buffer) return Effect.void
103+
pendingDeltas.delete(key)
104+
return session.updatePartDelta(buffer)
105+
})
106+
107+
const flushPart = (partID: string) =>
108+
Effect.forEach(
109+
[...pendingDeltas.entries()],
110+
([key, buffer]) => (buffer.partID === partID ? flushDelta(key) : Effect.void),
111+
{ discard: true },
112+
)
113+
114+
const queueDelta = (input: DeltaInput) =>
115+
Effect.gen(function* () {
116+
const key = `${input.sessionID}:${input.messageID}:${input.partID}:${input.field}`
117+
const buffer = pendingDeltas.get(key)
118+
if (buffer) {
119+
pendingDeltas.set(key, { ...buffer, delta: buffer.delta + input.delta })
120+
return
121+
}
122+
pendingDeltas.set(key, { ...input })
123+
yield* Effect.sleep(`${DELTA_FLUSH_MS} millis`).pipe(Effect.andThen(flushDelta(key)), Effect.forkIn(scope))
124+
})
93125
const status = yield* SessionStatus.Service
94126
const image = yield* Image.Service
95127
const events = yield* EventV2Bridge.Service
@@ -296,7 +328,7 @@ const layer = Layer.effect(
296328
if (!(value.id in ctx.reasoningMap)) return
297329
ctx.reasoningMap[value.id].text += value.text
298330
if (value.providerMetadata) ctx.reasoningMap[value.id].metadata = value.providerMetadata
299-
yield* session.updatePartDelta({
331+
yield* queueDelta({
300332
sessionID: ctx.reasoningMap[value.id].sessionID,
301333
messageID: ctx.reasoningMap[value.id].messageID,
302334
partID: ctx.reasoningMap[value.id].id,
@@ -309,6 +341,7 @@ const layer = Layer.effect(
309341
if (value.providerMetadata && value.id in ctx.reasoningMap) {
310342
ctx.reasoningMap[value.id].metadata = value.providerMetadata
311343
}
344+
if (value.id in ctx.reasoningMap) yield* flushPart(ctx.reasoningMap[value.id].id)
312345
yield* finishReasoning(value.id)
313346
return
314347

@@ -526,7 +559,7 @@ const layer = Layer.effect(
526559
if (!ctx.currentText) return
527560
ctx.currentText.text += value.text
528561
if (value.providerMetadata) ctx.currentText.metadata = value.providerMetadata
529-
yield* session.updatePartDelta({
562+
yield* queueDelta({
530563
sessionID: ctx.currentText.sessionID,
531564
messageID: ctx.currentText.messageID,
532565
partID: ctx.currentText.id,
@@ -553,6 +586,7 @@ const layer = Layer.effect(
553586
ctx.currentText.time = { start: ctx.currentText.time?.start ?? end, end }
554587
}
555588
if (value.providerMetadata) ctx.currentText.metadata = value.providerMetadata
589+
yield* flushPart(ctx.currentText.id)
556590
yield* session.updatePart(ctx.currentText)
557591
ctx.currentText = undefined
558592
return

0 commit comments

Comments
 (0)