Skip to content

Commit 7b4691c

Browse files
anandgupta42claude
andcommitted
fix(drivers): read byte-bounded Mongo cursors in 1,000-document batches
A byte budget made the cursor fetch one document per round trip, so a 127k-document materialize took 6-11 minutes. Reply batches stay bounded (server replies are at most 16 MiB) and the byte budget still stops at the first document past the cap. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent ed66150 commit 7b4691c

2 files changed

Lines changed: 9 additions & 3 deletions

File tree

‎packages/drivers/src/mongodb.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,9 @@ function extractFields(docs: Record<string, unknown>[]): Map<string, Set<string>
219219
return fieldTypes
220220
}
221221

222+
/** Documents per reply batch on byte-bounded reads. */
223+
export const BOUNDED_READ_BATCH = 1000
224+
222225
/** Server error code for an operation stopped by `maxTimeMS`. */
223226
const MONGO_MAX_TIME_EXPIRED = 50
224227

@@ -376,7 +379,10 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
376379
const consume = createResultByteBudget(maxResultBytes)
377380
const docs: Record<string, unknown>[] = []
378381
try {
379-
for await (const doc of cursor.batchSize(1)) {
382+
// One document per round trip made a 127k-document read take ~10 minutes. A reply
383+
// batch is still bounded (and a server reply never exceeds 16 MiB), and the byte
384+
// budget stops the read at the first document past the cap.
385+
for await (const doc of cursor.batchSize(BOUNDED_READ_BATCH)) {
380386
const serialized = Object.fromEntries(
381387
Object.entries(doc).map(([key, value]) => [key, serializeValue(value)]),
382388
)

‎packages/drivers/test/source-byte-limit.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { expect, mock, test } from "bun:test"
22
import { createRequire } from "node:module"
33
import { dirname, join } from "node:path"
44
import { connect as connectPostgres } from "../src/postgres"
5-
import { connect as connectMongo } from "../src/mongodb"
5+
import { connect as connectMongo, BOUNDED_READ_BATCH } from "../src/mongodb"
66

77
// Use pg's real row parser and accumulation behavior while replacing only transport/pooling.
88
// Import its implementation directly so other tests' pg module mocks cannot replace it.
@@ -147,7 +147,7 @@ test.each(["find", "aggregate"])("Mongo %s closes the cursor before fetching pas
147147
).rejects.toMatchObject({ code: "RESULT_BYTE_LIMIT" })
148148
expect(db.consumed()).toBe(2)
149149
expect(db.closed()).toBe(true)
150-
expect(db.batchSize()).toBe(1)
150+
expect(db.batchSize()).toBe(BOUNDED_READ_BATCH)
151151
} finally {
152152
await db.connector.close()
153153
}

0 commit comments

Comments
 (0)