Skip to content

Commit deb2828

Browse files
anandgupta42claude
andcommitted
fix: [review] MongoDB value profile renders BSON types, dotted keys and MongoDB notation
- render UUID, ObjectId, Decimal128, Long, Timestamp, regex, Code and MinKey/MaxKey as readable text; values with no text form report "values not profiled (type)" instead of "all NULL" - carry key paths as segments so keys containing "." are read correctly - name nested fields in MongoDB dot notation (`items.sku`) and mark array membership in a "Nested fields" listing - revert `describeTable` to its top-level behaviour; nested fields are derived in `schema_inspect` from the single document sample used for the value profile - bound the sample with `maxTimeMS` and the tool call's abort signal - add real-BSON unit tests, a describeTable regression test, and a real-server e2e case Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 3b81de8 commit deb2828

10 files changed

Lines changed: 381 additions & 244 deletions

File tree

‎packages/drivers/src/mongodb.ts‎

Lines changed: 45 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -245,60 +245,6 @@ function extractFields(docs: Record<string, unknown>[]): Map<string, Set<string>
245245
return fieldTypes
246246
}
247247

248-
/** Deepest dotted path reported by `extractNestedPaths`, counting `field[]` as part of its segment. */
249-
const NESTED_PATH_MAX_DEPTH = 3
250-
/** Cap on nested paths reported per collection. */
251-
const NESTED_PATH_MAX_COUNT = 100
252-
/** Documents sampled to infer a collection schema. */
253-
const SAMPLE_DOCUMENTS = 100
254-
255-
export interface NestedPath {
256-
types: Set<string>
257-
/** Sampled documents containing the path. */
258-
docs: number
259-
}
260-
261-
function isPlainObject(value: unknown): value is Record<string, unknown> {
262-
if (typeof value !== "object" || value === null) return false
263-
const proto = Object.getPrototypeOf(value)
264-
return proto === Object.prototype || proto === null
265-
}
266-
267-
/**
268-
* Extract dotted paths below the top level (`a.b`, `items[].sku`) up to depth 3, with
269-
* the types seen and the number of documents containing each path. Arrays of objects
270-
* contribute `field[]` followed by the element's keys.
271-
*/
272-
export function extractNestedPaths(docs: Record<string, unknown>[]): Map<string, NestedPath> {
273-
const paths = new Map<string, NestedPath>()
274-
for (const doc of docs) {
275-
const seen = new Set<string>()
276-
const visit = (value: unknown, path: string, depth: number) => {
277-
if (depth >= NESTED_PATH_MAX_DEPTH) return
278-
if (isPlainObject(value)) {
279-
for (const [key, child] of Object.entries(value)) record(child, `${path}.${key}`, depth + 1)
280-
} else if (Array.isArray(value)) {
281-
for (const element of value) {
282-
if (!isPlainObject(element)) continue
283-
for (const [key, child] of Object.entries(element)) record(child, `${path}[].${key}`, depth + 1)
284-
}
285-
}
286-
}
287-
const record = (value: unknown, path: string, depth: number) => {
288-
const entry = paths.get(path) ?? { types: new Set<string>(), docs: 0 }
289-
entry.types.add(inferType(value))
290-
if (!seen.has(path)) {
291-
seen.add(path)
292-
entry.docs++
293-
}
294-
paths.set(path, entry)
295-
visit(value, path, depth)
296-
}
297-
for (const [key, value] of Object.entries(doc)) visit(value, key, 1)
298-
}
299-
return paths
300-
}
301-
302248
/** Server error code for an operation stopped by `maxTimeMS`. */
303249
const MONGO_MAX_TIME_EXPIRED = 50
304250

@@ -409,15 +355,36 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
409355
return JSON.stringify(val)
410356
}
411357

412-
async function sampleDocuments(schema: string, table: string, limit = SAMPLE_DOCUMENTS): Promise<Record<string, unknown>[]> {
413-
// altimate_change start — format the existing cursor chain
414-
return client
415-
.db(schema)
416-
.collection(table)
417-
.find({}, withMaxTime(resolveQueryTimeout(config).ms))
418-
.limit(limit)
419-
.toArray()
420-
// altimate_change end
358+
/**
359+
* Raw documents for value profiling. `maxTimeMS` is the smaller of the caller's cap and the
360+
* connection's query timeout; `signal` abandons the read and closes the cursor.
361+
*/
362+
async function sampleDocuments(
363+
schema: string,
364+
table: string,
365+
limit = 100,
366+
options: { signal?: AbortSignal; maxTimeMS?: number } = {},
367+
): Promise<Record<string, unknown>[]> {
368+
options.signal?.throwIfAborted()
369+
const configured = resolveQueryTimeout(config).ms
370+
const maxTimeMS =
371+
options.maxTimeMS === undefined ? configured : configured === undefined ? options.maxTimeMS : Math.min(configured, options.maxTimeMS)
372+
const cursor = client.db(schema).collection(table).find({}, withMaxTime(maxTimeMS)).limit(limit)
373+
const { signal } = options
374+
if (!signal) return cursor.toArray()
375+
let abort: (() => void) | undefined
376+
const aborted = new Promise<never>((_, reject) => {
377+
abort = () => {
378+
void Promise.resolve(cursor.close?.()).catch(() => {})
379+
reject(signal.reason)
380+
}
381+
signal.addEventListener("abort", abort, { once: true })
382+
})
383+
try {
384+
return await Promise.race([cursor.toArray(), aborted])
385+
} finally {
386+
signal.removeEventListener("abort", abort!)
387+
}
421388
}
422389

423390
const connector: Connector & {
@@ -941,8 +908,16 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
941908
sampleDocuments,
942909

943910
async describeTable(schema: string, table: string): Promise<SchemaColumn[]> {
911+
const db = client.db(schema)
912+
const coll = db.collection(table)
913+
944914
// Sample up to 100 documents to infer schema
945-
const docs = await sampleDocuments(schema, table)
915+
// altimate_change start — format the existing cursor chain
916+
const docs = await coll
917+
.find({}, withMaxTime(resolveQueryTimeout(config).ms))
918+
.limit(100)
919+
.toArray()
920+
// altimate_change end
946921

947922
if (docs.length === 0) {
948923
return []
@@ -957,15 +932,13 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
957932
}
958933
}
959934

960-
const joinTypes = (types: Iterable<string>) => {
961-
const nonNullTypes = Array.from(types).filter((t) => t !== "null")
962-
return nonNullTypes.length === 0 ? "null" : nonNullTypes.length === 1 ? nonNullTypes[0] : nonNullTypes.join(" | ")
963-
}
964-
965935
const columns: SchemaColumn[] = []
966936
for (const [name, types] of fieldTypes) {
967-
const hasNull = types.has("null")
968-
const dataType = joinTypes(types)
937+
const typeArr = Array.from(types)
938+
const hasNull = typeArr.includes("null")
939+
const nonNullTypes = typeArr.filter((t) => t !== "null")
940+
const dataType =
941+
nonNullTypes.length === 0 ? "null" : nonNullTypes.length === 1 ? nonNullTypes[0] : nonNullTypes.join(" | ")
969942

970943
// Field is nullable if it has null values OR is missing from some documents
971944
const presentIn = fieldPresence.get(name) ?? 0
@@ -978,18 +951,7 @@ export async function connect(config: ConnectionConfig): Promise<Connector> {
978951
})
979952
}
980953

981-
const nested = [...extractNestedPaths(docs)].sort(([a], [b]) => a.localeCompare(b)).slice(0, NESTED_PATH_MAX_COUNT)
982-
for (const [name, info] of nested) {
983-
const presence = info.docs / docs.length
984-
columns.push({
985-
name,
986-
data_type: joinTypes(info.types),
987-
nullable: info.types.has("null") || presence < 1,
988-
presence,
989-
})
990-
}
991-
992-
// Sort: _id first, then alphabetical (nested paths land under their parent)
954+
// Sort: _id first, then alphabetical
993955
columns.sort((a, b) => {
994956
if (a.name === "_id") return -1
995957
if (b.name === "_id") return 1

‎packages/drivers/src/types.ts‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@ export interface SchemaColumn {
1818
name: string
1919
data_type: string
2020
nullable: boolean
21-
/** Share (0-1) of sampled documents containing this field; set on MongoDB nested paths. */
22-
presence?: number
2321
}
2422

2523
export interface SchemaMetadata {
@@ -137,8 +135,13 @@ export interface Connector {
137135
listSchemas(): Promise<string[]>
138136
listTables(schema: string): Promise<Array<{ name: string; type: string }>>
139137
describeTable(schema: string, table: string): Promise<SchemaColumn[]>
140-
/** Document stores only: the raw documents `describeTable` infers its schema from. */
141-
sampleDocuments?(schema: string, table: string, limit?: number): Promise<Record<string, unknown>[]>
138+
/** Document stores only: raw documents, bounded by `maxTimeMS` and cancellable through `signal`. */
139+
sampleDocuments?(
140+
schema: string,
141+
table: string,
142+
limit?: number,
143+
options?: { signal?: AbortSignal; maxTimeMS?: number },
144+
): Promise<Record<string, unknown>[]>
142145
introspectSchema?(signal?: AbortSignal): Promise<SchemaMetadata>
143146
close(): Promise<void>
144147
}

‎packages/drivers/test/mongodb-unit.test.ts‎

Lines changed: 23 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { describe, expect, mock, test } from "bun:test"
22
import { createRequire } from "node:module"
3-
import { extractNestedPaths, normalizeMqlCommand } from "../src/mongodb"
3+
import { normalizeMqlCommand } from "../src/mongodb"
44
import { restoreModuleMocksAfterTests } from "./module-mock"
55

66
await restoreModuleMocksAfterTests("mongodb")
@@ -13,6 +13,7 @@ async function fakeMongo(
1313
metadata: { collections?: Record<string, string[]>; error?: Error } = {},
1414
) {
1515
const finds: unknown[] = []
16+
const findOptions: unknown[] = []
1617
const pipelines: Record<string, unknown>[][] = []
1718
const aggregateOptions: unknown[] = []
1819
const counts: { filter: unknown; options: unknown }[] = []
@@ -50,8 +51,9 @@ async function fakeMongo(
5051
collections[database].push(name)
5152
},
5253
collection: () => ({
53-
find: (filter: unknown) => {
54+
find: (filter: unknown, options: unknown) => {
5455
finds.push(filter)
56+
findOptions.push(options)
5557
return { limit: () => ({ toArray: async () => docs, async *[Symbol.asyncIterator]() { yield* docs } }) }
5658
},
5759
aggregate: (pipeline: Record<string, unknown>[], options: unknown) => {
@@ -81,7 +83,7 @@ async function fakeMongo(
8183
const connector = await connect({ type: "mongodb", database: "test" })
8284
await connector.connect()
8385
const query = (command: string, extra: object = {}) => JSON.stringify({ collection: "articles", command, ...extra })
84-
return { connector, query, collections, finds, pipelines, aggregateOptions, counts, distincts, collectionLists, databaseLists, estimates }
86+
return { connector, query, collections, finds, findOptions, pipelines, aggregateOptions, counts, distincts, collectionLists, databaseLists, estimates }
8587
}
8688

8789
describe("MongoDB read command shapes", () => {
@@ -461,46 +463,30 @@ describe("MongoDB native command filters", () => {
461463
})
462464
})
463465

464-
describe("MongoDB nested field paths", () => {
466+
describe("MongoDB nested documents", () => {
465467
const docs = [
466-
{ _id: 1, user: { name: "a", geo: { lat: 1.5, deep: { too: "far" } } }, items: [{ sku: "x", qty: 1 }, { sku: "y" }], tags: ["t"] },
467-
{ _id: 2, user: { name: "b" }, items: [{ sku: "z", qty: null }], tags: [] },
468-
{ _id: 3, user: null, items: [] },
468+
{ _id: 1, user: { name: "a", geo: { lat: 1.5 } }, items: [{ sku: "x" }], "a.b": 1 },
469+
{ _id: 2, user: null, items: [] },
469470
]
470471

471-
test("extracts dotted and array paths to depth 3 with types and per-document presence", () => {
472-
const paths = extractNestedPaths(docs)
473-
expect([...paths.keys()].sort()).toEqual([
474-
"items[].qty",
475-
"items[].sku",
476-
"user.geo",
477-
"user.geo.deep",
478-
"user.geo.lat",
479-
"user.name",
472+
test("describeTable stays top-level: nested objects are 'object', arrays 'array', no nested paths", async () => {
473+
const { connector } = await fakeMongo(docs)
474+
expect(await connector.describeTable("test", "articles")).toEqual([
475+
{ name: "_id", data_type: "int32", nullable: false },
476+
{ name: "a.b", data_type: "int32", nullable: true },
477+
{ name: "items", data_type: "array", nullable: false },
478+
{ name: "user", data_type: "object", nullable: true },
480479
])
481-
expect(paths.get("user.geo.deep")!.types).toEqual(new Set(["object"]))
482-
// sku appears twice in document 1 but the document is counted once
483-
expect(paths.get("items[].sku")!.docs).toBe(2)
484-
expect([...paths.get("items[].qty")!.types].sort()).toEqual(["int32", "null"])
485-
expect(paths.get("user.name")!.docs).toBe(2)
486480
})
487481

488-
test("describeTable keeps top-level columns and adds nested paths with a presence share", async () => {
489-
const { connector } = await fakeMongo(docs)
490-
const columns = await connector.describeTable("test", "articles")
491-
const byName = Object.fromEntries(columns.map((c) => [c.name, c]))
492-
expect(byName.user).toEqual({ name: "user", data_type: "object", nullable: true })
493-
expect(byName.items).toEqual({ name: "items", data_type: "array", nullable: false })
494-
expect(byName["user.name"]).toEqual({ name: "user.name", data_type: "string", nullable: true, presence: 2 / 3 })
495-
expect(byName["items[].sku"]).toMatchObject({ data_type: "string", presence: 2 / 3 })
496-
expect(byName["items[].qty"]).toMatchObject({ data_type: "int32", nullable: true })
497-
expect(columns[0]!.name).toBe("_id")
498-
const names = columns.map((c) => c.name)
499-
expect(names.slice(names.indexOf("user"))).toEqual(["user", "user.geo", "user.geo.deep", "user.geo.lat", "user.name"])
500-
})
501-
502-
test("sampleDocuments returns the sampled documents", async () => {
482+
test("sampleDocuments returns raw documents under the smaller of the caller and connection deadlines", async () => {
483+
const { connector, findOptions } = await fakeMongo(docs)
484+
expect(await connector.sampleDocuments!("test", "articles", 50, { maxTimeMS: 15_000 })).toEqual(docs)
485+
expect(findOptions).toEqual([{ maxTimeMS: 15_000 }])
486+
})
487+
488+
test("sampleDocuments honours an aborted signal", async () => {
503489
const { connector } = await fakeMongo(docs)
504-
expect(await connector.sampleDocuments!("test", "articles", 50)).toEqual(docs)
490+
await expect(connector.sampleDocuments!("test", "articles", 50, { signal: AbortSignal.abort(new Error("stop")) })).rejects.toThrow("stop")
505491
})
506492
})

‎packages/opencode/src/altimate/native/connections/register.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -823,7 +823,6 @@ register("schema.inspect", async (params: SchemaInspectParams): Promise<SchemaIn
823823
data_type: c.data_type,
824824
nullable: c.nullable,
825825
primary_key: false, // would need additional query for PK detection
826-
...(c.presence === undefined ? {} : { presence: c.presence }),
827826
})),
828827
}
829828
} catch (e) {

‎packages/opencode/src/altimate/native/types.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,6 @@ export interface SchemaColumn {
120120
nullable: boolean
121121
primary_key: boolean
122122
description?: string
123-
/** Share (0-1) of sampled documents containing this field (MongoDB nested paths). */
124-
presence?: number
125123
}
126124

127125
export interface SchemaInspectResult {

‎packages/opencode/src/altimate/tools/schema-inspect.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -181,10 +181,8 @@ function formatSchema(result: Partial<SchemaInspectResult>): string {
181181
lines.push("Column | Type | Nullable | PK")
182182
lines.push("-------|------|----------|---")
183183
for (const col of result.columns ?? []) {
184-
// MongoDB nested paths carry the share of sampled documents that contain them.
185-
const presence = col.presence !== undefined && col.presence < 1 ? ` (in ${Math.round(col.presence * 100)}% of docs)` : ""
186184
lines.push(
187-
`${col.name} | ${col.data_type ?? "unknown"}${presence} | ${col.nullable ? "YES" : "NO"} | ${col.primary_key ? "YES" : ""}`,
185+
`${col.name} | ${col.data_type ?? "unknown"} | ${col.nullable ? "YES" : "NO"} | ${col.primary_key ? "YES" : ""}`,
188186
)
189187
}
190188
return lines.join("\n")

0 commit comments

Comments
 (0)