Skip to content

Commit db33839

Browse files
hkaltaiclaude
andcommitted
fix(opencode): dbt_lineage answers for the session's own dbt project; stricter lineage validation
- A session inside another dbt project than a saved dbt.json's (or than the server's launch folder) answers from its own project: the adapter is skipped with a reason, and the default manifest is that project's. - isModelLineage checks every declared field (focus.column, materialization, parents, dataType, isDocumentedOnly, stats values). - column reach: a backtick or bracket doubled inside a quoted column name read only outside the select list is a possible read, not dropped. - Test the 300-model cut in column reach. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 0197654 commit db33839

5 files changed

Lines changed: 153 additions & 11 deletions

File tree

‎packages/opencode/src/altimate/native/dbt/column-reach.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -159,9 +159,13 @@ export function selectsStar(sql: string): boolean {
159159
function namesColumn(sql: string, column: string): boolean {
160160
const code = stripSql(sql)
161161
const name = escapeRegExp(column)
162-
// Quoted, a quote in the name is written doubled: a"b is "a""b".
163-
const quoted = escapeRegExp(column.replaceAll('"', '""'))
164-
const ident = new RegExp(`(^|[^A-Za-z0-9_$])("${quoted}"|${name})(?![A-Za-z0-9_$])`, "gi")
162+
// Quoted, the closing delimiter in the name is written doubled: "a""b", `a``b`, [a]]b].
163+
const quoted = [
164+
`"${column.replaceAll('"', '""')}"`,
165+
`\`${column.replaceAll("`", "``")}\``,
166+
`[${column.replaceAll("]", "]]")}]`,
167+
].map(escapeRegExp)
168+
const ident = new RegExp(`(^|[^A-Za-z0-9_$])(${[...quoted, name].join("|")})(?![A-Za-z0-9_$])`, "gi")
165169
for (const match of code.matchAll(ident)) {
166170
const before = code.slice(0, match.index! + match[1]!.length)
167171
if (/\bas\s*`?$/i.test(before)) continue

‎packages/opencode/src/altimate/tools/dbt-lineage.ts‎

Lines changed: 64 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { Dispatcher } from "../native"
66
import type { DbtLineageResult } from "../native/types"
77
import { loadRawManifest } from "../native/dbt/helpers"
88
import { lineageGraph } from "../native/dbt/graph"
9+
import { projectRoot } from "../native/connections/registry"
910

1011
/** The structured result of dbt-integration's `getModelLineage` (version 1), relayed as is. */
1112
export type ModelLineage = {
@@ -59,7 +60,10 @@ export const DbtLineageTool = Tool.define("dbt_lineage", {
5960
// A call without write permission (MCP, `tool run`) and a plan-mode session's own calls are read-only;
6061
// a subagent plan mode starts runs under its own agent's permissions.
6162
const readOnly = ctx.extra?.forceReadOnly === true || ctx.agent === "plan"
62-
let viaAdapter = await fromAdapter(args.model, args.column, ctx.abort, readOnly)
63+
// Read once per call, and only when needed: it reads the dbt config.
64+
let project: ReturnType<typeof sessionProject> | undefined
65+
const session = () => (project ??= sessionProject())
66+
let viaAdapter = await fromAdapter(args.model, args.column, ctx.abort, readOnly, session)
6367
// The adapter answers for the project it found; a manifest named outside that project's folder
6468
// wins, so its graph and its lineage are never mixed with this one's.
6569
if (viaAdapter.result && args.manifest_path) {
@@ -70,7 +74,7 @@ export const DbtLineageTool = Tool.define("dbt_lineage", {
7074
const result = viaAdapter.result
7175
const what = result.focus.column ? `${result.focus.name}.${result.focus.column}` : result.focus.name
7276
// A lineage viewer also wants the manifest's graph: exposures and tests sit there, not here.
73-
const graph = args.include_graph ? manifestGraph(args.manifest_path ?? (await defaultManifestPath()), result.focus.uniqueId) : undefined
77+
const graph = args.include_graph ? manifestGraph(args.manifest_path ?? (await defaultManifestPath(session)), result.focus.uniqueId) : undefined
7478
return {
7579
title: `dbt Lineage: ${what} [${result.confidence}]`,
7680
metadata: {
@@ -85,7 +89,7 @@ export const DbtLineageTool = Tool.define("dbt_lineage", {
8589
}
8690
}
8791
const notUsed = viaAdapter.reason ? `dbt integration not used: ${viaAdapter.reason}` : undefined
88-
const manifestPath = args.manifest_path ?? (await defaultManifestPath())
92+
const manifestPath = args.manifest_path ?? (await defaultManifestPath(session))
8993
ctx.abort.throwIfAborted()
9094
try {
9195
const result = await Dispatcher.call("dbt.lineage", {
@@ -203,13 +207,15 @@ function manifestGraph(manifestPath: string, uniqueId: string): { graph?: Return
203207
/**
204208
* The lineage from the project's dbt adapter, used only where writes are allowed: starting it runs
205209
* dbt deps and writes target/, and it compiles and reads column metadata. A read-only call never
206-
* starts or uses it. `reason` says why it wasn't used.
210+
* starts or uses it, nor does a session in another dbt project than the adapter's. `reason` says why
211+
* it wasn't used.
207212
*/
208213
async function fromAdapter(
209214
model: string,
210215
column: string | undefined,
211216
signal: AbortSignal,
212217
readOnly: boolean,
218+
session: () => ReturnType<typeof sessionProject>,
213219
): Promise<{ result?: ModelLineage; reason?: string }> {
214220
try {
215221
const { getDbtAdapter, getDbtProjectRoot } = await import("../native/connections/register")
@@ -219,6 +225,10 @@ async function fromAdapter(
219225
// target/, and compiling renders Jinja, which can run SQL.
220226
// It answers from the manifest as main does, with no note: there's nothing for the user to fix.
221227
if (readOnly) return {}
228+
const project = await session()
229+
if (project?.other && project.root) {
230+
return { reason: `the dbt adapter is set up for ${project.root}, not ${project.other}` }
231+
}
222232
// Starting the adapter can take a while (dbt deps, a parse): a cancel doesn't wait for it, and
223233
// past the deadline the manifest answers. The start goes on, for the next call.
224234
let stop = () => {}
@@ -278,18 +288,29 @@ export function isModelLineage(value: unknown): value is ModelLineage {
278288
typeof v.confidence === "string" &&
279289
typeof v.stats === "object" &&
280290
v.stats !== null &&
291+
!Array.isArray(v.stats) &&
292+
Object.values(v.stats).every((n) => typeof n === "number") &&
281293
typeof v.focus?.uniqueId === "string" &&
282294
typeof v.focus?.name === "string" &&
295+
optional(v.focus.column, "string") &&
283296
Array.isArray(v.nodes) &&
284297
v.nodes.every(
285298
(n) =>
286299
typeof n?.uniqueId === "string" &&
287300
typeof n.name === "string" &&
288301
typeof n.type === "string" &&
302+
optional(n.materialization, "string") &&
289303
(n.direction === "focus" || n.direction === "upstream" || n.direction === "downstream") &&
290304
typeof n.depth === "number" &&
305+
Array.isArray(n.parents) &&
306+
n.parents.every((p) => typeof p === "string") &&
291307
Array.isArray(n.columns) &&
292-
n.columns.every((c) => typeof c?.name === "string"),
308+
n.columns.every(
309+
(c) =>
310+
typeof c?.name === "string" &&
311+
optional(c.dataType, "string") &&
312+
(c.isDocumentedOnly === undefined || c.isDocumentedOnly === true),
313+
),
293314
) &&
294315
Array.isArray(v.columnLineage) &&
295316
v.columnLineage.every(
@@ -306,15 +327,50 @@ export function isModelLineage(value: unknown): value is ModelLineage {
306327
)
307328
}
308329

330+
/** An optional field: absent, or of the type. */
331+
function optional(value: unknown, type: "string"): boolean {
332+
return value === undefined || typeof value === type
333+
}
334+
309335
/** A [node id, column] pair, as the lineage's edges name each end. */
310336
function isPair(value: unknown): value is [string, string] {
311337
return Array.isArray(value) && value.length === 2 && value.every((part) => typeof part === "string")
312338
}
313339

314-
/** target/manifest.json in the project the dbt adapter finds, which may be above the working directory. */
315-
async function defaultManifestPath(): Promise<string> {
340+
/**
341+
* target/manifest.json in the project the dbt adapter finds, which may be above the session's
342+
* directory, or in the session's own project when that is another one.
343+
*/
344+
async function defaultManifestPath(session: () => ReturnType<typeof sessionProject>): Promise<string> {
345+
const project = await session()
346+
const root = project ? (project.other ?? project.root) : await (await import("../native/connections/register")).getDbtProjectRoot()
347+
return path.join(root ?? "", "target", "manifest.json")
348+
}
349+
350+
/**
351+
* The adapter's project root and, when the session's directory is in another dbt project, that one
352+
* (`other`, both links resolved): a dbt.json saved for one project, or a server started in one,
353+
* doesn't answer for a session in another. Undefined, the dbt config not read, when the session's
354+
* directory is in no dbt project: the adapter's own discovery (a dbt.json, or a walk up from the
355+
* process's working directory) decides then.
356+
*/
357+
async function sessionProject(): Promise<{ root?: string; other?: string } | undefined> {
358+
const { findProjectRoot } = await import("../../../../dbt-tools/src/config")
359+
const found = findProjectRoot(projectRoot())
360+
if (!found) return undefined
316361
const { getDbtProjectRoot } = await import("../native/connections/register")
317-
return path.join((await getDbtProjectRoot()) ?? "", "target", "manifest.json")
362+
const root = await getDbtProjectRoot()
363+
const real = (p: string) => {
364+
try {
365+
return fs.realpathSync(p)
366+
} catch {
367+
return path.resolve(p)
368+
}
369+
}
370+
const here = real(found)
371+
// path.relative ignores case on win32.
372+
if (!root) return { other: here }
373+
return path.relative(real(root), here) ? { root: real(root), other: here } : { root }
318374
}
319375

320376
const MAX_TEXT_EDGES = 200

‎packages/opencode/test/altimate/column-reach.test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,44 @@ describe("columnReach where the engine's schema falls short", () => {
381381
const ids = [...reach.models, ...reach.uncertain].map((m) => m.unique_id)
382382
expect(ids).toContain("model.shop.rpt_quoted")
383383
})
384+
385+
test("stops at 300 traced models and says so", () => {
386+
const value = manifest()
387+
for (let i = 0; i < 300; i++) {
388+
value.nodes[`model.shop.copy_${i}`] = model(`copy_${i}`, ["model.shop.fct_orders"], "select revenue from fct_orders", [
389+
"revenue",
390+
])
391+
}
392+
const spy = spyOn(core, "columnLineage").mockImplementation((() => ({
393+
column_lineage: [{ source: '"shop"."main"."fct_orders"."revenue"', target: "revenue", lineage_type: "direct" }],
394+
column_dict: {},
395+
})) as any)
396+
try {
397+
const reach = columnReach(manifestFile(value), "fct_orders", "revenue")
398+
expect(reach.truncated).toBe(true)
399+
expect(reach.models.length + reach.uncertain.length + reach.untraced.length).toBe(300)
400+
} finally {
401+
spy.mockRestore()
402+
}
403+
})
404+
405+
test("a column read only in WHERE, its backtick or bracket in the name doubled, is uncertain", () => {
406+
for (const [spelling, quoted, dialect] of [
407+
["a`b", "`a``b`", "bigquery"],
408+
["a]b", "[a]]b]", "tsql"],
409+
]) {
410+
const value = manifest()
411+
value.nodes["model.shop.fct_orders"].columns = { [spelling!]: column(spelling!) }
412+
value.nodes["model.shop.rpt_filtered"] = model(
413+
"rpt_filtered",
414+
["model.shop.fct_orders"],
415+
`select order_id from fct_orders where ${quoted} > 0`,
416+
["order_id"],
417+
)
418+
const reach = columnReach(manifestFile(value), "fct_orders", spelling!, dialect)
419+
expect(reach.uncertain.map((m) => m.unique_id)).toContain("model.shop.rpt_filtered")
420+
}
421+
})
384422
})
385423

386424
describe("selectsStar", () => {

‎packages/opencode/test/altimate/dbt-lineage-adapter.test.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,14 @@ describe("dbt_lineage's wait for the dbt adapter", () => {
300300
{ ...TRACE, columnLineage: [{ ...edge, target: [null, null] }] },
301301
{ ...TRACE, columnLineage: [{ ...edge, viewsCode: [1] }] },
302302
{ ...TRACE, errors: [null] },
303+
{ ...TRACE, focus: { ...TRACE.focus, column: 1 } },
304+
{ ...TRACE, nodes: [{ ...node, materialization: 1 }] },
305+
{ ...TRACE, nodes: [{ ...node, parents: undefined }] },
306+
{ ...TRACE, nodes: [{ ...node, parents: [1] }] },
307+
{ ...TRACE, nodes: [{ ...node, columns: [{ name: "id", dataType: 1 }] }] },
308+
{ ...TRACE, nodes: [{ ...node, columns: [{ name: "id", isDocumentedOnly: "yes" }] }] },
309+
{ ...TRACE, stats: [] },
310+
{ ...TRACE, stats: { nodes: "3" } },
303311
]
304312
for (const unreadable of [{ version: 1, nodes: "?" }, partial, { ...TRACE, version: 2 }, ...nested]) {
305313
resetDbtAdapter()

‎packages/opencode/test/altimate/dbt-lineage-structured.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
// through `dbtAdapterInternals`, so these tests need no Python or dbt.
66
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, spyOn, test } from "bun:test"
77
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"
8+
import { realpathSync } from "node:fs"
9+
import { Instance } from "@/project/instance"
10+
import { disposeInstance } from "@/effect/instance-registry"
811
import os from "node:os"
912
import path from "node:path"
1013
import { Effect } from "effect"
@@ -341,6 +344,39 @@ describe("dbt_lineage without dbt-integration's lineage", () => {
341344
}
342345
})
343346

347+
test("a session in another dbt project than a saved dbt.json's answers from its own project", async () => {
348+
useConfig(false)
349+
let created = 0
350+
dbtAdapterInternals.create = async () => {
351+
created++
352+
return fakeAdapter(true, async () => TRACE)
353+
}
354+
const here = await mkdtemp(path.join(os.tmpdir(), "dbt-lineage-here-"))
355+
await writeFile(path.join(here, "dbt_project.yml"), "name: other\n")
356+
const inSession = (dir: string, context: Tool.Context) =>
357+
Instance.provide({ directory: dir, fn: async () => (await initTool(DbtLineageTool)).execute({ model: "customers" }, context) })
358+
const call = manifestOnly()
359+
try {
360+
const writable = await inSession(here, ctx)
361+
const readOnly = await inSession(here, { ...ctx, extra: { ...ctx.extra, forceReadOnly: true } })
362+
const ownManifest = { manifest_path: path.join(realpathSync(here), "target", "manifest.json") }
363+
expect(call.mock.calls.map((c) => c[1])).toMatchObject([ownManifest, ownManifest])
364+
expect(writable.output).toContain(`dbt integration not used: the dbt adapter is set up for ${realpathSync(project)}`)
365+
expect(readOnly.output).not.toContain("dbt integration not used")
366+
expect(created).toBe(0)
367+
// A subfolder of the saved project is that project: the adapter answers.
368+
await mkdir(path.join(project, "models"), { recursive: true })
369+
const sub = await inSession(path.join(project, "models"), ctx)
370+
expect(sub.output).not.toContain("dbt integration not used")
371+
expect(created).toBe(1)
372+
} finally {
373+
call.mockRestore()
374+
await disposeInstance(here)
375+
await disposeInstance(path.join(project, "models"))
376+
await rm(here, { recursive: true, force: true })
377+
}
378+
})
379+
344380
test("falls back with the reason when a dbt.json's project root isn't a path", async () => {
345381
dbtAdapterInternals.readConfig = async () => ({ projectRoot: 123 })
346382
const call = manifestOnly()

0 commit comments

Comments
 (0)