Skip to content

Commit 5b67069

Browse files
hkaltaiclaude
andcommitted
fix: a manifest is placed where the system opens it, through Filesystem.realLocation; an empty path is no model's; NATURAL JOIN gaps up to 10,000 characters
- links resolve before `..`, so `<project>/link/../x` lands where the link points; the duplicate resolver is gone, and a link loop cannot be placed - a request that is empty once normalized never matches a model without a file path - the window per natural grows to 10,000 characters, still bounded with the 100-word cap - tests for count_big(*)/count_if(*), a backticked output alias, link/.. and the empty path Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 299bdf0 commit 5b67069

5 files changed

Lines changed: 36 additions & 30 deletions

File tree

‎packages/opencode/src/altimate/native/dbt/column-reach.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -171,9 +171,10 @@ const GAP = String.raw`(?:\s|(?:--|#|\/\/)[^\r\n]*[\r\n]|\/\*(?:[^*]|\*(?!\/))*\
171171
// The join's modifiers in any dialect: LEFT/RIGHT/FULL/INNER/CROSS, OUTER, Spark's SEMI/ANTI, Snowflake's DIRECTED.
172172
const NATURAL = new RegExp(String.raw`^natural${GAP}(?:(?:left|right|full|inner|cross|outer|semi|anti|directed)${GAP}){0,4}join\b`, "i")
173173
// How far past a `natural` the rest of the join is looked for, and how many are looked at: past that
174-
// many the SQL may well have one, which only adds doubt. The work stays bounded on any input.
175-
// Limit: comments longer than the window between the keywords hide the join.
176-
const NATURAL_WINDOW = 1000
174+
// many the SQL may well have one, which only adds doubt. An unclosed `/*` makes each check read to the
175+
// window's end, so the window is what keeps crafted input cheap.
176+
// Limit: more than 10,000 characters of comments between the keywords hide the join.
177+
const NATURAL_WINDOW = 10_000
177178
const MAX_NATURALS = 100
178179
export function joinsNaturally(sql: string): boolean {
179180
let seen = 0

‎packages/opencode/src/altimate/native/dbt/lineage.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult {
4747
const filePath = (p: unknown) => String(p ?? "").replace(/\\/g, "/").replace(/^(?:\.\/)+/, "")
4848
const byPath = () =>
4949
Object.values<any>(nodes).find(
50-
(node) => ["model", "snapshot"].includes(node?.resource_type) && filePath(node.original_file_path) === filePath(params.model),
50+
(node) => !!filePath(params.model) && ["model", "snapshot"].includes(node?.resource_type) && filePath(node.original_file_path) === filePath(params.model),
5151
)
5252
const modelNode =
5353
findModel(nodes, params.model) ??

‎packages/opencode/src/altimate/tools/dbt-lineage.ts‎

Lines changed: 9 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { Dispatcher } from "../native"
66
import type { DbtLineageResult } from "../native/types"
77
import { loadRawManifest } from "../native/dbt/helpers"
88
import { lineageGraph } from "../native/dbt/graph"
9+
import { Filesystem } from "../../util/filesystem"
910

1011
/** The structured result of dbt-integration's `getModelLineage` (version 1), relayed as is. */
1112
export type ModelLineage = {
@@ -130,35 +131,21 @@ export const DbtLineageTool = Tool.define("dbt_lineage", {
130131
* by its path, its nearest existing folder's links resolved, so the project's own default path keeps
131132
* the adapter's answer; one with no dbt project above it at all (a default path resolved against
132133
* another folder) names no other project either. A link to a manifest not written yet is placed where
133-
* it points.
134+
* it points; a link loop cannot be placed.
134135
*/
135136
async function manifestOutsideAdapterProject(manifestPath: string): Promise<string | undefined> {
136137
const { getDbtProjectRoot } = await import("../native/connections/register")
137138
const root = await getDbtProjectRoot()
138139
if (!root) return "the dbt adapter has no project root"
139-
// The path with its links resolved, the parts not written yet appended; a link to a missing file is
140-
// followed to where it points. Undefined past 64 links (a loop).
140+
// Where the system would open the path: links resolved segment by segment before `..` (so
141+
// `<project>/link/../x` lands where the link points), parts not written yet kept, a link to a missing
142+
// file followed. A relative path is joined to the working folder as written, never normalized first.
141143
const real = (p: string) => {
142-
const rest: string[] = []
143-
let at = path.resolve(p)
144-
for (let step = 0; step < 64; step++) {
145-
try {
146-
return path.join(fs.realpathSync(at), ...rest)
147-
} catch {}
148-
let link: string | undefined
149-
try {
150-
if (fs.lstatSync(at).isSymbolicLink()) link = fs.readlinkSync(at)
151-
} catch {}
152-
if (link !== undefined) {
153-
// From the link's real folder, as the system resolves it: `..` climbs from there.
154-
at = path.resolve(fs.realpathSync(path.dirname(at)), link)
155-
continue
156-
}
157-
if (path.dirname(at) === at) return undefined
158-
rest.unshift(path.basename(at))
159-
at = path.dirname(at)
144+
try {
145+
return Filesystem.realLocation(path.isAbsolute(p) ? p : `${process.cwd()}${path.sep}${p}`)
146+
} catch {
147+
return undefined
160148
}
161-
return undefined
162149
}
163150
const projectDir = real(root)
164151
const manifest = real(manifestPath)

‎packages/opencode/test/altimate/column-reach.test.ts‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { afterEach, describe, expect, spyOn, test } from "bun:test"
22
import * as core from "@altimateai/altimate-core"
3-
import { mkdtempSync, rmSync, writeFileSync } from "node:fs"
3+
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
44
import { tmpdir } from "node:os"
55
import { join } from "node:path"
66
import { columnReach, exposuresOn, parentsNamed, joinsNaturally, selectsStar } from "../../src/altimate/native/dbt/column-reach"
@@ -74,9 +74,14 @@ describe("columnReach", () => {
7474

7575
test("an output alias or a comment naming the column is not a read", () => {
7676
const value = manifest()
77-
value.nodes["model.shop.rpt_counts"].compiled_code = "select count(order_id) as revenue /* revenue */ from fct_orders -- revenue"
78-
const reach = columnReach(manifestFile(value), "fct_orders", "revenue")
79-
expect(reach.uncertain.map((m) => m.name)).toEqual(["rpt_big_orders"])
77+
for (const sql of [
78+
"select count(order_id) as revenue /* revenue */ from fct_orders -- revenue",
79+
"select count(order_id) as `revenue` from fct_orders",
80+
]) {
81+
value.nodes["model.shop.rpt_counts"].compiled_code = sql
82+
const reach = columnReach(manifestFile(value), "fct_orders", "revenue")
83+
expect(reach.uncertain.map((m) => m.name)).toEqual(["rpt_big_orders"])
84+
}
8085
})
8186

8287
test("a downstream model without compiled SQL is reported untraced, and what reads it only may read the column", () => {
@@ -262,6 +267,8 @@ describe("selectsStar", () => {
262267
for (const sql of [
263268
"select revenue * 100 as cents from a",
264269
"select count(*) as n from a",
270+
"select count_big(*) as n from a",
271+
"select count_if(*) as n from a",
265272
"select a, b*c as d from t",
266273
"select '*' as s from a",
267274
"select (a + b) * c as d from t",
@@ -583,6 +590,10 @@ describe("dbtLineage", () => {
583590
// `./` and Windows separators spell the same path.
584591
expect(dbtLineage({ manifest_path: file, model: "./models/mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv")
585592
expect(dbtLineage({ manifest_path: file, model: "models\\mart_ltv.sql" }).model_unique_id).toBe("model.shop.mart_ltv")
593+
// An empty path is no model's, not the first one's without a file path.
594+
const pathless = JSON.parse(readFileSync(file, "utf8"))
595+
delete pathless.nodes["model.shop.fct_orders"].original_file_path
596+
expect(dbtLineage({ manifest_path: manifestFile(pathless), model: "./" }).confidence_factors).toContain("Model './' not found in manifest")
586597
const withSnapshot: any = manifest()
587598
withSnapshot.nodes["snapshot.shop.snap"] = { ...model("snap", ["model.shop.fct_orders"], "select order_id from fct_orders", ["order_id"]), resource_type: "snapshot", unique_id: "snapshot.shop.snap" }
588599
expect(dbtLineage({ manifest_path: manifestFile(withSnapshot), model: "snap" }).model_unique_id).toBe("snapshot.shop.snap")
@@ -635,6 +646,7 @@ describe("columnReach with a NATURAL JOIN", () => {
635646
"select a.id from a natural # mysql\njoin b",
636647
"select a.id from a natural // snowflake\njoin b",
637648
"select a.id from a natural -- old mac\rjoin b",
649+
"select a.id from a natural /* " + "x".repeat(5000) + " */ join b",
638650
// Many `natural` words in a string before the real join: each is checked in a bounded window.
639651
"select $$" + "natural /* ".repeat(4000) + "$$ as s from a natural join b",
640652
"select '/*' as s from a natural /* c */ join b",

‎packages/opencode/test/altimate/dbt-lineage-structured.test.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,12 @@ describe("dbt_lineage with the project's dbt adapter", () => {
221221
await symlink("../../otherproj/target/manifest.json", path.join(realOther, "proj", "target", "manifest.json"))
222222
const climbed = await tool.execute({ model: "customers", manifest_path: path.join(mine, "linked_target", "manifest.json") }, ctx)
223223
expect(climbed.metadata.structuredContent).toBeUndefined()
224+
// `..` after a link climbs from where the link points, as the system opens it: a manifest reached
225+
// through `<mine>/out/..` is the other project's, though the path reads as this one's.
226+
await mkdir(path.join(other, "deep"), { recursive: true })
227+
await symlink(path.join(other, "deep"), path.join(mine, "out"))
228+
const viaDots = await tool.execute({ model: "customers", manifest_path: `${mine}/out/../target/manifest.json` }, ctx)
229+
expect(viaDots.metadata.structuredContent).toBeUndefined()
224230
// A path through a file is not a folder: it is placed, not thrown on.
225231
const through = await tool.execute({ model: "customers", manifest_path: path.join(mine, "dbt_project.yml", "manifest.json") }, ctx)
226232
expect(through.metadata.structuredContent).toEqual(TRACE)

0 commit comments

Comments
 (0)