|
5 | 5 | */ |
6 | 6 |
|
7 | 7 | import * as core from "@altimateai/altimate-core" |
8 | | -import type { DbtLineageParams, DbtLineageResult, DbtModelLineageParams, ModelLineage } from "../types" |
| 8 | +import type { DbtLineageParams, DbtLineageResult } from "../types" |
9 | 9 | import { loadRawManifest, findModel, getUniqueId, detectDialect, buildSchemaContext } from "./helpers" |
10 | 10 |
|
11 | 11 | /** |
@@ -73,236 +73,3 @@ export function dbtLineage(params: DbtLineageParams): DbtLineageResult { |
73 | 73 | confidence_factors: rawLineage.error ? [String(rawLineage.error)] : [], |
74 | 74 | } |
75 | 75 | } |
76 | | - |
77 | | -type LineageNode = ModelLineage["nodes"][number] |
78 | | -type LineageEdge = ModelLineage["columnLineage"][number] |
79 | | - |
80 | | -/** |
81 | | - * A model's lineage from the manifest alone, in dbt-integration's `getModelLineage` format |
82 | | - * (version 1): the model and its direct parents, with the column edges altimate-core reads from the |
83 | | - * compiled SQL in the manifest. Nothing is compiled and no warehouse is read. |
84 | | - */ |
85 | | -export function dbtModelLineage(params: DbtModelLineageParams): ModelLineage { |
86 | | - const startedAt = Date.now() |
87 | | - const raw = dbtLineage(params) |
88 | | - const errors = [...raw.confidence_factors] |
89 | | - let manifest: any = null |
90 | | - try { |
91 | | - manifest = loadRawManifest(params.manifest_path) |
92 | | - } catch { |
93 | | - // dbtLineage has already reported it. |
94 | | - } |
95 | | - const nodes: Record<string, any> = manifest?.nodes ?? {} |
96 | | - const sources: Record<string, any> = manifest?.sources ?? {} |
97 | | - const model = manifest ? findModel(nodes, params.model) : null |
98 | | - const dialect = params.dialect || (manifest ? detectDialect(manifest) : "") |
99 | | - const focusId: string = model?.unique_id ?? raw.model_unique_id ?? params.model |
100 | | - const focusName: string = model?.name ?? params.model |
101 | | - const sql: string = model?.compiled_code || model?.compiled_sql || "" |
102 | | - const parentIds: string[] = model ? [...new Set<string>(model.depends_on?.nodes ?? [])] : [] |
103 | | - |
104 | | - const result: LineageNode[] = [] |
105 | | - if (model) { |
106 | | - const inResult = new Set([focusId, ...parentIds]) |
107 | | - const parentsOf = (node: any, self: string) => |
108 | | - [...new Set<string>(node?.depends_on?.nodes ?? [])].filter((p) => p !== self && inResult.has(p)) |
109 | | - result.push({ |
110 | | - uniqueId: focusId, |
111 | | - name: focusName, |
112 | | - type: "model", |
113 | | - ...(model.config?.materialized ? { materialization: model.config.materialized } : {}), |
114 | | - direction: "focus", |
115 | | - depth: 0, |
116 | | - parents: parentIds, |
117 | | - ...(model.original_file_path ? { path: model.original_file_path } : {}), |
118 | | - ...outputColumns(sql, dialect, focusName, raw.raw_lineage, errors), |
119 | | - }) |
120 | | - for (const uid of parentIds) { |
121 | | - const node = nodes[uid] ?? sources[uid] |
122 | | - const type: string = node?.resource_type ?? uid.split(".")[0] |
123 | | - if (!node) errors.push(`${uid} is not in the manifest, so it has no columns`) |
124 | | - const columns = Object.entries(node?.columns ?? {}).map(([key, c]: [string, any]) => ({ |
125 | | - name: c?.name || key, |
126 | | - ...(c?.data_type || c?.type ? { dataType: c.data_type || c.type } : {}), |
127 | | - ...(c?.description ? { description: c.description } : {}), |
128 | | - })) |
129 | | - const materialization = type === "source" || type === "seed" ? type : node?.config?.materialized |
130 | | - result.push({ |
131 | | - uniqueId: uid, |
132 | | - name: |
133 | | - type === "source" && node |
134 | | - ? `${node.source_name}.${node.name}` |
135 | | - : (node?.name ?? uid.split(".").slice(2).join(".")), |
136 | | - type, |
137 | | - ...(materialization ? { materialization } : {}), |
138 | | - direction: "upstream", |
139 | | - depth: 1, |
140 | | - parents: parentsOf(node, uid), |
141 | | - ...(node?.original_file_path ? { path: node.original_file_path } : {}), |
142 | | - columns, |
143 | | - columnsFrom: columns.length > 0 ? "manifest" : "none", |
144 | | - }) |
145 | | - } |
146 | | - } |
147 | | - |
148 | | - const lineage = raw.raw_lineage as { column_lineage?: unknown; errors?: unknown } |
149 | | - for (const e of Array.isArray(lineage.errors) ? lineage.errors : []) { |
150 | | - errors.push(`${focusName}: ${typeof e === "string" ? e : (e?.message ?? JSON.stringify(e))}`) |
151 | | - } |
152 | | - const { byTable, shared } = tablesOf(parentIds, nodes, sources) |
153 | | - if (shared.size > 0) { |
154 | | - errors.push( |
155 | | - `${focusName} reads ${[...shared].join(", ")}, which share a table name in altimate-core's schema, so their column edges into it were left out`, |
156 | | - ) |
157 | | - } |
158 | | - const edges = lineageEdges( |
159 | | - Array.isArray(lineage.column_lineage) ? lineage.column_lineage : [], |
160 | | - byTable, |
161 | | - { uniqueId: focusId, name: focusName }, |
162 | | - errors, |
163 | | - shared, |
164 | | - ) |
165 | | - const wanted = params.column?.toLowerCase() |
166 | | - const columnLineage = wanted ? edges.filter((e) => e.target[1] === wanted) : edges |
167 | | - return { |
168 | | - version: 1, |
169 | | - project: manifest?.metadata?.project_name ?? model?.package_name ?? "", |
170 | | - dialect, |
171 | | - focus: { uniqueId: focusId, name: focusName, ...(params.column ? { column: params.column } : {}) }, |
172 | | - nodes: result, |
173 | | - columnLineage, |
174 | | - confidence: errors.length === 0 ? "high" : "low", |
175 | | - errors, |
176 | | - stats: { |
177 | | - nodes: result.length, |
178 | | - edges: columnLineage.length, |
179 | | - compiled: sql ? 1 : 0, |
180 | | - fromWarehouse: 0, |
181 | | - ms: Date.now() - startedAt, |
182 | | - }, |
183 | | - } |
184 | | -} |
185 | | - |
186 | | -/** |
187 | | - * The focus model's columns: its compiled SQL's output columns. A `select *` the SQL alone can't |
188 | | - * expand takes the output columns altimate-core traced with the parents' schema instead, and says so. |
189 | | - */ |
190 | | -function outputColumns(sql: string, dialect: string, name: string, lineage: Record<string, unknown>, errors: string[]) { |
191 | | - if (!sql) return { columns: [], columnsFrom: "none" } |
192 | | - const resolved = Object.keys((lineage.column_dict as Record<string, unknown> | undefined) ?? {}) |
193 | | - let names: string[] = [] |
194 | | - let failure: unknown |
195 | | - try { |
196 | | - names = core.extractOutputColumns(sql, dialect) |
197 | | - } catch (e) { |
198 | | - failure = e |
199 | | - } |
200 | | - const explicit = names.filter((c) => c !== "*" && !c.endsWith(".*")) |
201 | | - if (failure === undefined && explicit.length === names.length && names.length > 0) { |
202 | | - return { columns: names.map((c) => ({ name: c })), columnsFrom: "compiled-sql" } |
203 | | - } |
204 | | - const why = |
205 | | - failure !== undefined |
206 | | - ? `Could not read ${name}'s output columns (${failure instanceof Error ? failure.message : String(failure)})` |
207 | | - : names.length === 0 |
208 | | - ? `altimate-core read no output columns from ${name}'s compiled SQL` |
209 | | - : `${name}'s compiled SQL selects *` |
210 | | - if (resolved.length > 0) { |
211 | | - errors.push(`${why}, so its columns are only the ones traced to a parent`) |
212 | | - return { columns: resolved.map((c) => ({ name: c })), columnsFrom: "compiled-sql" } |
213 | | - } |
214 | | - errors.push(explicit.length > 0 ? `${why}, so not all its columns are known` : why) |
215 | | - return { columns: explicit.map((c) => ({ name: c })), columnsFrom: explicit.length > 0 ? "compiled-sql" : "none" } |
216 | | -} |
217 | | - |
218 | | -/** |
219 | | - * Each parent's table names, lowercased, as altimate-core can report them. The schema names |
220 | | - * buildSchemaContext uses (`alias || name`) are claimed first, then the other names and a source's |
221 | | - * identifier; the first parent to claim a name keeps it. `shared` holds the parents whose schema |
222 | | - * name another parent has too: the schema keeps only one of them, so a column can bind to the wrong one. |
223 | | - */ |
224 | | -function tablesOf(parentIds: string[], nodes: Record<string, any>, sources: Record<string, any>) { |
225 | | - const byTable = new Map<string, string>() |
226 | | - const bySchemaName = new Map<string, string[]>() |
227 | | - const claim = (key: unknown, uid: string) => { |
228 | | - if (typeof key === "string" && key && !byTable.has(key.toLowerCase())) byTable.set(key.toLowerCase(), uid) |
229 | | - } |
230 | | - const nodeOf = (uid: string) => nodes[uid] ?? sources[uid] |
231 | | - for (const uid of parentIds) { |
232 | | - const schemaName = nodeOf(uid)?.alias || nodeOf(uid)?.name |
233 | | - if (typeof schemaName !== "string" || !schemaName) continue |
234 | | - claim(schemaName, uid) |
235 | | - const key = schemaName.toLowerCase() |
236 | | - bySchemaName.set(key, [...(bySchemaName.get(key) ?? []), uid]) |
237 | | - } |
238 | | - for (const uid of parentIds) { |
239 | | - for (const key of [nodeOf(uid)?.alias, nodeOf(uid)?.name, nodeOf(uid)?.identifier]) claim(key, uid) |
240 | | - } |
241 | | - const shared = new Set([...bySchemaName.values()].filter((ids) => ids.length > 1).flat()) |
242 | | - return { byTable, shared } |
243 | | -} |
244 | | - |
245 | | -/** A dotted SQL name's parts, unquoted: `"db"."schema"."t"."c"` gives db, schema, t, c. */ |
246 | | -function nameParts(name: string): string[] { |
247 | | - const parts: string[] = [] |
248 | | - let current = "" |
249 | | - let quote: string | undefined |
250 | | - for (const ch of name) { |
251 | | - if (quote) { |
252 | | - if (ch === quote) quote = undefined |
253 | | - else current += ch |
254 | | - } else if (ch === '"' || ch === "`") quote = ch |
255 | | - else if (ch === ".") { |
256 | | - parts.push(current) |
257 | | - current = "" |
258 | | - } else current += ch |
259 | | - } |
260 | | - parts.push(current) |
261 | | - return parts |
262 | | -} |
263 | | - |
264 | | -/** |
265 | | - * altimate-core's column edges as `getModelLineage`'s, from a parent's column to the focus model's. |
266 | | - * An edge whose table isn't one of the parents goes to `errors`; one from a parent in `leaveOut` |
267 | | - * (already reported) is dropped. Edges are deduplicated: altimate-core 0.9 can report one edge under |
268 | | - * a qualified and a bare table name. |
269 | | - */ |
270 | | -export function lineageEdges( |
271 | | - raw: any[], |
272 | | - byTable: Map<string, string>, |
273 | | - focus: { uniqueId: string; name: string }, |
274 | | - errors: string[], |
275 | | - leaveOut: Set<string> = new Set(), |
276 | | -): LineageEdge[] { |
277 | | - const seen = new Set<string>() |
278 | | - const edges: LineageEdge[] = [] |
279 | | - for (const e of raw) { |
280 | | - const from = nameParts(String(e?.source ?? "")) |
281 | | - const column = from.length >= 2 ? from[from.length - 1].toLowerCase() : "" |
282 | | - const table = from.length >= 2 ? from[from.length - 2].toLowerCase() : "" |
283 | | - // The target is an output column's name, which can contain a dot (`as "customer.id"`). |
284 | | - const target = String(e?.target ?? "").toLowerCase() |
285 | | - const parent = byTable.get(table) |
286 | | - if (!parent || !column || !target) { |
287 | | - errors.push( |
288 | | - `${e?.source} -> ${focus.name}.${e?.target}: ${table ? `${table} isn't one of ${focus.name}'s parents` : "no table"}, so the edge was left out`, |
289 | | - ) |
290 | | - continue |
291 | | - } |
292 | | - if (leaveOut.has(parent)) continue |
293 | | - const edge: LineageEdge = { |
294 | | - source: [parent, column], |
295 | | - target: [focus.uniqueId, target], |
296 | | - type: String(e?.lineage_type ?? ""), |
297 | | - viewsType: String(e?.lens_type ?? ""), |
298 | | - viewsCode: (Array.isArray(e?.lens_code) ? e.lens_code : []) |
299 | | - .map((step: any) => (typeof step === "string" ? step : step?.expression)) |
300 | | - .filter((code: unknown): code is string => typeof code === "string"), |
301 | | - } |
302 | | - const key = [...edge.source, ...edge.target, edge.type, edge.viewsType].join("\u0000") |
303 | | - if (seen.has(key)) continue |
304 | | - seen.add(key) |
305 | | - edges.push(edge) |
306 | | - } |
307 | | - return edges |
308 | | -} |
0 commit comments