Skip to content

Commit e550ac2

Browse files
anandgupta42claude
andcommitted
feat: [Phase 3+4] schema cache, finops, dbt, and local testing
Complete the native TypeScript replacement for all remaining Python bridge methods: schema caching, FinOps cost intelligence, dbt operations, and local testing. Schema cache (6 methods): - schema/cache.ts — SQLite-backed (better-sqlite3) with LIKE-based search - schema/pii-detector.ts — altimate-core classifyPii + cached schema - schema/tags.ts — Snowflake TAG_REFERENCES queries - schema/register.ts — schema.index, schema.search, schema.cache_status, schema.detect_pii, schema.tags, schema.tags_list FinOps (8 methods): - finops/credit-analyzer.ts — Snowflake/BigQuery/Databricks credit SQL - finops/query-history.ts — multi-warehouse query history templates - finops/warehouse-advisor.ts — sizing recommendations - finops/unused-resources.ts — stale table/idle warehouse detection - finops/role-access.ts — RBAC grants, hierarchy, user roles - finops/register.ts — all 8 finops.* dispatcher methods dbt (3 methods): - dbt/runner.ts — spawn dbt CLI with 300s timeout - dbt/manifest.ts — parse target/manifest.json - dbt/lineage.ts — manifest + altimate-core column lineage - dbt/register.ts — dbt.run, dbt.manifest, dbt.lineage Local testing (3 methods): - local/schema-sync.ts — introspect remote, create in DuckDB - local/test-local.ts — transpile + execute locally - local/register.ts — local.schema_sync, local.test, ping Add 50 unit tests covering registration, SQL templates, manifest parsing, upstream selectors, DuckDB type mapping, and error paths. Closes #218 Closes #219 Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent 2bbfec8 commit e550ac2

19 files changed

Lines changed: 3580 additions & 0 deletions
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
/**
2+
* dbt model lineage — column-level lineage from manifest + model name.
3+
*
4+
* Ported from Python altimate_engine.dbt.lineage.
5+
*/
6+
7+
import * as fs from "fs"
8+
import * as core from "@altimateai/altimate-core"
9+
import type {
10+
DbtLineageParams,
11+
DbtLineageResult,
12+
} from "../../bridge/protocol"
13+
14+
/**
15+
* Compute column-level lineage for a dbt model.
16+
*
17+
* Loads the manifest, finds the target model, extracts compiled SQL + upstream
18+
* schemas, and delegates to altimate-core's columnLineage().
19+
*/
20+
export function dbtLineage(params: DbtLineageParams): DbtLineageResult {
21+
const emptyResult = (factors: string[]): DbtLineageResult => ({
22+
model_name: params.model,
23+
raw_lineage: {},
24+
confidence: "low",
25+
confidence_factors: factors,
26+
})
27+
28+
if (!fs.existsSync(params.manifest_path)) {
29+
return emptyResult(["Manifest file not found"])
30+
}
31+
32+
let manifest: any
33+
try {
34+
const raw = fs.readFileSync(params.manifest_path, "utf-8")
35+
manifest = JSON.parse(raw)
36+
} catch (e) {
37+
return emptyResult([`Failed to parse manifest: ${e}`])
38+
}
39+
40+
const nodes = manifest.nodes || {}
41+
const sources = manifest.sources || {}
42+
43+
// Find target model by name or unique_id
44+
const modelNode = findModel(nodes, params.model)
45+
if (!modelNode) {
46+
return emptyResult([`Model '${params.model}' not found in manifest`])
47+
}
48+
49+
// Extract compiled SQL
50+
const sql = modelNode.compiled_code || modelNode.compiled_sql || ""
51+
if (!sql) {
52+
return emptyResult(["No compiled SQL found — run `dbt compile` first"])
53+
}
54+
55+
// Detect dialect
56+
let dialect = params.dialect
57+
if (!dialect) {
58+
dialect = detectDialect(manifest, modelNode)
59+
}
60+
61+
// Build schema context from upstream dependencies
62+
const upstreamIds: string[] = modelNode.depends_on?.nodes || []
63+
const schemaContext = buildSchemaContext(nodes, sources, upstreamIds)
64+
65+
// Delegate to altimate-core column_lineage
66+
let rawLineage: Record<string, unknown>
67+
try {
68+
const schema = schemaContext
69+
? core.Schema.fromJson(JSON.stringify(schemaContext))
70+
: undefined
71+
const result = core.columnLineage(sql, dialect, schema)
72+
rawLineage = JSON.parse(JSON.stringify(result))
73+
} catch (e) {
74+
rawLineage = { error: String(e) }
75+
}
76+
77+
return {
78+
model_name: modelNode.name || params.model,
79+
model_unique_id: getUniqueId(nodes, params.model),
80+
compiled_sql: sql,
81+
raw_lineage: rawLineage,
82+
confidence: rawLineage.error ? "low" : "high",
83+
confidence_factors: rawLineage.error ? [String(rawLineage.error)] : [],
84+
}
85+
}
86+
87+
function findModel(nodes: Record<string, any>, model: string): any | null {
88+
if (model in nodes) return nodes[model]
89+
for (const [, node] of Object.entries(nodes)) {
90+
if (node.resource_type !== "model") continue
91+
if (node.name === model) return node
92+
}
93+
return null
94+
}
95+
96+
function getUniqueId(nodes: Record<string, any>, model: string): string | undefined {
97+
if (model in nodes) return model
98+
for (const [nodeId, node] of Object.entries(nodes)) {
99+
if (node.resource_type === "model" && node.name === model) return nodeId
100+
}
101+
return undefined
102+
}
103+
104+
function detectDialect(manifest: any, modelNode: any): string {
105+
const metadata = manifest.metadata || {}
106+
const adapter = metadata.adapter_type || ""
107+
if (adapter) {
108+
const dialectMap: Record<string, string> = {
109+
snowflake: "snowflake",
110+
bigquery: "bigquery",
111+
databricks: "databricks",
112+
spark: "spark",
113+
postgres: "postgres",
114+
redshift: "redshift",
115+
duckdb: "duckdb",
116+
}
117+
return dialectMap[adapter] || adapter
118+
}
119+
return "snowflake"
120+
}
121+
122+
function buildSchemaContext(
123+
nodes: Record<string, any>,
124+
sources: Record<string, any>,
125+
upstreamIds: string[],
126+
): Record<string, any> | null {
127+
const tables: Record<string, any> = {}
128+
129+
for (const uid of upstreamIds) {
130+
const node = nodes[uid] || sources[uid]
131+
if (!node) continue
132+
133+
const tableName = node.alias || node.name || ""
134+
if (!tableName) continue
135+
136+
const columnsDict = node.columns || {}
137+
if (Object.keys(columnsDict).length === 0) continue
138+
139+
const cols = Object.entries(columnsDict).map(([colName, col]: [string, any]) => ({
140+
name: col.name || colName,
141+
type: col.data_type || col.type || "",
142+
}))
143+
144+
if (cols.length > 0) {
145+
tables[tableName] = { columns: cols }
146+
}
147+
}
148+
149+
if (Object.keys(tables).length === 0) return null
150+
return { tables, version: "1" }
151+
}
Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/**
2+
* dbt manifest.json parser — extract models, sources, and node information.
3+
*
4+
* Ported from Python altimate_engine.dbt.manifest.
5+
*/
6+
7+
import * as fs from "fs"
8+
import type {
9+
DbtManifestParams,
10+
DbtManifestResult,
11+
DbtModelInfo,
12+
DbtSourceInfo,
13+
ModelColumn,
14+
} from "../../bridge/protocol"
15+
16+
const LARGE_MANIFEST_BYTES = 50 * 1024 * 1024 // 50 MB
17+
18+
function extractColumns(columnsDict: Record<string, any>): ModelColumn[] {
19+
return Object.entries(columnsDict).map(([colName, col]) => ({
20+
name: col.name || colName,
21+
data_type: col.data_type || col.type || "",
22+
description: col.description || undefined,
23+
}))
24+
}
25+
26+
/**
27+
* Parse a dbt manifest.json and extract model, source, and node information.
28+
*/
29+
export function parseManifest(params: DbtManifestParams): DbtManifestResult {
30+
const emptyResult: DbtManifestResult = {
31+
models: [],
32+
sources: [],
33+
source_count: 0,
34+
model_count: 0,
35+
test_count: 0,
36+
snapshot_count: 0,
37+
seed_count: 0,
38+
}
39+
40+
if (!fs.existsSync(params.path)) {
41+
return emptyResult
42+
}
43+
44+
let raw: string
45+
try {
46+
const stat = fs.statSync(params.path)
47+
if (stat.size > LARGE_MANIFEST_BYTES) {
48+
// Log warning but continue
49+
}
50+
raw = fs.readFileSync(params.path, "utf-8")
51+
} catch {
52+
return emptyResult
53+
}
54+
55+
let manifest: any
56+
try {
57+
manifest = JSON.parse(raw)
58+
} catch {
59+
return emptyResult
60+
}
61+
62+
if (typeof manifest !== "object" || manifest === null) {
63+
return emptyResult
64+
}
65+
66+
const nodes = manifest.nodes || {}
67+
const sourcesDict = manifest.sources || {}
68+
69+
const models: DbtModelInfo[] = []
70+
let testCount = 0
71+
let snapshotCount = 0
72+
let seedCount = 0
73+
74+
for (const [nodeId, node] of Object.entries<any>(nodes)) {
75+
const resourceType = node.resource_type
76+
77+
if (resourceType === "model") {
78+
const dependsOnNodes = node.depends_on?.nodes || []
79+
const columns = extractColumns(node.columns || {})
80+
models.push({
81+
unique_id: nodeId,
82+
name: node.name || "",
83+
schema_name: node.schema || undefined,
84+
database: node.database || undefined,
85+
materialized: node.config?.materialized || undefined,
86+
depends_on: dependsOnNodes,
87+
columns,
88+
})
89+
} else if (resourceType === "test") {
90+
testCount++
91+
} else if (resourceType === "snapshot") {
92+
snapshotCount++
93+
} else if (resourceType === "seed") {
94+
seedCount++
95+
}
96+
}
97+
98+
const sources: DbtSourceInfo[] = []
99+
for (const [sourceId, source] of Object.entries<any>(sourcesDict)) {
100+
const columns = extractColumns(source.columns || {})
101+
sources.push({
102+
unique_id: sourceId,
103+
name: source.name || "",
104+
source_name: source.source_name || "",
105+
schema_name: source.schema || undefined,
106+
database: source.database || undefined,
107+
columns,
108+
})
109+
}
110+
111+
return {
112+
models,
113+
sources,
114+
source_count: sources.length,
115+
model_count: models.length,
116+
test_count: testCount,
117+
snapshot_count: snapshotCount,
118+
seed_count: seedCount,
119+
}
120+
}
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/**
2+
* Register dbt dispatcher methods.
3+
*/
4+
5+
import { register } from "../dispatcher"
6+
import { runDbt } from "./runner"
7+
import { parseManifest } from "./manifest"
8+
import { dbtLineage } from "./lineage"
9+
import type {
10+
DbtRunParams,
11+
DbtRunResult,
12+
DbtManifestParams,
13+
DbtManifestResult,
14+
DbtLineageParams,
15+
DbtLineageResult,
16+
} from "../../bridge/protocol"
17+
18+
/** Register all dbt.* native handlers. Exported for test re-registration. */
19+
export function registerAll(): void {
20+
21+
register("dbt.run", async (params: DbtRunParams): Promise<DbtRunResult> => {
22+
return runDbt(params)
23+
})
24+
25+
register("dbt.manifest", async (params: DbtManifestParams): Promise<DbtManifestResult> => {
26+
return parseManifest(params)
27+
})
28+
29+
register("dbt.lineage", async (params: DbtLineageParams): Promise<DbtLineageResult> => {
30+
return dbtLineage(params)
31+
})
32+
33+
} // end registerAll
34+
35+
// Auto-register on module load
36+
registerAll()
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/**
2+
* dbt CLI wrapper — spawn dbt commands as subprocesses.
3+
*
4+
* Ported from Python altimate_engine.dbt.runner.
5+
*/
6+
7+
import { execFile } from "child_process"
8+
import type {
9+
DbtRunParams,
10+
DbtRunResult,
11+
} from "../../bridge/protocol"
12+
13+
/**
14+
* Prepend + to selector for build/run/test to include upstream deps.
15+
*/
16+
function ensureUpstreamSelector(select: string, command: string): string {
17+
if (!["build", "run", "test"].includes(command)) return select
18+
if (select.startsWith("+")) return select
19+
// Tag/path/source selectors: don't add +
20+
if (select.includes(":") && !select.startsWith("+")) return select
21+
return `+${select}`
22+
}
23+
24+
/**
25+
* Run a dbt CLI command via subprocess.
26+
*/
27+
export function runDbt(params: DbtRunParams): Promise<DbtRunResult> {
28+
return new Promise((resolve) => {
29+
const command = params.command || "run"
30+
const args: string[] = [command]
31+
32+
if (params.select) {
33+
const select = ensureUpstreamSelector(params.select, command)
34+
args.push("--select", select)
35+
}
36+
37+
if (params.args) {
38+
args.push(...params.args)
39+
}
40+
41+
if (params.project_dir) {
42+
args.push("--project-dir", params.project_dir)
43+
}
44+
45+
execFile("dbt", args, { timeout: 300_000 }, (error, stdout, stderr) => {
46+
if (error) {
47+
if ((error as any).code === "ENOENT") {
48+
resolve({
49+
stdout: "",
50+
stderr: "dbt CLI not found. Install with: pip install dbt-core",
51+
exit_code: 127,
52+
})
53+
return
54+
}
55+
if (error.killed) {
56+
resolve({
57+
stdout: stdout || "",
58+
stderr: "dbt command timed out after 300 seconds",
59+
exit_code: 124,
60+
})
61+
return
62+
}
63+
resolve({
64+
stdout: stdout || "",
65+
stderr: stderr || error.message,
66+
exit_code: (error as any).code ?? 1,
67+
})
68+
return
69+
}
70+
resolve({
71+
stdout: stdout || "",
72+
stderr: stderr || "",
73+
exit_code: 0,
74+
})
75+
})
76+
})
77+
}
78+
79+
// Exported for testing
80+
export { ensureUpstreamSelector }

0 commit comments

Comments
 (0)