Skip to content

Commit 9f0afa8

Browse files
anandgupta42claude
andcommitted
fix: altimate-dbt compile, execute, and children commands — CLI fallbacks for dbt 1.11+ (#252)
The `@altimateai/dbt-integration` library's JSON output parsing breaks with newer dbt versions (1.11.x) where the log format changed. Three commands were affected: - `execute`: `dbt show` output no longer contains `data.preview` in the expected format — `d[0].data` throws when the filter returns empty. - `compile`: `dbt compile` output no longer contains `data.compiled` — same `o[0].data` pattern failure. - `children`: `nodeMetaMap.lookupByBaseName()` fails when the manifest file-path resolution doesn't populate the model-name lookup map. Additionally, `tryExecuteViaDbt` in opencode incorrectly expected `raw.table` on `QueryExecutionResult`, which actually has `{ columnNames, columnTypes, data }` — causing the dbt-first execution path to always fall through to native drivers silently. Fixes: - Add try-catch in execute/compile/graph commands with fallback to direct `dbt` CLI subprocess calls (`dbt show`, `dbt compile`, `dbt ls`) - New `dbt-cli.ts` module with resilient multi-format JSON output parsing (handles `data.preview`, `data.rows`, `data.compiled`, `data.compiled_code`, `result.node.compiled_code`) - Fix `tryExecuteViaDbt` to recognize `QueryExecutionResult` shape first, then fall back to legacy `raw.table` format Closes #252 Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent a53cffe commit 9f0afa8

7 files changed

Lines changed: 572 additions & 10 deletions

File tree

‎packages/dbt-tools/src/commands/compile.ts‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,35 @@
11
import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration"
2+
import { execDbtCompile, execDbtCompileInline } from "../dbt-cli"
23

34
export async function compile(adapter: DBTProjectIntegrationAdapter, args: string[]) {
45
const model = flag(args, "model")
56
if (!model) return { error: "Missing --model" }
6-
const sql = await adapter.unsafeCompileNode(model)
7-
return { sql }
7+
try {
8+
const sql = await adapter.unsafeCompileNode(model)
9+
return { sql }
10+
} catch (e) {
11+
const msg = e instanceof Error ? e.message : String(e)
12+
if (msg.includes("Cannot read properties of undefined")) {
13+
return execDbtCompile(model)
14+
}
15+
throw e
16+
}
817
}
918

1019
export async function query(adapter: DBTProjectIntegrationAdapter, args: string[]) {
1120
const sql = flag(args, "query")
1221
if (!sql) return { error: "Missing --query" }
1322
const model = flag(args, "model")
14-
const result = await adapter.unsafeCompileQuery(sql, model)
15-
return { sql: result }
23+
try {
24+
const result = await adapter.unsafeCompileQuery(sql, model)
25+
return { sql: result }
26+
} catch (e) {
27+
const msg = e instanceof Error ? e.message : String(e)
28+
if (msg.includes("Cannot read properties of undefined")) {
29+
return execDbtCompileInline(sql, model)
30+
}
31+
throw e
32+
}
1633
}
1734

1835
function flag(args: string[], name: string): string | undefined {

‎packages/dbt-tools/src/commands/execute.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,23 @@
11
import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration"
2+
import { execDbtShow } from "../dbt-cli"
23

34
export async function execute(adapter: DBTProjectIntegrationAdapter, args: string[]) {
45
const sql = flag(args, "query")
56
if (!sql) return { error: "Missing --query" }
67
const model = flag(args, "model") ?? ""
78
const raw = flag(args, "limit")
89
const limit = raw !== undefined ? parseInt(raw, 10) : undefined
9-
if (limit !== undefined && !Number.isNaN(limit)) return adapter.immediatelyExecuteSQLWithLimit(sql, model, limit)
10-
return adapter.immediatelyExecuteSQL(sql, model)
10+
try {
11+
if (limit !== undefined && !Number.isNaN(limit)) return await adapter.immediatelyExecuteSQLWithLimit(sql, model, limit)
12+
return await adapter.immediatelyExecuteSQL(sql, model)
13+
} catch (e) {
14+
// Library's dbt show parsing may fail with newer dbt versions — fall back to direct CLI
15+
const msg = e instanceof Error ? e.message : String(e)
16+
if (msg.includes("Cannot read properties of undefined") || msg.includes("Could not find previewLine")) {
17+
return execDbtShow(sql, limit)
18+
}
19+
throw e
20+
}
1121
}
1222

1323
function flag(args: string[], name: string): string | undefined {

‎packages/dbt-tools/src/commands/graph.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,32 @@
11
import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration"
2+
import { execDbtLs } from "../dbt-cli"
23

34
export function children(adapter: DBTProjectIntegrationAdapter, args: string[]) {
45
const model = flag(args, "model")
56
if (!model) return { error: "Missing --model" }
6-
return adapter.getChildrenModels({ table: model })
7+
try {
8+
return adapter.getChildrenModels({ table: model })
9+
} catch (e) {
10+
const msg = e instanceof Error ? e.message : String(e)
11+
if (msg.includes("nodeMetaMap has no entries") || msg.includes("graphMetaMap")) {
12+
return execDbtLs(model, "children")
13+
}
14+
throw e
15+
}
716
}
817

918
export function parents(adapter: DBTProjectIntegrationAdapter, args: string[]) {
1019
const model = flag(args, "model")
1120
if (!model) return { error: "Missing --model" }
12-
return adapter.getParentModels({ table: model })
21+
try {
22+
return adapter.getParentModels({ table: model })
23+
} catch (e) {
24+
const msg = e instanceof Error ? e.message : String(e)
25+
if (msg.includes("nodeMetaMap has no entries") || msg.includes("graphMetaMap")) {
26+
return execDbtLs(model, "parents")
27+
}
28+
throw e
29+
}
1330
}
1431

1532
function flag(args: string[], name: string): string | undefined {

‎packages/dbt-tools/src/dbt-cli.ts‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
/**
2+
* Direct dbt CLI fallbacks for when the library's output parsing fails.
3+
*
4+
* Newer dbt versions (1.11+) may produce JSON log output that the
5+
* @altimateai/dbt-integration library cannot parse. These functions run dbt
6+
* commands directly and parse the output with more resilient logic.
7+
*/
8+
9+
import { execFile } from "child_process"
10+
11+
function run(args: string[]): Promise<{ stdout: string; stderr: string }> {
12+
return new Promise((resolve, reject) => {
13+
execFile("dbt", args, { timeout: 120_000, maxBuffer: 10 * 1024 * 1024 }, (err, stdout, stderr) => {
14+
if (err) reject(err)
15+
else resolve({ stdout, stderr })
16+
})
17+
})
18+
}
19+
20+
/**
21+
* Parse structured JSON log lines from dbt CLI output.
22+
* dbt emits one JSON object per line when --log-format json is used.
23+
*/
24+
function parseJsonLines(stdout: string): any[] {
25+
return stdout
26+
.trim()
27+
.split("\n")
28+
.map((line) => {
29+
try {
30+
return JSON.parse(line.trim())
31+
} catch {
32+
return null
33+
}
34+
})
35+
.filter(Boolean)
36+
}
37+
38+
/**
39+
* Execute SQL via `dbt show` and return results in QueryExecutionResult shape.
40+
*/
41+
export async function execDbtShow(sql: string, limit?: number) {
42+
const args = ["show", "--inline", sql, "--output", "json", "--log-format", "json"]
43+
if (limit !== undefined) args.push("--limit", String(limit))
44+
45+
const { stdout } = await run(args)
46+
const lines = parseJsonLines(stdout)
47+
48+
// Try multiple known dbt output formats for the preview data
49+
const previewLine =
50+
lines.find((l) => l.data?.preview) ??
51+
lines.find((l) => l.data?.rows) ??
52+
lines.find((l) => l.result?.preview)
53+
54+
const sqlLine =
55+
lines.find((l) => l.data?.sql) ??
56+
lines.find((l) => l.data?.compiled_sql)
57+
58+
if (!previewLine) {
59+
// Last resort: try agate table format from non-JSON output
60+
throw new Error(
61+
"Could not parse dbt show output. Ensure dbt supports --output json. " +
62+
`Got ${lines.length} JSON lines, none with preview data.`,
63+
)
64+
}
65+
66+
const preview = previewLine.data?.preview ?? previewLine.data?.rows ?? previewLine.result?.preview
67+
const rows: Record<string, unknown>[] = typeof preview === "string" ? JSON.parse(preview) : preview
68+
69+
const columnNames = rows.length > 0 && rows[0] ? Object.keys(rows[0]) : []
70+
const compiledSql = sqlLine?.data?.sql ?? sqlLine?.data?.compiled_sql ?? sql
71+
72+
return {
73+
columnNames,
74+
columnTypes: columnNames.map(() => "string"),
75+
data: rows,
76+
rawSql: sql,
77+
compiledSql,
78+
}
79+
}
80+
81+
/**
82+
* Compile a model via `dbt compile --select <model>` and return compiled SQL.
83+
*/
84+
export async function execDbtCompile(model: string): Promise<{ sql: string }> {
85+
const args = ["compile", "--select", model, "--output", "json", "--log-format", "json"]
86+
const { stdout } = await run(args)
87+
const lines = parseJsonLines(stdout)
88+
89+
const compiledLine =
90+
lines.find((l) => l.data?.compiled) ??
91+
lines.find((l) => l.data?.compiled_code) ??
92+
lines.find((l) => l.result?.node?.compiled_code)
93+
94+
if (compiledLine) {
95+
const sql =
96+
compiledLine.data?.compiled ??
97+
compiledLine.data?.compiled_code ??
98+
compiledLine.result?.node?.compiled_code
99+
return { sql }
100+
}
101+
102+
// Fallback: try plain text output
103+
const { stdout: plainOut } = await run(["compile", "--select", model])
104+
return { sql: plainOut.trim() }
105+
}
106+
107+
/**
108+
* Compile an inline query via `dbt compile --inline <sql>`.
109+
*/
110+
export async function execDbtCompileInline(
111+
sql: string,
112+
model?: string | null,
113+
): Promise<{ sql: string }> {
114+
const args = ["compile", "--inline", sql, "--output", "json", "--log-format", "json"]
115+
const { stdout } = await run(args)
116+
const lines = parseJsonLines(stdout)
117+
118+
const compiledLine =
119+
lines.find((l) => l.data?.compiled) ??
120+
lines.find((l) => l.data?.compiled_code) ??
121+
lines.find((l) => l.result?.node?.compiled_code)
122+
123+
if (compiledLine) {
124+
const result =
125+
compiledLine.data?.compiled ??
126+
compiledLine.data?.compiled_code ??
127+
compiledLine.result?.node?.compiled_code
128+
return { sql: result }
129+
}
130+
131+
const { stdout: plainOut } = await run(["compile", "--inline", sql])
132+
return { sql: plainOut.trim() }
133+
}
134+
135+
/**
136+
* List children or parents of a model via `dbt ls`.
137+
*/
138+
export async function execDbtLs(
139+
model: string,
140+
direction: "children" | "parents",
141+
): Promise<{ table: string; label: string }[]> {
142+
const selector = direction === "children" ? `${model}+` : `+${model}`
143+
const args = ["ls", "--select", selector, "--resource-types", "model", "--output", "json"]
144+
145+
const { stdout } = await run(args)
146+
const lines = parseJsonLines(stdout)
147+
148+
return lines
149+
.filter((l) => {
150+
const name = l.name ?? l.unique_id?.split(".").pop()
151+
return name && name !== model
152+
})
153+
.map((l) => ({
154+
table: l.name ?? l.unique_id?.split(".").pop() ?? "unknown",
155+
label: l.name ?? l.unique_id?.split(".").pop() ?? "unknown",
156+
}))
157+
}

0 commit comments

Comments
 (0)