|
| 1 | +/** |
| 2 | + * Direct dbt CLI fallbacks for when the library's output parsing fails. |
| 3 | + * |
| 4 | + * Newer dbt versions (1.11+) may produce JSON log output that the |
| 5 | + * @altimateai/dbt-integration library cannot parse. These functions run dbt |
| 6 | + * commands directly and parse the output with more resilient logic. |
| 7 | + */ |
| 8 | + |
| 9 | +import { execFile } from "child_process" |
| 10 | + |
| 11 | +function run(args: string[]): Promise<{ stdout: string; stderr: string }> { |
| 12 | + return new Promise((resolve, reject) => { |
| 13 | + execFile("dbt", args, { timeout: 120_000, maxBuffer: 10 * 1024 * 1024 }, (err, stdout, stderr) => { |
| 14 | + if (err) reject(err) |
| 15 | + else resolve({ stdout, stderr }) |
| 16 | + }) |
| 17 | + }) |
| 18 | +} |
| 19 | + |
| 20 | +/** |
| 21 | + * Parse structured JSON log lines from dbt CLI output. |
| 22 | + * dbt emits one JSON object per line when --log-format json is used. |
| 23 | + */ |
| 24 | +function parseJsonLines(stdout: string): any[] { |
| 25 | + return stdout |
| 26 | + .trim() |
| 27 | + .split("\n") |
| 28 | + .map((line) => { |
| 29 | + try { |
| 30 | + return JSON.parse(line.trim()) |
| 31 | + } catch { |
| 32 | + return null |
| 33 | + } |
| 34 | + }) |
| 35 | + .filter(Boolean) |
| 36 | +} |
| 37 | + |
| 38 | +/** |
| 39 | + * Execute SQL via `dbt show` and return results in QueryExecutionResult shape. |
| 40 | + */ |
| 41 | +export async function execDbtShow(sql: string, limit?: number) { |
| 42 | + const args = ["show", "--inline", sql, "--output", "json", "--log-format", "json"] |
| 43 | + if (limit !== undefined) args.push("--limit", String(limit)) |
| 44 | + |
| 45 | + const { stdout } = await run(args) |
| 46 | + const lines = parseJsonLines(stdout) |
| 47 | + |
| 48 | + // Try multiple known dbt output formats for the preview data |
| 49 | + const previewLine = |
| 50 | + lines.find((l) => l.data?.preview) ?? |
| 51 | + lines.find((l) => l.data?.rows) ?? |
| 52 | + lines.find((l) => l.result?.preview) |
| 53 | + |
| 54 | + const sqlLine = |
| 55 | + lines.find((l) => l.data?.sql) ?? |
| 56 | + lines.find((l) => l.data?.compiled_sql) |
| 57 | + |
| 58 | + if (!previewLine) { |
| 59 | + // Last resort: try agate table format from non-JSON output |
| 60 | + throw new Error( |
| 61 | + "Could not parse dbt show output. Ensure dbt supports --output json. " + |
| 62 | + `Got ${lines.length} JSON lines, none with preview data.`, |
| 63 | + ) |
| 64 | + } |
| 65 | + |
| 66 | + const preview = previewLine.data?.preview ?? previewLine.data?.rows ?? previewLine.result?.preview |
| 67 | + const rows: Record<string, unknown>[] = typeof preview === "string" ? JSON.parse(preview) : preview |
| 68 | + |
| 69 | + const columnNames = rows.length > 0 && rows[0] ? Object.keys(rows[0]) : [] |
| 70 | + const compiledSql = sqlLine?.data?.sql ?? sqlLine?.data?.compiled_sql ?? sql |
| 71 | + |
| 72 | + return { |
| 73 | + columnNames, |
| 74 | + columnTypes: columnNames.map(() => "string"), |
| 75 | + data: rows, |
| 76 | + rawSql: sql, |
| 77 | + compiledSql, |
| 78 | + } |
| 79 | +} |
| 80 | + |
| 81 | +/** |
| 82 | + * Compile a model via `dbt compile --select <model>` and return compiled SQL. |
| 83 | + */ |
| 84 | +export async function execDbtCompile(model: string): Promise<{ sql: string }> { |
| 85 | + const args = ["compile", "--select", model, "--output", "json", "--log-format", "json"] |
| 86 | + const { stdout } = await run(args) |
| 87 | + const lines = parseJsonLines(stdout) |
| 88 | + |
| 89 | + const compiledLine = |
| 90 | + lines.find((l) => l.data?.compiled) ?? |
| 91 | + lines.find((l) => l.data?.compiled_code) ?? |
| 92 | + lines.find((l) => l.result?.node?.compiled_code) |
| 93 | + |
| 94 | + if (compiledLine) { |
| 95 | + const sql = |
| 96 | + compiledLine.data?.compiled ?? |
| 97 | + compiledLine.data?.compiled_code ?? |
| 98 | + compiledLine.result?.node?.compiled_code |
| 99 | + return { sql } |
| 100 | + } |
| 101 | + |
| 102 | + // Fallback: try plain text output |
| 103 | + const { stdout: plainOut } = await run(["compile", "--select", model]) |
| 104 | + return { sql: plainOut.trim() } |
| 105 | +} |
| 106 | + |
| 107 | +/** |
| 108 | + * Compile an inline query via `dbt compile --inline <sql>`. |
| 109 | + */ |
| 110 | +export async function execDbtCompileInline( |
| 111 | + sql: string, |
| 112 | + model?: string | null, |
| 113 | +): Promise<{ sql: string }> { |
| 114 | + const args = ["compile", "--inline", sql, "--output", "json", "--log-format", "json"] |
| 115 | + const { stdout } = await run(args) |
| 116 | + const lines = parseJsonLines(stdout) |
| 117 | + |
| 118 | + const compiledLine = |
| 119 | + lines.find((l) => l.data?.compiled) ?? |
| 120 | + lines.find((l) => l.data?.compiled_code) ?? |
| 121 | + lines.find((l) => l.result?.node?.compiled_code) |
| 122 | + |
| 123 | + if (compiledLine) { |
| 124 | + const result = |
| 125 | + compiledLine.data?.compiled ?? |
| 126 | + compiledLine.data?.compiled_code ?? |
| 127 | + compiledLine.result?.node?.compiled_code |
| 128 | + return { sql: result } |
| 129 | + } |
| 130 | + |
| 131 | + const { stdout: plainOut } = await run(["compile", "--inline", sql]) |
| 132 | + return { sql: plainOut.trim() } |
| 133 | +} |
| 134 | + |
| 135 | +/** |
| 136 | + * List children or parents of a model via `dbt ls`. |
| 137 | + */ |
| 138 | +export async function execDbtLs( |
| 139 | + model: string, |
| 140 | + direction: "children" | "parents", |
| 141 | +): Promise<{ table: string; label: string }[]> { |
| 142 | + const selector = direction === "children" ? `${model}+` : `+${model}` |
| 143 | + const args = ["ls", "--select", selector, "--resource-types", "model", "--output", "json"] |
| 144 | + |
| 145 | + const { stdout } = await run(args) |
| 146 | + const lines = parseJsonLines(stdout) |
| 147 | + |
| 148 | + return lines |
| 149 | + .filter((l) => { |
| 150 | + const name = l.name ?? l.unique_id?.split(".").pop() |
| 151 | + return name && name !== model |
| 152 | + }) |
| 153 | + .map((l) => ({ |
| 154 | + table: l.name ?? l.unique_id?.split(".").pop() ?? "unknown", |
| 155 | + label: l.name ?? l.unique_id?.split(".").pop() ?? "unknown", |
| 156 | + })) |
| 157 | +} |
0 commit comments