Skip to content

Commit 3f3cae2

Browse files
fix(dbt-tools): move to dbt-integration 0.3.17 so the dbt adapter starts under Bun (#1368)
1 parent 817bf69 commit 3f3cae2

21 files changed

Lines changed: 572 additions & 640 deletions

‎.github/workflows/release.yml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -286,9 +286,19 @@ jobs:
286286
echo "::error::dbt-tools bundle contains hardcoded absolute path in __dirname"
287287
exit 1
288288
fi
289-
# Verify __dirname was patched to runtime resolution
290-
if ! grep -q 'import.meta.dirname' packages/dbt-tools/dist/index.js; then
291-
echo "::error::dbt-tools bundle missing import.meta.dirname patch"
289+
# The bridge path is built at runtime from the bundle's own URL
290+
if ! grep -q 'fileURLToPath(import.meta.url)' packages/dbt-tools/dist/index.js; then
291+
echo "::error::dbt-tools bundle lacks fileURLToPath(import.meta.url), which the bridge path is built from"
292+
exit 1
293+
fi
294+
# ...joined with the bare script name, not a literal path ending in it
295+
if ! grep -qF '"node_python_bridge.py"' packages/dbt-tools/dist/index.js; then
296+
echo "::error::dbt-tools bundle lacks the \"node_python_bridge.py\" name the bridge path is joined from"
297+
exit 1
298+
fi
299+
# No path from this checkout baked into the bundle
300+
if [ -z "$GITHUB_WORKSPACE" ] || grep -qF "$GITHUB_WORKSPACE" packages/dbt-tools/dist/index.js; then
301+
echo "::error::dbt-tools bundle contains the build checkout path, or GITHUB_WORKSPACE is unset"
292302
exit 1
293303
fi
294304
echo "dbt-tools smoke test passed"

‎bun.lock‎

Lines changed: 18 additions & 64 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/dbt-tools/package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@
99
"scripts": {
1010
"build": "bun build src/index.ts --outdir dist --target node --format esm && bun run script/copy-python.ts",
1111
"typecheck": "tsc --noEmit",
12-
"test": "bun test test/cli.test.ts test/config.test.ts test/dbt-cli.test.ts test/dbt-resolve.test.ts test/build-integrity.test.ts --timeout 30000",
12+
"test": "bun test test/cli.test.ts test/config.test.ts test/dbt-cli.test.ts test/dbt-resolve.test.ts test/adapter-bun.test.ts test/stale-manifest.test.ts test/build-integrity.test.ts --timeout 30000",
1313
"test:e2e": "bun test test/e2e/ --timeout 300000"
1414
},
1515
"dependencies": {
16-
"@altimateai/dbt-integration": "^0.2.2"
16+
"@altimateai/dbt-integration": "^0.3.17"
1717
},
1818
"devDependencies": {
1919
"@tsconfig/bun": "catalog:",
Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { cpSync, existsSync, readFileSync, writeFileSync } from "fs"
1+
import { cpSync, existsSync, readFileSync } from "fs"
22
import { dirname, join } from "path"
33

44
const dist = join(import.meta.dir, "..", "dist")
@@ -20,23 +20,18 @@ if (!existsSync(bridgePy)) {
2020
cpSync(bridgePy, join(dist, "node_python_bridge.py"))
2121
console.log(`Copied node_python_bridge.py → dist/`)
2222

23-
// 3. Fix the hardcoded __dirname that bun bakes at compile time.
24-
// Replace it with a runtime resolution so the bridge script is found
25-
// relative to the built index.js, not the CI runner's node_modules.
23+
// 3. dbt-integration resolves node_python_bridge.py at runtime from its own
24+
// location (`fileURLToPath(import.meta.url)`), which in this bundle is dist/,
25+
// where step 2 put the script. This only checks that the bundle still has
26+
// the references that path is built from and no path baked in at build time;
27+
// test/build-integrity.test.ts runs the packaged CLI from another directory
28+
// to check where the script is found.
2629
const indexPath = join(dist, "index.js")
27-
let code = readFileSync(indexPath, "utf8")
28-
const pattern = /var __dirname\s*=\s*"[^"]*python-bridge[^"]*"/
29-
if (pattern.test(code)) {
30-
// import.meta.dirname is supported by Bun and Node >= 20.11.0.
31-
// Fallback via __require handles older runtimes where import.meta.dirname is unavailable.
32-
const replacement = `var __dirname = typeof import.meta.dirname === "string" ? import.meta.dirname : __require("path").dirname(__require("url").fileURLToPath(import.meta.url))`
33-
code = code.replace(pattern, replacement)
34-
writeFileSync(indexPath, code)
35-
console.log(`Patched __dirname in dist/index.js`)
36-
} else {
37-
const found = code.match(/var __dirname[^;]*/)?.[0] ?? "(not found)"
38-
console.error(`ERROR: could not find python-bridge __dirname to patch — the bundle format may have changed`)
39-
console.error(` Pattern: ${pattern}`)
40-
console.error(` Nearest match: ${found}`)
30+
const code = readFileSync(indexPath, "utf8")
31+
const baked = /var __dirname\s*=\s*"(?:[A-Za-z]:\\\\|\/)/
32+
if (!code.includes("fileURLToPath(import.meta.url)") || !code.includes(`"node_python_bridge.py"`) || baked.test(code)) {
33+
console.error(`ERROR: dist/index.js lacks the references the bridge path is built from, or has a build-time path`)
34+
console.error(` Has the @altimateai/dbt-integration bundle format changed?`)
4135
process.exit(1)
4236
}
37+
console.log(`Checked dist/index.js: bridge path references present, no build-time path`)

‎packages/dbt-tools/src/adapter.ts‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import type { Config } from "./config"
22
import { bufferLog } from "./log-buffer"
3+
import { recordManifest } from "./stale-manifest"
34
export { getRecentDbtLogs, clearDbtLogs } from "./log-buffer"
45
import {
56
DBTProjectIntegrationAdapter,
7+
DBTProjectIntegrationAdapterEvents,
68
DEFAULT_CONFIGURATION_VALUES,
79
DBTCommandFactory,
810
DBTCommandExecutionInfrastructure,
@@ -15,6 +17,8 @@ import {
1517
GraphParser,
1618
SourceParser,
1719
TestParser,
20+
UnitTestParser,
21+
SemanticModelParser,
1822
ExposureParser,
1923
FunctionParser,
2024
DocParser,
@@ -63,7 +67,10 @@ function terminal(): DBTTerminal {
6367
show: async () => {},
6468
log: (msg: string) => bufferLog(`[dbt] ${msg}`),
6569
trace: () => {},
66-
debug: () => {},
70+
// A crash at bridge startup is told only on the bridge's stderr, which the library logs here.
71+
debug: (name: string, msg: string) => {
72+
if (name === "PythonBridgeStderr") bufferLog(`[dbt:stderr] ${msg}`)
73+
},
6774
info: (_name: string, msg: string) => bufferLog(`[dbt] ${msg}`),
6875
warn: (_name: string, msg: string) => bufferLog(`[dbt:warn] ${msg}`),
6976
error: (_name: string, msg: string, e: unknown) => {
@@ -82,6 +89,12 @@ function env(cfg: Config): RuntimePythonEnvironment {
8289
}
8390
}
8491

92+
/** Whether this dbt-integration answers a model's lineage, for a caller deciding whether to `create`. */
93+
export function hasModelLineage(): boolean {
94+
const prototype = DBTProjectIntegrationAdapter.prototype as { getModelLineage?: unknown }
95+
return typeof prototype.getModelLineage === "function"
96+
}
97+
8598
export async function create(cfg: Config): Promise<DBTProjectIntegrationAdapter> {
8699
const config = configuration(cfg)
87100
const term = terminal()
@@ -141,6 +154,8 @@ export async function create(cfg: Config): Promise<DBTProjectIntegrationAdapter>
141154
defer,
142155
changed,
143156
cloudVariantDetector,
157+
// Every variant but fusion-stable runs on a dbt Core command engine, which needs these.
158+
{ pythonDBTCommandExecutionStrategy: python, dbtConfiguration: config, dbtIntegrationClient: client },
144159
)
145160

146161
const command = (
@@ -170,19 +185,7 @@ export async function create(cfg: Config): Promise<DBTProjectIntegrationAdapter>
170185
defer: DeferConfig,
171186
changed: () => void,
172187
): DBTProjectIntegration =>
173-
new DBTFusionCommandProjectIntegration(
174-
infra,
175-
factory,
176-
cli,
177-
runtime,
178-
provider,
179-
term,
180-
root,
181-
diag,
182-
defer,
183-
changed,
184-
cloudVariantDetector,
185-
)
188+
new DBTFusionCommandProjectIntegration(infra, factory, cli, runtime, provider, term, root, diag, defer, changed)
186189

187190
const adapter = new DBTProjectIntegrationAdapter(
188191
config,
@@ -200,13 +203,17 @@ export async function create(cfg: Config): Promise<DBTProjectIntegrationAdapter>
200203
new GraphParser(term),
201204
new SourceParser(term),
202205
new TestParser(term),
206+
new UnitTestParser(term),
203207
new ExposureParser(term),
204208
new FunctionParser(term),
205209
new DocParser(term),
206210
term,
207211
new ModelDepthParser(term, client, config),
212+
new SemanticModelParser(term),
208213
)
209214

215+
// Before `initialize`, whose parse is the one the commands answer from.
216+
adapter.on(DBTProjectIntegrationAdapterEvents.MANIFEST_PARSED, recordManifest)
210217
try {
211218
await adapter.initialize()
212219
} catch (e) {

‎packages/dbt-tools/src/commands/graph.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
import type { DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration"
22
import { execDbtLs } from "../dbt-cli"
3+
import { staleManifestNote } from "../stale-manifest"
34

45
export async function children(adapter: DBTProjectIntegrationAdapter, args: string[]) {
56
const model = flag(args, "model")
67
if (!model) return { error: "Missing --model" }
78
try {
8-
return await adapter.getChildrenModels({ table: model })
9+
return fromManifest(await adapter.getChildrenModels({ table: model }))
910
} catch (e) {
1011
// nodeMetaMap/graphMetaMap errors are specific to the library's manifest parsing.
1112
// Also catch TypeError for property-access failures on undefined nodes.
@@ -23,7 +24,7 @@ export async function parents(adapter: DBTProjectIntegrationAdapter, args: strin
2324
const model = flag(args, "model")
2425
if (!model) return { error: "Missing --model" }
2526
try {
26-
return await adapter.getParentModels({ table: model })
27+
return fromManifest(await adapter.getParentModels({ table: model }))
2728
} catch (e) {
2829
if (
2930
e instanceof TypeError ||
@@ -35,6 +36,12 @@ export async function parents(adapter: DBTProjectIntegrationAdapter, args: strin
3536
}
3637
}
3738

39+
// The note comes first: a long answer can be cut off on its way to the reader.
40+
function fromManifest<T>(models: T[]) {
41+
const stale_manifest = staleManifestNote()
42+
return stale_manifest ? { stale_manifest, models } : models
43+
}
44+
3845
function flag(args: string[], name: string): string | undefined {
3946
const i = args.indexOf(`--${name}`)
4047
return i >= 0 ? args[i + 1] : undefined

‎packages/dbt-tools/src/commands/schema-verify.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { ColumnMetaData, DBTProjectIntegrationAdapter } from "@altimateai/dbt-integration"
2+
import { staleManifestNote } from "../stale-manifest"
23

34
/**
45
* Verify that a model's actual produced columns match the spec declared in
@@ -22,8 +23,16 @@ import type { ColumnMetaData, DBTProjectIntegrationAdapter } from "@altimateai/d
2223
* Skip cases:
2324
* - "no-spec": schema.yml doesn't declare columns for this model — nothing
2425
* to verify; agent has no contract to fail against.
26+
*
27+
* Every answer rests on the manifest, its errors included, so each one says when that's a stale one.
2528
*/
2629
export async function schemaVerify(adapter: DBTProjectIntegrationAdapter, args: string[]) {
30+
const answer = await verify(adapter, args)
31+
const stale_manifest = staleManifestNote()
32+
return stale_manifest ? { stale_manifest, ...answer } : answer
33+
}
34+
35+
async function verify(adapter: DBTProjectIntegrationAdapter, args: string[]) {
2736
const model = flag(args, "model")
2837
if (!model) return { error: "Missing --model" }
2938

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
/** What `--format text` prints on stderr for an error answer. */
2+
export function failureLines(failed: Record<string, unknown>): string[] {
3+
const lines: string[] = []
4+
// The note comes first, as in the JSON answer: the error rests on that manifest.
5+
const stale = failed.stale_manifest
6+
if (stale && typeof stale === "object" && "message" in stale) lines.push(String(stale.message))
7+
lines.push(String(failed.error))
8+
if ("fix" in failed) lines.push(String(failed.fix))
9+
if (Array.isArray(failed.logs)) lines.push(failed.logs.join("\n"))
10+
return lines
11+
}

‎packages/dbt-tools/src/index.ts‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { existsSync } from "fs"
33
import { read, type Config } from "./config"
44
import { init } from "./commands/init"
55
import { validate } from "./check"
6+
import { failureLines } from "./failure-text"
67

78
const USAGE = {
89
commands: {
@@ -37,12 +38,6 @@ function flag(args: string[], name: string): string | undefined {
3738

3839
function diagnose(err: Error): { error: string; fix?: string } {
3940
const msg = err.message || String(err)
40-
if (msg.includes("private field") || msg.includes("#stdin") || msg.includes("#stdout")) {
41-
return {
42-
error: "Internal error: Bun runtime incompatibility with python-bridge. Please report this issue.",
43-
fix: "Try updating: bun install, or file a bug at https://github.com/AltimateAI/altimate-code/issues",
44-
}
45-
}
4641
if (msg.includes("IPC channel") || msg.includes("ERR_IPC_CHANNEL_CLOSED")) {
4742
return {
4843
error: "Failed to start Python bridge. The dbt Python process exited unexpectedly.",
@@ -75,8 +70,7 @@ function output(result: unknown) {
7570
if (fmt && typeof result === "object" && result !== null) {
7671
const obj = result as Record<string, unknown>
7772
if ("error" in obj) {
78-
console.error(String(obj.error))
79-
if ("fix" in obj) console.error(String(obj.fix))
73+
console.error(failureLines(obj).join("\n"))
8074
process.exit(1)
8175
}
8276
if ("sql" in obj) {
@@ -92,7 +86,7 @@ function output(result: unknown) {
9286
if (result && typeof result === "object" && "error" in result) process.exit(1)
9387
}
9488

95-
function bail(err: unknown): never {
89+
function failure(err: unknown) {
9690
const result: Record<string, unknown> = diagnose(err instanceof Error ? err : new Error(String(err)))
9791
// Include buffered dbt logs for diagnostics (see #249 — logs are buffered
9892
// in-memory instead of written to stderr to avoid TUI corruption).
@@ -105,11 +99,15 @@ function bail(err: unknown): never {
10599
} catch {
106100
// log-buffer might not have been loaded yet
107101
}
108-
console.log(JSON.stringify(result, null, 2))
102+
return result
103+
}
104+
105+
function bail(err: unknown): never {
106+
console.log(JSON.stringify(failure(err), null, 2))
109107
process.exit(1)
110108
}
111109

112-
// Catch unhandled rejections (bluebird throws outside normal promise chains)
110+
// An error that escapes main() still ends as a JSON error on stdout
113111
process.on("unhandledRejection", bail)
114112
process.on("uncaughtException", bail)
115113

@@ -142,13 +140,13 @@ async function main() {
142140
const { configure } = await import("./dbt-cli")
143141
configure({ pythonPath: cfg.pythonPath, projectRoot: cfg.projectRoot })
144142

145-
// Lazy import to avoid loading python-bridge until needed
143+
// Lazy import: the adapter loads dbt-integration, which commands above don't need
146144
let adapter
147145
try {
148146
const { create } = await import("./adapter")
149147
adapter = await create(cfg)
150148
} catch (err) {
151-
return diagnose(err instanceof Error ? err : new Error(String(err)))
149+
return failure(err)
152150
}
153151

154152
let result: unknown
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import type { ParsedManifest, StaleManifest } from "@altimateai/dbt-integration"
2+
3+
// Set when the project check failed and the last parse read what an earlier parse left on disk.
4+
let stale: StaleManifest | undefined
5+
6+
/** Takes each parsed manifest from the adapter, whose graph lookups don't say which one they used. */
7+
export function recordManifest(parsed: ParsedManifest): void {
8+
stale = parsed.stale
9+
}
10+
11+
function ago(ms: number): string {
12+
const format = new Intl.RelativeTimeFormat("en")
13+
const minutes = Math.round(ms / 60_000)
14+
if (minutes < 60) return format.format(-minutes, "minute")
15+
const hours = Math.round(minutes / 60)
16+
if (hours < 48) return format.format(-hours, "hour")
17+
return format.format(-Math.round(hours / 24), "day")
18+
}
19+
20+
/** What an answer built from the manifest adds, or undefined when the manifest is current. */
21+
export function staleManifestNote(now = Date.now()) {
22+
if (!stale) return undefined
23+
const age = Math.max(0, now - stale.writtenAt)
24+
return {
25+
message: `The dbt project check failed, so this comes from the manifest of the last successful parse, written ${ago(age)}. It may not match the project as it is now.`,
26+
manifest_path: stale.manifestPath,
27+
written_at: new Date(stale.writtenAt).toISOString(),
28+
age_seconds: Math.round(age / 1000),
29+
}
30+
}

0 commit comments

Comments
 (0)