Skip to content

Commit 6cec6f4

Browse files
fix(ci): rebuild stale dbt version venvs; stop the versions suite skipping
- setup-versions.sh rebuilds a cached venv whose dbt does not run - key both caches on the setup-python version - versions test fails on a missing requested version, and points dbt-cli at each venv - job timeout 20 min Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_0174HUYiaceApqHP7EKMNgnb
1 parent dd32af6 commit 6cec6f4

3 files changed

Lines changed: 48 additions & 11 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -610,7 +610,9 @@ jobs:
610610
needs: changes
611611
if: github.event_name == 'push' || needs.changes.outputs.dbt-tools == 'true'
612612
runs-on: ubuntu-latest
613-
timeout-minutes: 10
613+
# The multi-version suite runs dbt seed/build/show/compile for each version;
614+
# on a cache miss the venvs are rebuilt first.
615+
timeout-minutes: 20
614616
# Runs pull-request code (setup scripts, tests), so it gets the same
615617
# read-only token scope and no persisted credentials as tracker-leaks.
616618
permissions:
@@ -639,7 +641,11 @@ jobs:
639641
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
640642
with:
641643
path: packages/dbt-tools/test/.dbt-venvs
642-
key: dbt-venvs-${{ runner.os }}-1.8-1.10-1.11
644+
# Both caches hold venvs linked to the setup-python interpreter, which
645+
# is not cached. Keying on its exact version starts a fresh cache when
646+
# the runner's 3.11 patch release moves, instead of restoring venvs
647+
# that no longer run.
648+
key: dbt-venvs-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-1.8-1.10-1.11
643649

644650
- name: Cache Python env scenarios
645651
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
@@ -648,11 +654,13 @@ jobs:
648654
# v2: the v1 cache held a uv venv linked to a uv-managed Python that
649655
# fresh runners lack (bin/dbt present, exec ENOENT). setup-resolve.sh
650656
# now validates cached envs and pins uv to the system interpreter.
651-
key: dbt-resolve-envs-${{ runner.os }}-v2
657+
key: dbt-resolve-envs-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-v2
652658

653659
- name: Set up dbt versions
654660
run: ./test/e2e/setup-versions.sh 1.8 1.10 1.11
655661
working-directory: packages/dbt-tools
662+
env:
663+
DBT_E2E_PYTHON: ${{ steps.python.outputs.python-path }}
656664

657665
- name: Set up Python env scenarios
658666
run: ./test/e2e/setup-resolve.sh venv uv system

‎packages/dbt-tools/test/e2e/dbt-versions.test.ts‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@
1212
* If no dbt is available at all, the entire suite is skipped.
1313
*
1414
* Environment variables:
15-
* DBT_E2E_VERSIONS — comma-separated list of versions to test (e.g. "1.8,1.9")
15+
* DBT_E2E_VERSIONS — comma-separated list of versions to test (e.g. "1.8,1.9");
16+
* a listed version without a working dbt fails the run
1617
* DBT_E2E_SKIP — set to "1" to skip e2e tests entirely
1718
*/
1819

@@ -29,6 +30,7 @@ import { tmpdir } from "os"
2930
const FIXTURE_DIR = resolve(import.meta.dir, "../fixture")
3031
const VENVS_DIR = resolve(import.meta.dir, "../.dbt-venvs")
3132
const SKIP = process.env.DBT_E2E_SKIP === "1"
33+
const REQUESTED = process.env.DBT_E2E_VERSIONS?.split(",").map((v) => v.trim())
3234

3335
/** Timeout for dbt commands (seed + build can be slow on first run) */
3436
const DBT_TIMEOUT = 120_000
@@ -51,13 +53,12 @@ interface DbtVersion {
5153
}
5254

5355
function discoverVersions(): DbtVersion[] {
54-
const filterVersions = process.env.DBT_E2E_VERSIONS?.split(",").map((v) => v.trim())
5556
const versions: DbtVersion[] = []
5657

5758
// Check venvs
5859
if (existsSync(VENVS_DIR)) {
5960
for (const entry of readdirSync(VENVS_DIR)) {
60-
if (filterVersions && !filterVersions.includes(entry)) continue
61+
if (REQUESTED && !REQUESTED.includes(entry)) continue
6162
const dbtPath = join(VENVS_DIR, entry, "bin", "dbt")
6263
const pythonPath = join(VENVS_DIR, entry, "bin", "python")
6364
if (!existsSync(dbtPath)) continue
@@ -98,6 +99,14 @@ if (!HAS_DBT && !SKIP) {
9899
)
99100
}
100101

102+
// Asking for versions (CI does) makes a missing one a failure, not a skip: a
103+
// stale venv cache once skipped every test below on every run without a sign.
104+
describe.skipIf(SKIP || !REQUESTED)("requested dbt versions", () => {
105+
test("every version in DBT_E2E_VERSIONS has a dbt that runs", () => {
106+
expect(VERSIONS.map((v) => v.label).sort()).toEqual([...(REQUESTED ?? [])].sort())
107+
})
108+
})
109+
101110
// ---------------------------------------------------------------------------
102111
// Helpers
103112
// ---------------------------------------------------------------------------
@@ -205,10 +214,15 @@ describe.skipIf(!HAS_DBT)("altimate-dbt e2e", () => {
205214
describe(`dbt ${version.label} (${version.full})`, () => {
206215
let workDir: string
207216

208-
beforeAll(() => {
217+
beforeAll(async () => {
209218
console.log(`\n→ Setting up dbt ${version.full} project...`)
210219
workDir = setupProject(version)
211220
console.log(` Project: ${workDir}`)
221+
// dbt-cli.ts caches the dbt it resolves on first use. configure()
222+
// clears that cache and points it at this version's venv; without it
223+
// every version after the first runs the first version's dbt.
224+
const { configure } = await import("../../src/dbt-cli")
225+
configure({ pythonPath: version.pythonPath, projectRoot: workDir })
212226
}, DBT_TIMEOUT * 2)
213227

214228
afterAll(() => {

‎packages/dbt-tools/test/e2e/setup-versions.sh‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,17 @@ get_pip_flags() {
4747
esac
4848
}
4949

50+
# CI passes the interpreter actions/setup-python installed; locally the first
51+
# python3 on PATH is used.
52+
PYTHON="${DBT_E2E_PYTHON:-python3}"
53+
54+
# Prints e.g. `core=1.8.7`, or nothing when the binary does not run. A restored
55+
# cache can hold a venv whose bin/python links to an interpreter that no longer
56+
# exists, so bin/dbt being on disk proves nothing.
57+
installed_version() {
58+
"$1" --version 2>&1 | grep -oE 'installed: [0-9.a-z]+' | head -1 | sed 's/installed: /core=/' || true
59+
}
60+
5061
mkdir -p "$VENVS_DIR"
5162

5263
for ver in "${VERSIONS[@]}"; do
@@ -60,13 +71,17 @@ for ver in "${VERSIONS[@]}"; do
6071
fi
6172

6273
if [ -f "$venv_dir/bin/dbt" ]; then
63-
existing=$("$venv_dir/bin/dbt" --version 2>&1 | grep -oE 'installed: [0-9.a-z]+' | head -1 | sed 's/installed: /core=/' | head -1 || echo "unknown")
64-
echo "✓ dbt $ver already installed ($existing) at $venv_dir"
65-
continue
74+
existing=$(installed_version "$venv_dir/bin/dbt")
75+
if [ -n "$existing" ]; then
76+
echo "✓ dbt $ver already installed ($existing) at $venv_dir"
77+
continue
78+
fi
79+
echo "↻ dbt $ver cache is stale (dbt does not run) — rebuilding..."
80+
rm -rf "$venv_dir"
6681
fi
6782

6883
echo "→ Installing dbt $ver..."
69-
python3 -m venv "$venv_dir"
84+
"$PYTHON" -m venv "$venv_dir"
7085
"$venv_dir/bin/pip" install --quiet --upgrade pip
7186
# shellcheck disable=SC2086
7287
"$venv_dir/bin/pip" install --quiet $extra_flags $install_spec

0 commit comments

Comments
 (0)