Skip to content

Commit 4216cd0

Browse files
Merge branch 'develop' into nemo/spatialbench-duckdb
2 parents fb76037 + e6d0e08 commit 4216cd0

393 files changed

Lines changed: 7029 additions & 4833 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,25 @@ env:
2525
NIGHTLY_TOOLCHAIN: nightly-2026-02-05
2626

2727
jobs:
28+
duckdb-mirror:
29+
name: "Mirror DuckDB to R2"
30+
if: github.event_name == 'pull_request'
31+
uses: ./.github/workflows/duckdb-r2.yml
32+
secrets: inherit
33+
34+
duckdb-ready:
35+
name: "DuckDB libraries available in R2"
36+
needs: duckdb-mirror
37+
if: ${{ !cancelled() }}
38+
runs-on: ubuntu-latest
39+
timeout-minutes: 5
40+
steps:
41+
- name: Verify DuckDB mirror
42+
if: ${{ needs.duckdb-mirror.result == 'failure' }}
43+
run: |
44+
echo "DuckDB mirror failed; downstream builds would 404"
45+
exit 1
46+
2847
lint-toml:
2948
runs-on: ubuntu-latest
3049
timeout-minutes: 10
@@ -115,6 +134,7 @@ jobs:
115134

116135
rust-docs:
117136
name: "Rust (docs)"
137+
needs: duckdb-ready
118138
timeout-minutes: 30
119139
runs-on: >-
120140
${{ github.repository == 'vortex-data/vortex'
@@ -204,6 +224,7 @@ jobs:
204224

205225
rust-lint:
206226
name: "Rust (lint)"
227+
needs: duckdb-ready
207228
timeout-minutes: 30
208229
runs-on: >-
209230
${{ github.repository == 'vortex-data/vortex'
@@ -301,6 +322,7 @@ jobs:
301322
302323
rust-test-other:
303324
name: "Rust tests (${{ matrix.os }})"
325+
needs: duckdb-ready
304326
timeout-minutes: 30
305327
strategy:
306328
fail-fast: false
@@ -422,6 +444,7 @@ jobs:
422444

423445
sqllogic-test:
424446
name: "SQL logic tests"
447+
needs: duckdb-ready
425448
runs-on: >-
426449
${{ github.repository == 'vortex-data/vortex'
427450
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=sql-logic-test', github.run_id)

‎.github/workflows/duckdb-r2.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: DuckDB R2 mirror
2+
3+
# If we have an archive for a commit or a tag in R2, return it.
4+
# Otherwise, if it's a tag, download release archive from Duckdb Github
5+
# page and upload it to R2.
6+
# If it's a commit, build Duckdb from source and upload it to R2.
7+
on:
8+
workflow_call: { }
9+
10+
concurrency:
11+
group: duckdb-r2-${{ github.event.pull_request.number || github.ref }}
12+
cancel-in-progress: false
13+
14+
permissions:
15+
contents: read
16+
17+
env:
18+
PUBLIC_BASE_URL: "https://ci-builds.vortex.dev"
19+
R2_BUCKET: "duckdb-builds"
20+
R2_ENDPOINT_URL: "https://52bdeab5651e1584747feefd051fd566.r2.cloudflarestorage.com"
21+
22+
jobs:
23+
check:
24+
name: "Resolve DuckDB version and check R2"
25+
runs-on: ubuntu-latest
26+
timeout-minutes: 10
27+
outputs:
28+
version: ${{ steps.resolve.outputs.version }}
29+
ref_dir: ${{ steps.resolve.outputs.ref_dir }}
30+
release: ${{ steps.resolve.outputs.release }}
31+
matrix: ${{ steps.resolve.outputs.matrix }}
32+
any_missing: ${{ steps.resolve.outputs.any_missing }}
33+
steps:
34+
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
35+
- name: Resolve version and check R2
36+
id: resolve
37+
run: sh scripts/duckdb-r2-resolve.sh
38+
39+
mirror:
40+
name: "Mirror DuckDB ${{ matrix.archive }} to R2"
41+
needs: check
42+
if: >-
43+
needs.check.outputs.any_missing == 'true' &&
44+
github.repository == 'vortex-data/vortex' &&
45+
github.event.pull_request.head.repo.full_name == github.repository
46+
environment: duckdb-build
47+
timeout-minutes: 120
48+
strategy:
49+
fail-fast: false
50+
matrix: ${{ fromJSON(needs.check.outputs.matrix) }}
51+
runs-on: ${{ matrix.runner }}
52+
steps:
53+
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
54+
55+
- name: Install build dependencies (Linux)
56+
if: needs.check.outputs.release != 'true' && runner.os == 'Linux'
57+
run: |
58+
sudo apt-get update
59+
sudo apt-get install -y ninja-build libcurl4-openssl-dev zip unzip
60+
61+
# MacOS already has ninja and p7zip
62+
63+
- name: Prepare ${{ matrix.archive }}
64+
env:
65+
ARCHIVE: ${{ matrix.archive }}
66+
REF_DIR: ${{ needs.check.outputs.ref_dir }}
67+
RELEASE: ${{ needs.check.outputs.release }}
68+
PLATFORM_OS: ${{ matrix.os }}
69+
run: sh scripts/duckdb-r2-prepare.sh
70+
71+
- name: Upload to R2
72+
env:
73+
AWS_ACCESS_KEY_ID: ${{ secrets.DUCKDB_R2_ACCESS_KEY_ID }}
74+
AWS_SECRET_ACCESS_KEY: ${{ secrets.DUCKDB_R2_SECRET_ACCESS_KEY }}
75+
AWS_REGION: "us-east-1"
76+
AWS_ENDPOINT_URL: ${{ env.R2_ENDPOINT_URL }}
77+
run: |
78+
set -Eeuo pipefail
79+
python3 scripts/s3-upload.py \
80+
--bucket "$R2_BUCKET" \
81+
--key "${{ needs.check.outputs.ref_dir }}/${{ matrix.archive }}" \
82+
--body "${{ matrix.archive }}" \
83+
--checksum-algorithm CRC32

‎.github/workflows/rust-instrumented.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,28 @@ env:
2222
NIGHTLY_TOOLCHAIN: nightly-2026-02-05
2323

2424
jobs:
25+
duckdb-mirror:
26+
name: "Mirror DuckDB to R2"
27+
if: github.event_name == 'pull_request'
28+
uses: ./.github/workflows/duckdb-r2.yml
29+
secrets: inherit
30+
31+
duckdb-ready:
32+
name: "DuckDB libraries available in R2"
33+
needs: duckdb-mirror
34+
if: ${{ !cancelled() }}
35+
runs-on: ubuntu-latest
36+
timeout-minutes: 5
37+
steps:
38+
- name: Verify DuckDB mirror
39+
if: ${{ needs.duckdb-mirror.result == 'failure' }}
40+
run: |
41+
echo "DuckDB mirror failed"
42+
exit 1
43+
2544
rust-coverage:
2645
name: "Rust tests (coverage) (${{ matrix.suite }})"
46+
needs: duckdb-ready
2747
timeout-minutes: 30
2848
permissions:
2949
id-token: write

‎Cargo.lock‎

Lines changed: 4 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)