Skip to content

Commit d53be9c

Browse files
authored
Merge branch 'develop' into view_mapping_p
2 parents 3107e20 + 41bb2bf commit d53be9c

524 files changed

Lines changed: 18409 additions & 10979 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.cargo/config.toml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,15 @@ timeout = 3600
2121
# Environment variables for PyO3. Ensures reproducible builds and avoids spurious recompilations.
2222
[env]
2323
PYO3_ENVIRONMENT_SIGNATURE = { value = "cpython-3.11-64bit", force = true }
24+
25+
[cache.global-clean]
26+
# Anything older than this duration will be deleted in the source cache.
27+
max-src-age = "1 week"
28+
# Anything older than this duration will be deleted in the compressed crate cache.
29+
max-crate-age = "1 week"
30+
# Any git checkout older than this duration will be deleted from the checkout cache.
31+
max-git-co-age = "1 week"
32+
# Any index older than this duration will be deleted from the index cache.
33+
max-index-age = "1 month"
34+
# Any git clone older than this duration will be deleted from the git cache.
35+
max-git-db-age = "1 month"

‎.github/actions/setup-rust/action.yml‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ inputs:
1919
description: "optional suffix for cache key to isolate builds with different RUSTFLAGS (e.g. 'sanitizer')"
2020
required: false
2121
default: ""
22+
timestamp:
23+
description: "Timestamp cache with cargo sweep"
24+
default: "false"
2225

2326
runs:
2427
using: "composite"
@@ -30,10 +33,8 @@ runs:
3033
DEFAULT_VERSION=$(grep channel rust-toolchain.toml | awk -F'"' '{print $2}')
3134
TOOLCHAIN="${TOOLCHAIN_OVERRIDE:-$DEFAULT_VERSION}"
3235
echo "toolchain=$TOOLCHAIN" >> $GITHUB_OUTPUT
33-
echo "targets=$TARGETS" >> $GITHUB_OUTPUT
3436
env:
3537
TOOLCHAIN_OVERRIDE: ${{ inputs.toolchain }}
36-
TARGETS: ${{ inputs.targets }}
3738

3839
- name: Install Mold
3940
if: runner.os == 'Linux'
@@ -45,14 +46,14 @@ runs:
4546
if: steps.rustup-cache.outputs.cache-hit != 'true'
4647
with:
4748
toolchain: "${{ steps.toolchain-config.outputs.toolchain }}"
48-
targets: "${{ steps.toolchain-config.outputs.targets }}"
49+
targets: "${{ inputs.targets }}"
4950
components: "${{ inputs.components || 'clippy, rustfmt' }}"
5051

5152
- name: Rust Dependency Cache
5253
uses: Swatinem/rust-cache@v2
5354
with:
5455
save-if: ${{ github.ref_name == 'develop' }}
55-
shared-key: "rust-cache-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.toolchain-config.outputs.toolchain }}-${{ steps.toolchain-config.outputs.targets }}${{ inputs.cache-suffix && format('-{0}', inputs.cache-suffix) || '' }}"
56+
shared-key: "rust-cache-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.rust-toolchain.outputs.cachekey }}-${{ inputs.targets }}${{ inputs.cache-suffix && format('-{0}', inputs.cache-suffix) || '' }}"
5657

5758
- name: Rust Compile Cache
5859
uses: mozilla-actions/[email protected]
@@ -63,3 +64,13 @@ runs:
6364
with:
6465
version: "29.3"
6566
repo-token: ${{ inputs.repo-token }}
67+
68+
- name: Install Sweep
69+
shell: bash
70+
if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }}
71+
run: cargo install cargo-sweep
72+
73+
- name: Timestamp Cache
74+
shell: bash
75+
if: ${{ inputs.timestamp == 'true' && github.ref_name == 'develop' }}
76+
run: cargo sweep --stamp

‎.github/runs-on.yml‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,3 @@ images:
99
arch: "arm64"
1010
name: "vortex-ci-*"
1111
owner: "375504701696"
12-
13-
pools:
14-
# Windows pool - covers UK (8am-6pm GMT) and US-East (8am-6pm EST)
15-
# Combined in UK timezone: 8am-11pm (UK 8-6 + NYC 8-6 = UK 8-11pm)
16-
# Stopped instances only (~20-30s boot vs 2-3min cold-start)
17-
# Cost: ~$5/month (2 × $2.40 EBS storage)
18-
windows-x64:
19-
runner: family=m7i/cpu=8/image=windows22-full-x64/tag=rust-test-windows
20-
timezone: "Europe/London"
21-
schedule:
22-
- name: working-hours
23-
match:
24-
day: ["monday", "tuesday", "wednesday", "thursday", "friday"]
25-
time: ["08:00", "23:00"]
26-
stopped: 2
27-
- name: default
28-
stopped: 1

‎.github/scripts/run-sql-bench.sh‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#!/bin/bash
2+
# SPDX-License-Identifier: Apache-2.0
3+
# SPDX-FileCopyrightText: Copyright the Vortex contributors
4+
#
5+
# Runs SQL benchmarks (datafusion-bench, duckdb-bench, lance-bench) for the given targets.
6+
# This script is used by the sql-benchmarks.yml workflow.
7+
#
8+
# Usage:
9+
# run-sql-bench.sh <subcommand> <targets> [options]
10+
#
11+
# Arguments:
12+
# subcommand The benchmark subcommand (e.g., tpch, clickbench, tpcds)
13+
# targets Comma-separated list of engine:format pairs
14+
# (e.g., "datafusion:parquet,datafusion:vortex,duckdb:parquet")
15+
#
16+
# Options:
17+
# --scale-factor <sf> Scale factor for the benchmark (e.g., 1.0, 10.0)
18+
# --remote-storage <url> Remote storage URL (e.g., s3://bucket/path/)
19+
# If provided, runs in remote mode (no lance support).
20+
# --benchmark-id <id> Benchmark ID for error messages (e.g., tpch-s3)
21+
22+
set -Eeu -o pipefail
23+
24+
subcommand="$1"
25+
targets="$2"
26+
shift 2
27+
28+
scale_factor=""
29+
remote_storage=""
30+
benchmark_id=""
31+
32+
while [[ $# -gt 0 ]]; do
33+
case "$1" in
34+
--scale-factor)
35+
scale_factor="$2"
36+
shift 2
37+
;;
38+
--remote-storage)
39+
remote_storage="$2"
40+
shift 2
41+
;;
42+
--benchmark-id)
43+
benchmark_id="$2"
44+
shift 2
45+
;;
46+
*)
47+
echo "Unknown option: $1" >&2
48+
exit 1
49+
;;
50+
esac
51+
done
52+
53+
is_remote=false
54+
if [[ -n "$remote_storage" ]]; then
55+
is_remote=true
56+
fi
57+
58+
# Lance on remote storage is not supported. The infrastructure to generate and upload lance files
59+
# to S3 does not exist. If you need lance on S3, you must first implement:
60+
# 1. Lance data generation in data-gen (or a separate step)
61+
# 2. Lance data upload to S3 before this step runs
62+
if $is_remote && echo "$targets" | grep -q 'lance'; then
63+
echo "ERROR: Lance format is not supported for remote storage benchmarks."
64+
echo "Remove 'datafusion:lance' from targets for benchmark '${benchmark_id:-unknown}'."
65+
exit 1
66+
fi
67+
68+
# Extract formats for each engine from the targets string.
69+
# Example input: "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet"
70+
#
71+
# Pipeline: split by comma -> filter by engine prefix -> remove prefix -> rejoin with commas
72+
#
73+
# Lance is filtered out of df_formats because it uses a separate binary (lance-bench).
74+
#
75+
# The `|| true` is needed because some benchmarks don't use all engines (e.g., statpopgen only has
76+
# duckdb targets). grep returns exit code 1 when no matches are found. Both greps must be in the
77+
# subshell so that `|| true` covers the case where grep -v receives empty input.
78+
df_formats=$(echo "$targets" | tr ',' '\n' | (grep '^datafusion:' | grep -v ':lance$' || true) | sed 's/datafusion://' | tr '\n' ',' | sed 's/,$//')
79+
ddb_formats=$(echo "$targets" | tr ',' '\n' | (grep '^duckdb:' || true) | sed 's/duckdb://' | tr '\n' ',' | sed 's/,$//')
80+
has_lance=$(echo "$targets" | grep -q 'datafusion:lance' && echo "true" || echo "false")
81+
82+
# Build options string.
83+
opts=""
84+
if $is_remote; then
85+
opts="--opt remote-data-dir=$remote_storage"
86+
fi
87+
if [[ -n "$scale_factor" ]]; then
88+
if [[ -n "$opts" ]]; then
89+
opts="--opt scale-factor=$scale_factor $opts"
90+
else
91+
opts="--opt scale-factor=$scale_factor"
92+
fi
93+
fi
94+
95+
touch results.json
96+
97+
if [[ -n "$df_formats" ]]; then
98+
# shellcheck disable=SC2086
99+
target/release_debug/datafusion-bench "$subcommand" \
100+
-d gh-json \
101+
--formats "$df_formats" \
102+
$opts \
103+
-o df-results.json
104+
105+
cat df-results.json >> results.json
106+
fi
107+
108+
if [[ -n "$ddb_formats" ]]; then
109+
# shellcheck disable=SC2086
110+
target/release_debug/duckdb-bench "$subcommand" \
111+
-d gh-json \
112+
--formats "$ddb_formats" \
113+
$opts \
114+
--delete-duckdb-database \
115+
-o ddb-results.json
116+
117+
cat ddb-results.json >> results.json
118+
fi
119+
120+
# Lance-bench only runs for local benchmarks.
121+
if ! $is_remote && [[ "$has_lance" == "true" ]] && [[ -f "target/release_debug/lance-bench" ]]; then
122+
# shellcheck disable=SC2086
123+
target/release_debug/lance-bench "$subcommand" \
124+
-d gh-json \
125+
$opts \
126+
-o lance-results.json
127+
128+
cat lance-results.json >> results.json
129+
fi

‎.github/workflows/bench-pr.yml‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ permissions:
1919
contents: read
2020
pull-requests: write # for commenting on PRs
2121
id-token: write # enables AWS-GitHub OIDC
22-
deployments: write # for Polar Signals profiling
2322

2423
jobs:
2524
label_trigger:
@@ -45,9 +44,9 @@ jobs:
4544
strategy:
4645
matrix:
4746
benchmark:
48-
- id: random_access
47+
- id: random-access-bench
4948
name: Random Access
50-
- id: compress
49+
- id: compress-bench
5150
name: Compression
5251
if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
5352
steps:
@@ -73,7 +72,7 @@ jobs:
7372
env:
7473
RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
7574
run: |
76-
cargo build --bin ${{ matrix.benchmark.id }} --package bench-vortex --profile release_debug
75+
cargo build --package ${{ matrix.benchmark.id }} --profile release_debug
7776
7877
- name: Setup Polar Signals
7978
if: github.event.pull_request.head.repo.fork == false
@@ -90,7 +89,7 @@ jobs:
9089
env:
9190
RUST_BACKTRACE: full
9291
run: |
93-
target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o ${{ matrix.benchmark.id }}.json
92+
target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json
9493
9594
- name: Setup AWS CLI
9695
if: github.event.pull_request.head.repo.fork == false
@@ -124,7 +123,7 @@ jobs:
124123
125124
echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md
126125
echo '' >> comment.md
127-
uv run --no-project scripts/compare-benchmark-jsons.py base.json ${{ matrix.benchmark.id }}.json "${{ matrix.benchmark.name }}" \
126+
uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \
128127
>> comment.md
129128
130129
- name: Comment PR

0 commit comments

Comments
 (0)