@@ -47,107 +47,129 @@ jobs:
4747 # Only this workflow defines the CUDA benchmark jobs.
4848 - ".github/workflows/codspeed.yml"
4949
50- bench-codspeed :
50+ # CodSpeed rejects an upload with too many benchmarks, so the simulation benchmarks run in
51+ # shards of at most 800. The planner reads only Cargo.toml files and the committed benchmark
52+ # counts, so this job needs no Rust toolchain and takes seconds.
53+ plan-codspeed :
54+ name : " Plan Codspeed shards"
5155 if : github.event.pull_request.draft != true
56+ runs-on : ubuntu-latest
57+ timeout-minutes : 5
58+ outputs :
59+ active : ${{ steps.plan.outputs.active }}
60+ shards : ${{ steps.plan.outputs.shards }}
61+ steps :
62+ - uses : actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
63+ - id : plan
64+ run : python3 scripts/codspeed_shards.py plan >> "$GITHUB_OUTPUT"
65+
66+ bench-codspeed :
67+ needs : [plan-codspeed]
5268 strategy :
5369 matrix :
54- include :
55- # Distribute both compilation and simulation work across shards. Rebalance
56- # these groups as new benchmarks are added, using CI build and run durations.
57- - { shard: 1, name: "Array compute", packages: "vortex-array", array-group: "compute", features: "--features _test-harness" }
58- - { shard: 2, name: "Array selection", packages: "vortex-array", array-group: "selection", features: "--features _test-harness" }
59- - { shard: 3, name: "Array dictionaries", packages: "vortex-array", array-group: "dictionary", features: "--features _test-harness" }
60- - { shard: 4, name: "Array builders & views", packages: "vortex-array", array-group: "builders", features: "--features _test-harness" }
61- - { shard: 5, name: "Core & encodings", packages: "vortex-buffer vortex-mask vortex-compute vortex-file vortex-alp vortex-datetime-parts" }
62- - { shard: 6, name: "Main library & storage", packages: "vortex vortex-compressor vortex-row" }
63- - { shard: 7, name: "FastLanes & decimals", packages: "vortex-decimal-byte-parts vortex-fastlanes", features: "--features _test-harness" }
64- - { shard: 8, name: "Encodings & layout", packages: "vortex-elias-fano vortex-pco vortex-sparse vortex-layout" }
65- - { shard: 9, name: "Strings, tensor & spatial", packages: "vortex-fsst vortex-tensor vortex-spatial", features: "--features _test-harness" }
66- name : " Benchmark with Codspeed (Shard #${{ matrix.shard }} - ${{ matrix.name }})"
67- timeout-minutes : 30
68- # Each CodSpeed runner is pinned to one instance type, because results depend on the CPU.
70+ # Fixed slots, so that each one can be a required check. The plan fills as many as it
71+ # needs; an unused slot runs no steps and succeeds. Keep in sync with MAX_SHARDS in
72+ # scripts/codspeed_shards.py.
73+ shard : [1, 2, 3, 4, 5]
74+ name : " Benchmark with Codspeed (Shard ${{ matrix.shard }})"
75+ timeout-minutes : 60
76+ env :
77+ SHARD_ACTIVE : ${{ contains(fromJSON(needs.plan-codspeed.outputs.active), matrix.shard) }}
78+ # Each CodSpeed runner is pinned to one instance type, because results depend on the CPU. An
79+ # unused slot takes a GitHub-hosted runner instead.
6980 runs-on : >-
70- ${{ github.repository == 'vortex-data/vortex'
81+ ${{ contains(fromJSON(needs.plan-codspeed.outputs.active), matrix.shard)
82+ && github.repository == 'vortex-data/vortex'
7183 && format('runs-on={0}/region=us-east-1/runner=amd64-codspeed-simulation/tag=bench-codspeed-{1}', github.run_id, matrix.shard)
7284 || 'ubuntu-latest' }}
7385 steps :
7486 - uses : runs-on/action@v2
75- if : github.repository == 'vortex-data/vortex'
87+ if : env.SHARD_ACTIVE == 'true' && github.repository == 'vortex-data/vortex'
7688 with :
7789 sccache : s3
78- - uses : actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
79- - uses : ./.github/actions/setup-prebuild
80- with :
81- enable-sccache : ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
82- - uses : ./.github/actions/system-info
83- - name : Install Codspeed
84- uses : taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
85- with :
86- tool : cargo-codspeed
87- - name : Select array benchmarks
88- if : matrix.array-group
89- id : select
90+ - name : Select shard
91+ id : shard
9092 env :
91- ARRAY_GROUP : ${{ matrix.array-group }}
93+ SHARDS : ${{ needs.plan-codspeed.outputs.shards }}
94+ SHARD : ${{ matrix.shard }}
9295 run : |
9396 python3 - <<'EOF' >> "$GITHUB_OUTPUT"
94- import os, pathlib, tomllib
97+ import json, os
9598
96- groups = {
97- "dictionary": ("dict_", "chunked_dict_", "patches_"),
98- "selection": ("take_", "filter_", "piecewise_sequence_"),
99- "compute": ("aggregate_", "cast_", "expr_", "binary_ops", "compare", "kleene_bool", "like", "list_sum", "widen"),
100- }
101- manifest = tomllib.loads(pathlib.Path("vortex-array/Cargo.toml").read_text())
102- benches = []
103- for bench in manifest["bench"]:
104- name = bench["name"]
105- # Unclassified and newly added targets stay covered by the builders shard.
106- group = next((group for group, prefixes in groups.items() if name.startswith(prefixes)), "builders")
107- if group == os.environ["ARRAY_GROUP"]:
108- benches.append(name)
109- if not benches:
110- raise SystemExit("array shard has no benchmark targets")
111- print("benches=" + " ".join(f"--bench {name}" for name in benches))
99+ shard = json.loads(os.environ["SHARDS"]).get(os.environ["SHARD"])
100+ if shard is None:
101+ print("active=false")
102+ else:
103+ print("active=true")
104+ print(f"build-args={shard['build-args']}")
105+ print(f"exclude-allocations={str(shard['exclude-allocations']).lower()}")
112106 EOF
113- - name : Build benchmarks
114- env :
115- RUSTFLAGS : " -C target-feature=+avx2 -C force-frame-pointers=yes -C link-arg=-fuse-ld=mold"
116- # Benchmarks carrying `#[cpu_features]` belong to a walltime leg below and are
117- # skipped here. Untagged ones run as they always have, under bare names, so this
118- # job's CodSpeed history is unaffected.
119- VORTEX_BENCH_VARIANT : simulation
120- run : cargo codspeed build --locked ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) ${{ steps.select.outputs.benches }} --profile bench
121- - name : Run benchmarks
122- uses : CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5
123- env :
124- # codspeed-divan-compat 6 runs each benchmark in a new process by default. Simulation
125- # measures one iteration without warmup, so each benchmark then also pays for a cold
126- # allocator and cold caches. This adds about 15 µs to each benchmark and hides small
127- # changes in short benchmarks. Run all benchmarks of a target in one process, as before.
128- CODSPEED_PROCESS_ISOLATION : " false"
129- # Allocators return freed memory to the OS and grow their heaps with system calls. A
130- # system call inside a measured iteration is left out of the measure, so CodSpeed reports
131- # that benchmark as dominated by system calls. Keep freed memory mapped instead.
132- #
133- # mimalloc, which every benchmark binary uses, returns memory 10 ms after it is freed.
134- # The simulation runs many times slower than native code, so that delay can end inside a
135- # later benchmark's measured iteration.
136- MIMALLOC_PURGE_DELAY : " -1"
137- # glibc `malloc`, which C dependencies such as zstd use, shrinks its heap after large
138- # frees and grows it again with `brk`. Shrink only when 1 GiB is free at the top of the
139- # heap, and grow in 64 MiB steps so that a later benchmark rarely needs to grow it.
140- MALLOC_TRIM_THRESHOLD_ : " 1073741824"
141- MALLOC_TOP_PAD_ : " 67108864"
107+ - uses : actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
108+ if : steps.shard.outputs.active == 'true'
109+ - uses : ./.github/actions/setup-prebuild
110+ if : steps.shard.outputs.active == 'true'
142111 with :
143- run : cargo codspeed run
144- token : ${{ secrets.CODSPEED_TOKEN }}
145- mode : " simulation"
146- # Exclude deferred allocator cleanup from the affected array shards.
147- # Allocation-focused benchmarks in the core shard still measure allocator time.
148- exclude-allocations : ${{ matrix.array-group == 'dictionary' || matrix.array-group == 'selection' }}
149- # Cached package restoration fails on dangling libc6-dbg documentation symlinks.
150- cache-instruments : " false"
112+ enable-sccache : ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
113+ - parallel :
114+ - uses : ./.github/actions/system-info
115+ if : steps.shard.outputs.active == 'true'
116+ - name : Install Codspeed
117+ if : steps.shard.outputs.active == 'true'
118+ uses : taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
119+ with :
120+ tool : cargo-codspeed
121+ # The CodSpeed action does its setup (CLI install, its Valgrind package, runner start-up)
122+ # inside the step that runs the benchmarks, so it starts alongside the build and its
123+ # command waits for the build to report an exit status before running anything.
124+ - parallel :
125+ - name : Build benchmarks
126+ if : steps.shard.outputs.active == 'true'
127+ env :
128+ RUSTFLAGS : " -C target-feature=+avx2 -C force-frame-pointers=yes -C link-arg=-fuse-ld=mold"
129+ # Benchmarks carrying `#[cpu_features]` belong to a walltime leg below and are
130+ # skipped here. Untagged ones run as they always have, under bare names, so this
131+ # job's CodSpeed history is unaffected.
132+ VORTEX_BENCH_VARIANT : simulation
133+ BUILD_STATUS : ${{ runner.temp }}/codspeed-build-status
134+ run : |
135+ trap 'echo $? > "$BUILD_STATUS"' EXIT
136+ cargo codspeed build --locked ${{ steps.shard.outputs.build-args }} --profile bench
137+ - name : Run benchmarks
138+ if : steps.shard.outputs.active == 'true'
139+ uses : CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5
140+ env :
141+ # codspeed-divan-compat 6 runs each benchmark in a new process by default. Simulation
142+ # measures one iteration without warmup, so each benchmark then also pays for a cold
143+ # allocator and cold caches. This adds about 15 µs to each benchmark and hides small
144+ # changes in short benchmarks. Run all benchmarks of a target in one process, as before.
145+ CODSPEED_PROCESS_ISOLATION : " false"
146+ # Allocators return freed memory to the OS and grow their heaps with system calls. A
147+ # system call inside a measured iteration is left out of the measure, so CodSpeed reports
148+ # that benchmark as dominated by system calls. Keep freed memory mapped instead.
149+ #
150+ # mimalloc, which every benchmark binary uses, returns memory 10 ms after it is freed.
151+ # The simulation runs many times slower than native code, so that delay can end inside a
152+ # later benchmark's measured iteration.
153+ MIMALLOC_PURGE_DELAY : " -1"
154+ # glibc `malloc`, which C dependencies such as zstd use, shrinks its heap after large
155+ # frees and grows it again with `brk`. Shrink only when 1 GiB is free at the top of the
156+ # heap, and grow in 64 MiB steps so that a later benchmark rarely needs to grow it.
157+ MALLOC_TRIM_THRESHOLD_ : " 1073741824"
158+ MALLOC_TOP_PAD_ : " 67108864"
159+ with :
160+ run : |
161+ BUILD_STATUS='${{ runner.temp }}/codspeed-build-status'
162+ until [ -s "$BUILD_STATUS" ]; do sleep 2; done
163+ if [ "$(cat "$BUILD_STATUS")" != 0 ]; then
164+ echo "Benchmark build failed; nothing to run." >&2
165+ exit 1
166+ fi
167+ cargo codspeed run
168+ token : ${{ secrets.CODSPEED_TOKEN }}
169+ mode : " simulation"
170+ exclude-allocations : ${{ steps.shard.outputs.exclude-allocations }}
171+ # Cached package restoration fails on dangling libc6-dbg documentation symlinks.
172+ cache-instruments : " false"
151173
152174 bench-codspeed-python :
153175 name : " Benchmark with Codspeed (Python)"
@@ -176,28 +198,41 @@ jobs:
176198 with :
177199 enable-sccache : ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
178200 - uses : ./.github/actions/system-info
179- # Build the extension before measurement, so the benchmark step does not compile it.
180- - name : Build Python extension
181- run : uv sync --all-packages
182- # Simulation runs all of the benchmark process on Valgrind, so write the input files first.
183- - name : Write benchmark data
184- working-directory : vortex-python
185- env :
186- VORTEX_BENCH_DATA_DIR : ${{ runner.temp }}/vortex-bench-data
187- run : uv run --all-packages --no-sync python -m benchmark.data "$VORTEX_BENCH_DATA_DIR"
188- - name : Run benchmarks
189- uses : CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5
190- env :
191- VORTEX_BENCH_DATA_DIR : ${{ runner.temp }}/vortex-bench-data
192- # Drive scans on the calling thread. Background workers wake every 100 ms to check
193- # for shutdown. Under simulation, those timer wakeups can dominate the measured work.
194- VORTEX_MAX_THREADS : " 0"
195- with :
196- run : cd vortex-python && uv run --all-packages --no-sync pytest benchmark/ --codspeed
197- token : ${{ secrets.CODSPEED_TOKEN }}
198- mode : " simulation"
199- # Cached package restoration fails on dangling libc6-dbg documentation symlinks.
200- cache-instruments : " false"
201+ # As in `bench-codspeed`, the CodSpeed action's setup runs alongside the build, and its
202+ # command waits for the build to report an exit status.
203+ - parallel :
204+ # Build the extension before measurement, so the benchmark step does not compile it.
205+ # Simulation runs all of the benchmark process on Valgrind, so write the input files
206+ # first too.
207+ - name : Build Python extension and write benchmark data
208+ env :
209+ VORTEX_BENCH_DATA_DIR : ${{ runner.temp }}/vortex-bench-data
210+ BUILD_STATUS : ${{ runner.temp }}/codspeed-build-status
211+ run : |
212+ trap 'echo $? > "$BUILD_STATUS"' EXIT
213+ uv sync --all-packages
214+ cd vortex-python
215+ uv run --all-packages --no-sync python -m benchmark.data "$VORTEX_BENCH_DATA_DIR"
216+ - name : Run benchmarks
217+ uses : CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5
218+ env :
219+ VORTEX_BENCH_DATA_DIR : ${{ runner.temp }}/vortex-bench-data
220+ # Drive scans on the calling thread. Background workers wake every 100 ms to check
221+ # for shutdown. Under simulation, those timer wakeups can dominate the measured work.
222+ VORTEX_MAX_THREADS : " 0"
223+ with :
224+ run : |
225+ BUILD_STATUS='${{ runner.temp }}/codspeed-build-status'
226+ until [ -s "$BUILD_STATUS" ]; do sleep 2; done
227+ if [ "$(cat "$BUILD_STATUS")" != 0 ]; then
228+ echo "Python extension build failed; nothing to run." >&2
229+ exit 1
230+ fi
231+ cd vortex-python && uv run --all-packages --no-sync pytest benchmark/ --codspeed
232+ token : ${{ secrets.CODSPEED_TOKEN }}
233+ mode : " simulation"
234+ # Cached package restoration fails on dangling libc6-dbg documentation symlinks.
235+ cache-instruments : " false"
201236
202237 # Compile on smaller VMs from the same CPU family as the measurement hosts. Globally
203238 # enabled features also affect build scripts, so the build host must support them.
@@ -407,16 +442,10 @@ jobs:
407442 bench-codspeed-cuda :
408443 if : github.repository == 'vortex-data/vortex'
409444 needs : [bench-codspeed-cuda-build]
410- strategy :
411- matrix :
412- include :
413- - { shard: 1, name: "Bitpacked", benches: "bitpacked_cuda" }
414- - { shard: 2, name: "Dynamic dispatch", benches: "dynamic_dispatch_cuda" }
415- - { shard: 3, name: "Standalone kernels", benches: "alp_cuda date_time_parts_cuda delta_cuda dict_cuda fsst_cuda runend_cuda" }
416- name : " Benchmark with Codspeed (CUDA Shard #${{ matrix.shard }} - ${{ matrix.name }})"
417- timeout-minutes : 30
445+ name : " Benchmark with Codspeed (CUDA)"
446+ timeout-minutes : 60
418447 runs-on : >-
419- runs-on=${{ github.run_id }}/region=us-east-1/runner=gpu-codspeed/tag=bench-codspeed-cuda-${{ matrix.shard }}
448+ runs-on=${{ github.run_id }}/region=us-east-1/runner=gpu-codspeed/tag=bench-codspeed-cuda
420449 steps :
421450 - uses : runs-on/action@v2
422451 with :
@@ -450,6 +479,12 @@ jobs:
450479 env :
451480 CARGO_MANIFEST_DIR : ${{ github.workspace }}/vortex-cuda
452481 with :
453- run : cargo codspeed run $(printf -- '--bench %s ' ${{ matrix.benches }})
482+ # Name the benchmarks: the artifact directory also holds libvortex_cub.so, which a bare
483+ # `cargo codspeed run` would try to execute as a benchmark.
484+ run : >-
485+ cargo codspeed run
486+ --bench bitpacked_cuda --bench dynamic_dispatch_cuda --bench alp_cuda
487+ --bench date_time_parts_cuda --bench delta_cuda --bench dict_cuda
488+ --bench fsst_cuda --bench runend_cuda
454489 token : ${{ secrets.CODSPEED_TOKEN }}
455490 mode : " walltime"
0 commit comments