Skip to content

Commit 3039ba1

Browse files
ci: shard CodSpeed simulation benchmarks dynamically
Signed-off-by: Joe Isaacs <[email protected]>
1 parent 172ea59 commit 3039ba1

5 files changed

Lines changed: 689 additions & 119 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,14 @@ jobs:
9898
# benchmarks-database fact row. The golden vectors are a frozen artifact (see the note inside
9999
# the JSON); this job fails if a change to scripts/_measurement_id.py drifts the hash, which
100100
# would break ON CONFLICT idempotency against the rows already in production.
101-
- name: Pytest - measurement_id, benchmark reporting and ingestion
101+
- name: Pytest - measurement_id, benchmark reporting, ingestion and CodSpeed shards
102102
if: "!cancelled()"
103103
run: |
104104
uv run --no-project --with pytest --with pandas --with numpy --with orjson \
105105
--with tabulate --with xxhash \
106106
pytest scripts/tests/test_measurement_id.py \
107-
scripts/tests/test_benchmark_reporting.py scripts/tests/test_post_ingest.py
107+
scripts/tests/test_benchmark_reporting.py scripts/tests/test_post_ingest.py \
108+
scripts/tests/test_codspeed_shards.py
108109
109110
python-test:
110111
name: "Python (test)"

‎.github/workflows/codspeed.yml‎

Lines changed: 152 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -47,107 +47,129 @@ jobs:
4747
# Only this workflow defines the CUDA benchmark jobs.
4848
- ".github/workflows/codspeed.yml"
4949
50-
bench-codspeed:
50+
# CodSpeed rejects an upload with too many benchmarks, so the simulation benchmarks run in
51+
# shards of at most 800. The planner reads only Cargo.toml files and the committed benchmark
52+
# counts, so this job needs no Rust toolchain and takes seconds.
53+
plan-codspeed:
54+
name: "Plan Codspeed shards"
5155
if: github.event.pull_request.draft != true
56+
runs-on: ubuntu-latest
57+
timeout-minutes: 5
58+
outputs:
59+
active: ${{ steps.plan.outputs.active }}
60+
shards: ${{ steps.plan.outputs.shards }}
61+
steps:
62+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
63+
- id: plan
64+
run: python3 scripts/codspeed_shards.py plan >> "$GITHUB_OUTPUT"
65+
66+
bench-codspeed:
67+
needs: [plan-codspeed]
5268
strategy:
5369
matrix:
54-
include:
55-
# Distribute both compilation and simulation work across shards. Rebalance
56-
# these groups as new benchmarks are added, using CI build and run durations.
57-
- { shard: 1, name: "Array compute", packages: "vortex-array", array-group: "compute", features: "--features _test-harness" }
58-
- { shard: 2, name: "Array selection", packages: "vortex-array", array-group: "selection", features: "--features _test-harness" }
59-
- { shard: 3, name: "Array dictionaries", packages: "vortex-array", array-group: "dictionary", features: "--features _test-harness" }
60-
- { shard: 4, name: "Array builders & views", packages: "vortex-array", array-group: "builders", features: "--features _test-harness" }
61-
- { shard: 5, name: "Core & encodings", packages: "vortex-buffer vortex-mask vortex-compute vortex-file vortex-alp vortex-datetime-parts" }
62-
- { shard: 6, name: "Main library & storage", packages: "vortex vortex-compressor vortex-row" }
63-
- { shard: 7, name: "FastLanes & decimals", packages: "vortex-decimal-byte-parts vortex-fastlanes", features: "--features _test-harness" }
64-
- { shard: 8, name: "Encodings & layout", packages: "vortex-elias-fano vortex-pco vortex-sparse vortex-layout" }
65-
- { shard: 9, name: "Strings, tensor & spatial", packages: "vortex-fsst vortex-tensor vortex-spatial", features: "--features _test-harness" }
66-
name: "Benchmark with Codspeed (Shard #${{ matrix.shard }} - ${{ matrix.name }})"
67-
timeout-minutes: 30
68-
# Each CodSpeed runner is pinned to one instance type, because results depend on the CPU.
70+
# Fixed slots, so that each one can be a required check. The plan fills as many as it
71+
# needs; an unused slot runs no steps and succeeds. Keep in sync with MAX_SHARDS in
72+
# scripts/codspeed_shards.py.
73+
shard: [1, 2, 3, 4, 5]
74+
name: "Benchmark with Codspeed (Shard ${{ matrix.shard }})"
75+
timeout-minutes: 60
76+
env:
77+
SHARD_ACTIVE: ${{ contains(fromJSON(needs.plan-codspeed.outputs.active), matrix.shard) }}
78+
# Each CodSpeed runner is pinned to one instance type, because results depend on the CPU. An
79+
# unused slot takes a GitHub-hosted runner instead.
6980
runs-on: >-
70-
${{ github.repository == 'vortex-data/vortex'
81+
${{ contains(fromJSON(needs.plan-codspeed.outputs.active), matrix.shard)
82+
&& github.repository == 'vortex-data/vortex'
7183
&& format('runs-on={0}/region=us-east-1/runner=amd64-codspeed-simulation/tag=bench-codspeed-{1}', github.run_id, matrix.shard)
7284
|| 'ubuntu-latest' }}
7385
steps:
7486
- uses: runs-on/action@v2
75-
if: github.repository == 'vortex-data/vortex'
87+
if: env.SHARD_ACTIVE == 'true' && github.repository == 'vortex-data/vortex'
7688
with:
7789
sccache: s3
78-
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
79-
- uses: ./.github/actions/setup-prebuild
80-
with:
81-
enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
82-
- uses: ./.github/actions/system-info
83-
- name: Install Codspeed
84-
uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
85-
with:
86-
tool: cargo-codspeed
87-
- name: Select array benchmarks
88-
if: matrix.array-group
89-
id: select
90+
- name: Select shard
91+
id: shard
9092
env:
91-
ARRAY_GROUP: ${{ matrix.array-group }}
93+
SHARDS: ${{ needs.plan-codspeed.outputs.shards }}
94+
SHARD: ${{ matrix.shard }}
9295
run: |
9396
python3 - <<'EOF' >> "$GITHUB_OUTPUT"
94-
import os, pathlib, tomllib
97+
import json, os
9598
96-
groups = {
97-
"dictionary": ("dict_", "chunked_dict_", "patches_"),
98-
"selection": ("take_", "filter_", "piecewise_sequence_"),
99-
"compute": ("aggregate_", "cast_", "expr_", "binary_ops", "compare", "kleene_bool", "like", "list_sum", "widen"),
100-
}
101-
manifest = tomllib.loads(pathlib.Path("vortex-array/Cargo.toml").read_text())
102-
benches = []
103-
for bench in manifest["bench"]:
104-
name = bench["name"]
105-
# Unclassified and newly added targets stay covered by the builders shard.
106-
group = next((group for group, prefixes in groups.items() if name.startswith(prefixes)), "builders")
107-
if group == os.environ["ARRAY_GROUP"]:
108-
benches.append(name)
109-
if not benches:
110-
raise SystemExit("array shard has no benchmark targets")
111-
print("benches=" + " ".join(f"--bench {name}" for name in benches))
99+
shard = json.loads(os.environ["SHARDS"]).get(os.environ["SHARD"])
100+
if shard is None:
101+
print("active=false")
102+
else:
103+
print("active=true")
104+
print(f"build-args={shard['build-args']}")
105+
print(f"exclude-allocations={str(shard['exclude-allocations']).lower()}")
112106
EOF
113-
- name: Build benchmarks
114-
env:
115-
RUSTFLAGS: "-C target-feature=+avx2 -C force-frame-pointers=yes -C link-arg=-fuse-ld=mold"
116-
# Benchmarks carrying `#[cpu_features]` belong to a walltime leg below and are
117-
# skipped here. Untagged ones run as they always have, under bare names, so this
118-
# job's CodSpeed history is unaffected.
119-
VORTEX_BENCH_VARIANT: simulation
120-
run: cargo codspeed build --locked ${{ matrix.features }} $(printf -- '-p %s ' ${{ matrix.packages }}) ${{ steps.select.outputs.benches }} --profile bench
121-
- name: Run benchmarks
122-
uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5
123-
env:
124-
# codspeed-divan-compat 6 runs each benchmark in a new process by default. Simulation
125-
# measures one iteration without warmup, so each benchmark then also pays for a cold
126-
# allocator and cold caches. This adds about 15 µs to each benchmark and hides small
127-
# changes in short benchmarks. Run all benchmarks of a target in one process, as before.
128-
CODSPEED_PROCESS_ISOLATION: "false"
129-
# Allocators return freed memory to the OS and grow their heaps with system calls. A
130-
# system call inside a measured iteration is left out of the measure, so CodSpeed reports
131-
# that benchmark as dominated by system calls. Keep freed memory mapped instead.
132-
#
133-
# mimalloc, which every benchmark binary uses, returns memory 10 ms after it is freed.
134-
# The simulation runs many times slower than native code, so that delay can end inside a
135-
# later benchmark's measured iteration.
136-
MIMALLOC_PURGE_DELAY: "-1"
137-
# glibc `malloc`, which C dependencies such as zstd use, shrinks its heap after large
138-
# frees and grows it again with `brk`. Shrink only when 1 GiB is free at the top of the
139-
# heap, and grow in 64 MiB steps so that a later benchmark rarely needs to grow it.
140-
MALLOC_TRIM_THRESHOLD_: "1073741824"
141-
MALLOC_TOP_PAD_: "67108864"
107+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
108+
if: steps.shard.outputs.active == 'true'
109+
- uses: ./.github/actions/setup-prebuild
110+
if: steps.shard.outputs.active == 'true'
142111
with:
143-
run: cargo codspeed run
144-
token: ${{ secrets.CODSPEED_TOKEN }}
145-
mode: "simulation"
146-
# Exclude deferred allocator cleanup from the affected array shards.
147-
# Allocation-focused benchmarks in the core shard still measure allocator time.
148-
exclude-allocations: ${{ matrix.array-group == 'dictionary' || matrix.array-group == 'selection' }}
149-
# Cached package restoration fails on dangling libc6-dbg documentation symlinks.
150-
cache-instruments: "false"
112+
enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
113+
- parallel:
114+
- uses: ./.github/actions/system-info
115+
if: steps.shard.outputs.active == 'true'
116+
- name: Install Codspeed
117+
if: steps.shard.outputs.active == 'true'
118+
uses: taiki-e/cache-cargo-install-action@66c9585ef5ca780ee69399975a5e911f47905995
119+
with:
120+
tool: cargo-codspeed
121+
# The CodSpeed action does its setup (CLI install, its Valgrind package, runner start-up)
122+
# inside the step that runs the benchmarks, so it starts alongside the build and its
123+
# command waits for the build to report an exit status before running anything.
124+
- parallel:
125+
- name: Build benchmarks
126+
if: steps.shard.outputs.active == 'true'
127+
env:
128+
RUSTFLAGS: "-C target-feature=+avx2 -C force-frame-pointers=yes -C link-arg=-fuse-ld=mold"
129+
# Benchmarks carrying `#[cpu_features]` belong to a walltime leg below and are
130+
# skipped here. Untagged ones run as they always have, under bare names, so this
131+
# job's CodSpeed history is unaffected.
132+
VORTEX_BENCH_VARIANT: simulation
133+
BUILD_STATUS: ${{ runner.temp }}/codspeed-build-status
134+
run: |
135+
trap 'echo $? > "$BUILD_STATUS"' EXIT
136+
cargo codspeed build --locked ${{ steps.shard.outputs.build-args }} --profile bench
137+
- name: Run benchmarks
138+
if: steps.shard.outputs.active == 'true'
139+
uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5
140+
env:
141+
# codspeed-divan-compat 6 runs each benchmark in a new process by default. Simulation
142+
# measures one iteration without warmup, so each benchmark then also pays for a cold
143+
# allocator and cold caches. This adds about 15 µs to each benchmark and hides small
144+
# changes in short benchmarks. Run all benchmarks of a target in one process, as before.
145+
CODSPEED_PROCESS_ISOLATION: "false"
146+
# Allocators return freed memory to the OS and grow their heaps with system calls. A
147+
# system call inside a measured iteration is left out of the measure, so CodSpeed reports
148+
# that benchmark as dominated by system calls. Keep freed memory mapped instead.
149+
#
150+
# mimalloc, which every benchmark binary uses, returns memory 10 ms after it is freed.
151+
# The simulation runs many times slower than native code, so that delay can end inside a
152+
# later benchmark's measured iteration.
153+
MIMALLOC_PURGE_DELAY: "-1"
154+
# glibc `malloc`, which C dependencies such as zstd use, shrinks its heap after large
155+
# frees and grows it again with `brk`. Shrink only when 1 GiB is free at the top of the
156+
# heap, and grow in 64 MiB steps so that a later benchmark rarely needs to grow it.
157+
MALLOC_TRIM_THRESHOLD_: "1073741824"
158+
MALLOC_TOP_PAD_: "67108864"
159+
with:
160+
run: |
161+
BUILD_STATUS='${{ runner.temp }}/codspeed-build-status'
162+
until [ -s "$BUILD_STATUS" ]; do sleep 2; done
163+
if [ "$(cat "$BUILD_STATUS")" != 0 ]; then
164+
echo "Benchmark build failed; nothing to run." >&2
165+
exit 1
166+
fi
167+
cargo codspeed run
168+
token: ${{ secrets.CODSPEED_TOKEN }}
169+
mode: "simulation"
170+
exclude-allocations: ${{ steps.shard.outputs.exclude-allocations }}
171+
# Cached package restoration fails on dangling libc6-dbg documentation symlinks.
172+
cache-instruments: "false"
151173

152174
bench-codspeed-python:
153175
name: "Benchmark with Codspeed (Python)"
@@ -176,28 +198,41 @@ jobs:
176198
with:
177199
enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }}
178200
- uses: ./.github/actions/system-info
179-
# Build the extension before measurement, so the benchmark step does not compile it.
180-
- name: Build Python extension
181-
run: uv sync --all-packages
182-
# Simulation runs all of the benchmark process on Valgrind, so write the input files first.
183-
- name: Write benchmark data
184-
working-directory: vortex-python
185-
env:
186-
VORTEX_BENCH_DATA_DIR: ${{ runner.temp }}/vortex-bench-data
187-
run: uv run --all-packages --no-sync python -m benchmark.data "$VORTEX_BENCH_DATA_DIR"
188-
- name: Run benchmarks
189-
uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5
190-
env:
191-
VORTEX_BENCH_DATA_DIR: ${{ runner.temp }}/vortex-bench-data
192-
# Drive scans on the calling thread. Background workers wake every 100 ms to check
193-
# for shutdown. Under simulation, those timer wakeups can dominate the measured work.
194-
VORTEX_MAX_THREADS: "0"
195-
with:
196-
run: cd vortex-python && uv run --all-packages --no-sync pytest benchmark/ --codspeed
197-
token: ${{ secrets.CODSPEED_TOKEN }}
198-
mode: "simulation"
199-
# Cached package restoration fails on dangling libc6-dbg documentation symlinks.
200-
cache-instruments: "false"
201+
# As in `bench-codspeed`, the CodSpeed action's setup runs alongside the build, and its
202+
# command waits for the build to report an exit status.
203+
- parallel:
204+
# Build the extension before measurement, so the benchmark step does not compile it.
205+
# Simulation runs all of the benchmark process on Valgrind, so write the input files
206+
# first too.
207+
- name: Build Python extension and write benchmark data
208+
env:
209+
VORTEX_BENCH_DATA_DIR: ${{ runner.temp }}/vortex-bench-data
210+
BUILD_STATUS: ${{ runner.temp }}/codspeed-build-status
211+
run: |
212+
trap 'echo $? > "$BUILD_STATUS"' EXIT
213+
uv sync --all-packages
214+
cd vortex-python
215+
uv run --all-packages --no-sync python -m benchmark.data "$VORTEX_BENCH_DATA_DIR"
216+
- name: Run benchmarks
217+
uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5
218+
env:
219+
VORTEX_BENCH_DATA_DIR: ${{ runner.temp }}/vortex-bench-data
220+
# Drive scans on the calling thread. Background workers wake every 100 ms to check
221+
# for shutdown. Under simulation, those timer wakeups can dominate the measured work.
222+
VORTEX_MAX_THREADS: "0"
223+
with:
224+
run: |
225+
BUILD_STATUS='${{ runner.temp }}/codspeed-build-status'
226+
until [ -s "$BUILD_STATUS" ]; do sleep 2; done
227+
if [ "$(cat "$BUILD_STATUS")" != 0 ]; then
228+
echo "Python extension build failed; nothing to run." >&2
229+
exit 1
230+
fi
231+
cd vortex-python && uv run --all-packages --no-sync pytest benchmark/ --codspeed
232+
token: ${{ secrets.CODSPEED_TOKEN }}
233+
mode: "simulation"
234+
# Cached package restoration fails on dangling libc6-dbg documentation symlinks.
235+
cache-instruments: "false"
201236

202237
# Compile on smaller VMs from the same CPU family as the measurement hosts. Globally
203238
# enabled features also affect build scripts, so the build host must support them.
@@ -407,16 +442,10 @@ jobs:
407442
bench-codspeed-cuda:
408443
if: github.repository == 'vortex-data/vortex'
409444
needs: [bench-codspeed-cuda-build]
410-
strategy:
411-
matrix:
412-
include:
413-
- { shard: 1, name: "Bitpacked", benches: "bitpacked_cuda" }
414-
- { shard: 2, name: "Dynamic dispatch", benches: "dynamic_dispatch_cuda" }
415-
- { shard: 3, name: "Standalone kernels", benches: "alp_cuda date_time_parts_cuda delta_cuda dict_cuda fsst_cuda runend_cuda" }
416-
name: "Benchmark with Codspeed (CUDA Shard #${{ matrix.shard }} - ${{ matrix.name }})"
417-
timeout-minutes: 30
445+
name: "Benchmark with Codspeed (CUDA)"
446+
timeout-minutes: 60
418447
runs-on: >-
419-
runs-on=${{ github.run_id }}/region=us-east-1/runner=gpu-codspeed/tag=bench-codspeed-cuda-${{ matrix.shard }}
448+
runs-on=${{ github.run_id }}/region=us-east-1/runner=gpu-codspeed/tag=bench-codspeed-cuda
420449
steps:
421450
- uses: runs-on/action@v2
422451
with:
@@ -450,6 +479,12 @@ jobs:
450479
env:
451480
CARGO_MANIFEST_DIR: ${{ github.workspace }}/vortex-cuda
452481
with:
453-
run: cargo codspeed run $(printf -- '--bench %s ' ${{ matrix.benches }})
482+
# Name the benchmarks: the artifact directory also holds libvortex_cub.so, which a bare
483+
# `cargo codspeed run` would try to execute as a benchmark.
484+
run: >-
485+
cargo codspeed run
486+
--bench bitpacked_cuda --bench dynamic_dispatch_cuda --bench alp_cuda
487+
--bench date_time_parts_cuda --bench delta_cuda --bench dict_cuda
488+
--bench fsst_cuda --bench runend_cuda
454489
token: ${{ secrets.CODSPEED_TOKEN }}
455490
mode: "walltime"

0 commit comments

Comments
 (0)