Skip to content

Commit e6344fc

Browse files
kushudaij178
kushudai
and
j178
authored
Add a workflow for hyperfine benchmarks (#1304)
Link to uploaded benchmark: https://github.com/j178/prek/actions/runs/22839343954/artifacts/5823487446 Cannot test the comment workflow until it is merged. Fixes #992 --------- Co-authored-by: Jo <[email protected]>
1 parent 423e847 commit e6344fc

3 files changed

Lines changed: 380 additions & 1 deletion

File tree

‎.github/scripts/hyperfine-run-benchmarks.sh‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
TARGET_WORKSPACE=${HYPERFINE_BENCHMARK_WORKSPACE:?HYPERFINE_BENCHMARK_WORKSPACE is required}
5+
COMMENT=${HYPERFINE_RESULTS_FILE:?HYPERFINE_RESULTS_FILE is required}
6+
HEAD_BINARY=${HYPERFINE_HEAD_BINARY:?HYPERFINE_HEAD_BINARY is required}
7+
BASE_BINARY=${HYPERFINE_BASE_BINARY:?HYPERFINE_BASE_BINARY is required}
8+
REPO_WORKSPACE=$(pwd)
9+
OUT_DIR=$(dirname "$COMMENT")
10+
META_WORKSPACE="${TARGET_WORKSPACE}-meta"
11+
12+
failed=false
13+
14+
mkdir -p "$OUT_DIR"
15+
OUT_MD="$OUT_DIR/out.md"
16+
OUT_JSON="$OUT_DIR/out.json"
17+
18+
CURRENT_PREK_VERSION=$(
19+
"$HEAD_BINARY" --version | sed -n '1p'
20+
)
21+
22+
write_line() {
23+
printf '%s\n' "$1" >> "$COMMENT"
24+
}
25+
26+
write_blank_line() {
27+
printf '\n' >> "$COMMENT"
28+
}
29+
30+
write_section() {
31+
local title="$1"
32+
local description="${2:-}"
33+
34+
write_blank_line
35+
write_line "## $title"
36+
if [ -n "$description" ]; then
37+
write_line "$description"
38+
fi
39+
}
40+
41+
# Compare the two commands in out.json (reference vs current).
42+
# Hyperfine's JSON has results[0] = reference and results[1] = current.
43+
# A ratio > 1 means current is slower (regression), < 1 means faster (improvement).
44+
check_variance() {
45+
local cmd="$1"
46+
local num_results
47+
num_results=$(jq '.results | length' "$OUT_JSON")
48+
49+
if [ "$num_results" -lt 2 ]; then
50+
return
51+
fi
52+
53+
local ref_mean current_mean ratio pct
54+
ref_mean=$(jq '.results[0].mean' "$OUT_JSON")
55+
current_mean=$(jq '.results[1].mean' "$OUT_JSON")
56+
ratio=$(echo "scale=4; $current_mean / $ref_mean" | bc)
57+
pct=$(echo "scale=2; ($ratio - 1) * 100" | bc)
58+
59+
if (( $(echo "${pct#-} > 10" | bc -l) )); then
60+
if (( $(echo "$ratio < 1" | bc -l) )); then
61+
write_line "✅ Performance improvement for \`$cmd\`: ${pct#-}% faster"
62+
else
63+
write_line "⚠️ Warning: Performance regression for \`$cmd\`: ${pct}% slower"
64+
failed=true
65+
fi
66+
fi
67+
}
68+
69+
write_benchmark_details() {
70+
write_line "<details>"
71+
write_line "<summary>Benchmark details</summary>"
72+
write_blank_line
73+
cat "$OUT_MD" >> "$COMMENT"
74+
write_blank_line
75+
write_line "</details>"
76+
}
77+
78+
benchmark() {
79+
local label="$1"
80+
local cmd="$2"
81+
local warmup="${3:-3}"
82+
local runs="${4:-30}"
83+
local setup="${5:-}"
84+
local prepare="${6:-}"
85+
local check_change="${7:-false}"
86+
local -a hyperfine_args=(-i -N -w "$warmup" -r "$runs" --export-markdown "$OUT_MD" --export-json "$OUT_JSON" --show-output)
87+
88+
if [ -n "$setup" ]; then
89+
hyperfine_args+=(--setup "$setup")
90+
fi
91+
92+
if [ -n "$prepare" ]; then
93+
hyperfine_args+=(--prepare "$prepare")
94+
fi
95+
96+
write_line "### \`$label\`"
97+
if ! hyperfine "${hyperfine_args[@]}" --reference "$BASE_BINARY $cmd" "$HEAD_BINARY $cmd"; then
98+
write_line "⚠️ Benchmark failed for: $cmd"
99+
return 1
100+
fi
101+
write_benchmark_details
102+
if [ "$check_change" = "true" ]; then
103+
check_variance "$cmd"
104+
fi
105+
}
106+
107+
create_meta_workspace() {
108+
rm -rf "$META_WORKSPACE"
109+
mkdir -p "$META_WORKSPACE"
110+
cd "$META_WORKSPACE"
111+
git init || { echo "Failed to init git for meta hooks"; exit 1; }
112+
git config user.name "Benchmark"
113+
git config user.email "[email protected]"
114+
115+
cp "$TARGET_WORKSPACE"/*.txt "$TARGET_WORKSPACE"/*.json . 2>/dev/null || true
116+
117+
cat > .pre-commit-config.yaml << 'EOF'
118+
repos:
119+
- repo: meta
120+
hooks:
121+
- id: check-hooks-apply
122+
- id: check-useless-excludes
123+
- id: identity
124+
- repo: builtin
125+
hooks:
126+
- id: trailing-whitespace
127+
- id: end-of-file-fixer
128+
EOF
129+
130+
git add -A
131+
git commit -m "Meta hooks test" || { echo "Failed to commit meta hooks test"; exit 1; }
132+
prek install-hooks
133+
}
134+
135+
# Add environment metadata
136+
write_line "## Hyperfine Performance"
137+
write_blank_line
138+
write_line "**Environment:**"
139+
write_line "- OS: $(uname -s) $(uname -r)"
140+
write_line "- CPU: $(nproc) cores"
141+
write_line "- prek version: $CURRENT_PREK_VERSION"
142+
write_line "- Rust version: $(rustc --version)"
143+
write_line "- Hyperfine version: $(hyperfine --version)"
144+
145+
# Benchmark in the main repo
146+
CMDS=(
147+
"--version"
148+
"list"
149+
"validate-config .pre-commit-config.yaml"
150+
"sample-config"
151+
)
152+
for cmd in "${CMDS[@]}"; do
153+
if [[ "$cmd" == "validate-config"* ]] && [ ! -f ".pre-commit-config.yaml" ]; then
154+
write_line "### \`prek $cmd\`"
155+
write_line "⏭️ Skipped: .pre-commit-config.yaml not found"
156+
continue
157+
fi
158+
159+
if [[ "$cmd" == "--version" ]] || [[ "$cmd" == "list" ]]; then
160+
benchmark "prek $cmd" "$cmd" 5 100
161+
else
162+
benchmark "prek $cmd" "$cmd" 3 50
163+
fi
164+
check_variance "$cmd"
165+
done
166+
167+
# Benchmark builtin hooks in test directory
168+
cd "$TARGET_WORKSPACE"
169+
170+
# Cold vs warm benchmarks before polluting cache
171+
write_section "Cold vs Warm Runs" "Comparing first run (cold) vs subsequent runs (warm cache):"
172+
benchmark "prek run --all-files (cold - no cache)" "run --all-files" 0 10 "rm -rf ~/.cache/prek" "git checkout -- ."
173+
benchmark "prek run --all-files (warm - with cache)" "run --all-files" 3 20 "" "git checkout -- ."
174+
175+
# Full benchmark suite with cache warmed up
176+
write_section "Full Hook Suite" "Running the builtin hook suite on the benchmark workspace:"
177+
benchmark "prek run --all-files (full builtin hook suite)" "run --all-files" 3 50 "" "git checkout -- ." true
178+
179+
# Individual hook performance
180+
write_section "Individual Hook Performance" "Benchmarking each hook individually on the test repo:"
181+
182+
INDIVIDUAL_HOOKS=(
183+
"trailing-whitespace"
184+
"end-of-file-fixer"
185+
"check-json"
186+
"check-yaml"
187+
"check-toml"
188+
"check-xml"
189+
)
190+
191+
for hook in "${INDIVIDUAL_HOOKS[@]}"; do
192+
benchmark "prek run $hook --all-files" "run $hook --all-files" 3 30 "" "git checkout -- ."
193+
done
194+
195+
# Installation performance
196+
write_section "Installation Performance" "Benchmarking hook installation (fast path hooks skip Python setup):"
197+
benchmark "prek install-hooks (cold - no cache)" "install-hooks" 1 5 "rm -rf ~/.cache/prek/hooks ~/.cache/prek/repos"
198+
benchmark "prek install-hooks (warm - with cache)" "install-hooks" 1 5
199+
200+
# File filtering/scoping performance
201+
write_section "File Filtering/Scoping Performance" "Testing different file selection modes:"
202+
203+
git add -A
204+
benchmark "prek run (staged files only)" "run" 3 20 "" "sh -c 'git checkout -- . && git add -A'"
205+
benchmark "prek run --files '*.json' (specific file type)" "run --files '*.json'" 3 20
206+
207+
# Workspace discovery & initialization
208+
write_section "Workspace Discovery & Initialization" "Benchmarking hook discovery and initialization overhead:"
209+
benchmark "prek run --dry-run --all-files (measures init overhead)" "run --dry-run --all-files" 3 20
210+
211+
# Meta hooks performance
212+
write_section "Meta Hooks Performance" "Benchmarking meta hooks separately:"
213+
create_meta_workspace
214+
215+
META_HOOKS=(
216+
"check-hooks-apply"
217+
"check-useless-excludes"
218+
"identity"
219+
)
220+
221+
for hook in "${META_HOOKS[@]}"; do
222+
benchmark "prek run $hook --all-files" "run $hook --all-files" 3 15 "" "git checkout -- ."
223+
done
224+
225+
if [ "$failed" = true ]; then
226+
exit 1
227+
fi

‎.github/scripts/hyperfine-setup-test-env.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
TARGET_WORKSPACE=${HYPERFINE_BENCHMARK_WORKSPACE:?HYPERFINE_BENCHMARK_WORKSPACE is required}
5+
6+
# Create a clean test directory with files to run builtin hooks against
7+
rm -rf "$TARGET_WORKSPACE"
8+
mkdir -p "$TARGET_WORKSPACE"
9+
cd "$TARGET_WORKSPACE"
10+
git init || { echo "Failed to init git"; exit 1; }
11+
git config user.name "Benchmark"
12+
git config user.email "[email protected]"
13+
14+
# Files with trailing whitespace and no final newline
15+
for i in {1..50}; do
16+
printf "line with trailing whitespace \nanother line " > "file$i.txt"
17+
done
18+
19+
# JSON files
20+
for i in {1..30}; do
21+
echo '{"key": "value", "number": '$i'}' > "file$i.json"
22+
done
23+
24+
# YAML files
25+
for i in {1..30}; do
26+
echo "key: value" > "file$i.yaml"
27+
echo "number: $i" >> "file$i.yaml"
28+
done
29+
30+
# TOML files
31+
for i in {1..30}; do
32+
echo "[section]" > "file$i.toml"
33+
echo "key = \"value$i\"" >> "file$i.toml"
34+
done
35+
36+
# XML files
37+
for i in {1..30}; do
38+
echo '<?xml version="1.0"?><root><item id="'$i'">value</item></root>' > "file$i.xml"
39+
done
40+
41+
# Files with mixed line endings
42+
for i in {1..20}; do
43+
printf "line1\r\nline2\nline3\r\n" > "mixed$i.txt"
44+
done
45+
46+
# Files with UTF-8 BOM
47+
for i in {1..20}; do
48+
printf '\xef\xbb\xbfContent with BOM' > "bom$i.txt"
49+
done
50+
51+
# Executable files (for shebang check)
52+
for i in {1..10}; do
53+
echo "#!/bin/bash" > "script$i.sh"
54+
echo "echo hello" >> "script$i.sh"
55+
chmod +x "script$i.sh"
56+
done
57+
58+
# Files that might contain private keys (but don't)
59+
for i in {1..10}; do
60+
echo "# This is not a private key" > "config$i.txt"
61+
echo "api_key = fake_key_$i" >> "config$i.txt"
62+
done
63+
64+
# Create symlinks for check-symlinks
65+
for i in {1..10}; do
66+
ln -s "file$i.txt" "link$i.txt"
67+
done
68+
69+
# Create a config that uses all builtin hooks
70+
cat > .pre-commit-config.yaml << 'EOF'
71+
repos:
72+
- repo: builtin
73+
hooks:
74+
- id: trailing-whitespace
75+
- id: end-of-file-fixer
76+
- id: check-json
77+
- id: check-yaml
78+
- id: check-toml
79+
- id: check-xml
80+
- id: mixed-line-ending
81+
- id: fix-byte-order-marker
82+
- id: check-executables-have-shebangs
83+
- id: detect-private-key
84+
- id: check-case-conflict
85+
- id: check-merge-conflict
86+
- id: check-symlinks
87+
EOF
88+
89+
git add -A
90+
git commit -m "Initial commit" || { echo "Failed to commit"; exit 1; }

‎.github/workflows/performance.yml‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ env:
1717
CARGO_TERM_COLOR: always
1818
RUSTUP_MAX_RETRIES: 10
1919

20+
concurrency:
21+
group: ${{ github.workflow }}-${{ github.ref }}
22+
cancel-in-progress: true
23+
2024
jobs:
2125
bloat-check:
2226
runs-on: ubuntu-latest
@@ -98,7 +102,65 @@ jobs:
98102
- name: Upload bloat results
99103
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
100104
with:
101-
# NOTE: prek-ci-bot uses this artifact name to post comments on PRs.
105+
# NOTE: https://github.com/j178/prek-ci-bot uses this artifact name to post comments on PRs.
102106
# Make sure to update the bot if you rename the artifact.
103107
name: bloat-check-results
104108
path: bloat-comparison.txt
109+
110+
hyperfine-benchmark:
111+
runs-on: ubuntu-latest
112+
name: "hyperfine benchmark"
113+
timeout-minutes: 30
114+
env:
115+
HYPERFINE_BENCHMARK_WORKSPACE: /tmp/prek-bench
116+
HYPERFINE_RESULTS_FILE: ${{ github.workspace }}/hyperfine-benchmark.md
117+
HYPERFINE_HEAD_BINARY: ${{ github.workspace }}/target/profiling/prek
118+
HYPERFINE_BASE_BINARY: ${{ github.workspace }}/../bin/prek-${{ github.event.pull_request.base.sha }}
119+
steps:
120+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
121+
with:
122+
fetch-depth: 0
123+
persist-credentials: false
124+
125+
- uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2
126+
with:
127+
save-if: ${{ inputs.save-rust-cache == 'true' }}
128+
129+
- id: base-binary-cache
130+
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
131+
with:
132+
path: ${{ env.HYPERFINE_BASE_BINARY }}
133+
key: prek-hyperfine-base-${{ github.event.pull_request.base.sha }}-${{ hashFiles('Cargo.lock') }}-${{ runner.os }}-${{ runner.arch }}
134+
135+
- name: Build base version
136+
if: ${{ steps.base-binary-cache.outputs.cache-hit != 'true' }}
137+
env:
138+
BASE_VERSION: ${{ github.event.pull_request.base.sha }}
139+
run: |
140+
mkdir -p "$(dirname "$HYPERFINE_BASE_BINARY")"
141+
git checkout ${{ github.event.pull_request.base.sha }}
142+
cargo build --profile profiling && mv target/profiling/prek "$HYPERFINE_BASE_BINARY"
143+
git checkout ${{ github.sha }}
144+
145+
- name: Build head version
146+
run: |
147+
cargo build --profile profiling
148+
149+
- name: Install hyperfine
150+
uses: taiki-e/install-action@f92912fad184299a31e22ad070a5059fd07d4f59 # v2.68.7
151+
with:
152+
tool: hyperfine
153+
154+
- name: Setup test environment for builtin hooks
155+
run: .github/scripts/hyperfine-setup-test-env.sh
156+
157+
- name: Run benchmarks
158+
run: .github/scripts/hyperfine-run-benchmarks.sh
159+
160+
- name: Upload benchmark results
161+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
162+
with:
163+
# NOTE: https://github.com/j178/prek-ci-bot uses this artifact name to post comments on PRs.
164+
# Make sure to update the bot if you rename the artifact.
165+
name: hyperfine-benchmark-results
166+
path: ${{ env.HYPERFINE_RESULTS_FILE }}

0 commit comments

Comments
 (0)