Skip to content

Commit 2410f78

Browse files
bench: fail loudly when the benchmark comparison cannot see git history (#9507)
Signed-off-by: Joe Isaacs <[email protected]>
1 parent c196596 commit 2410f78

4 files changed

Lines changed: 155 additions & 35 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,21 @@ jobs:
161161
uv run --no-project --with pytest --with xxhash \
162162
pytest scripts/tests/test_measurement_id.py
163163
164+
benchmark-reporting:
165+
name: "Benchmark reporting"
166+
runs-on: ubuntu-latest
167+
timeout-minutes: 10
168+
steps:
169+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
170+
# sync: false -- the tests run via `uv run --no-project` and need no workspace packages.
171+
- uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
172+
with:
173+
sync: false
174+
- name: Pytest - benchmark reporting
175+
run: |
176+
uv run --no-project --with pytest --with pandas --with numpy --with orjson \
177+
--with tabulate pytest scripts/tests/test_benchmark_reporting.py
178+
164179
python-cuda-test:
165180
name: "Python CUDA (test)"
166181
if: github.repository == 'vortex-data/vortex'

‎.github/workflows/pr-bench-runner.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ jobs:
8989
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
9090
with:
9191
ref: ${{ github.event.pull_request.head.sha || github.sha }}
92+
# `compare-benchmark-jsons.py` resolves its baseline by walking `git rev-list HEAD`,
93+
# which reaches only the tip of a shallow clone.
94+
fetch-depth: 0
9295

9396
- name: Install DuckDB
9497
uses: ./.github/actions/setup-duckdb

‎scripts/compare-benchmark-jsons.py‎

Lines changed: 52 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,26 @@ def read_jsonl_rows_for_commit(path: str, commit_id: str) -> pd.DataFrame:
231231

232232

233233
def git_tree_commit_ids() -> set[str]:
234-
"""Return every commit reachable from the checked-out branch head."""
234+
"""Return every commit reachable from the checked-out branch head.
235+
236+
A shallow checkout reaches only its own tip, so every recorded baseline
237+
commit looks unreachable and the report silently degrades into "no
238+
baseline is available for this benchmark yet" — indistinguishable from a
239+
genuinely new benchmark, and wrong. Refuse the guess and say what to fix.
240+
"""
241+
242+
is_shallow = subprocess.run(
243+
["git", "rev-parse", "--is-shallow-repository"],
244+
check=True,
245+
capture_output=True,
246+
text=True,
247+
).stdout.strip()
248+
if is_shallow == "true":
249+
raise SystemExit(
250+
"Cannot resolve a baseline from a shallow checkout: the baseline is the newest "
251+
"recorded commit reachable from HEAD, and a shallow clone reaches only its own "
252+
"tip. Check the repository out with `fetch-depth: 0`."
253+
)
235254

236255
commits = subprocess.run(
237256
["git", "rev-list", "HEAD"],
@@ -662,25 +681,32 @@ def format_performance(
662681
return f"{ratio:.3f}x {emoji}"
663682

664683

665-
def format_cold_summary(
666-
groups: list[tuple[str, pd.DataFrame]],
684+
def format_target_summary(
685+
group: pd.DataFrame,
686+
hot_ratio: float,
667687
improvement_threshold: float,
668688
regression_threshold: float,
669-
) -> str | None:
670-
"""Render the cold-run geomeans, or nothing when no row recorded its runs."""
689+
target_name: str,
690+
) -> str:
691+
"""Render one target's hot geomean with its cold geomean beside it.
671692
672-
parts = []
673-
for label, group in groups:
674-
if group.empty:
675-
continue
676-
ratio = calculate_geometric_mean(group, "cold_ratio")
677-
if pd.isna(ratio):
678-
continue
679-
parts.append(f"{label} {format_performance(ratio, improvement_threshold, regression_threshold, label)}")
693+
The detail tables put a row's hot and cold runs side by side, so the summary
694+
reads the same way: a cold-start-only change is visible on the target's own
695+
line instead of on a separate one the reader has to cross-reference. Rows that
696+
recorded no per-run timings have no cold geomean, and the target then reports
697+
its hot geomean alone.
698+
"""
680699

681-
if not parts:
682-
return None
683-
return " · ".join(parts)
700+
hot = format_performance(hot_ratio, improvement_threshold, regression_threshold, target_name)
701+
if pd.isna(hot_ratio):
702+
return hot
703+
704+
cold_ratio = calculate_geometric_mean(group, "cold_ratio")
705+
if pd.isna(cold_ratio):
706+
return f"hot {hot}"
707+
708+
cold = format_performance(cold_ratio, improvement_threshold, regression_threshold, target_name)
709+
return f"hot {hot} · cold {cold}"
684710

685711

686712
def format_measurement_value(value: float) -> str:
@@ -1071,8 +1097,10 @@ def format_report_help() -> str:
10711097
"the noise floor, and control-run noise.",
10721098
"- **Hot vs cold**: Every measurement is run several times. The first run is "
10731099
"reported as the cold run, and the median of the runs after it is reported as "
1074-
"the hot run. The verdict, geomeans, and significance all use hot runs; the "
1075-
"cold column shows first-run cost separately, and `hot/cold` is how much of "
1100+
"the hot run. The verdict and significance use hot runs; each target's "
1101+
"geomean reports hot and cold beside each other where the individual runs "
1102+
"were recorded and hot alone where they were not, the cold column shows "
1103+
"first-run cost per row, and `hot/cold` is how much of "
10761104
"each run the warm path saves. Rows whose results predate per-run reporting "
10771105
"show only one value, taken from the value the runner reported.",
10781106
"- **Table cells**: Each cell reads `PR / base / %diff`. The hot and cold "
@@ -1186,29 +1214,23 @@ def main() -> None:
11861214
summary_fields.append(f"**Engines**: {engine_summary}")
11871215

11881216
if len(vortex_df) > 0:
1189-
vortex_performance = format_performance(
1217+
vortex_performance = format_target_summary(
1218+
vortex_df,
11901219
vortex_geometric_mean_ratio,
11911220
improvement_threshold,
11921221
regression_threshold,
11931222
"vortex",
11941223
)
1195-
summary_fields.append(f"**Vortex (hot geomean)**: {vortex_performance}")
1224+
summary_fields.append(f"**Vortex (geomean)**: {vortex_performance}")
11961225
if len(parquet_df) > 0:
1197-
parquet_performance = format_performance(
1226+
parquet_performance = format_target_summary(
1227+
parquet_df,
11981228
parquet_geometric_mean_ratio,
11991229
improvement_threshold,
12001230
regression_threshold,
12011231
"parquet",
12021232
)
1203-
summary_fields.append(f"**Parquet (hot geomean)**: {parquet_performance}")
1204-
1205-
cold_summary = format_cold_summary(
1206-
[("Vortex", vortex_df), ("Parquet", parquet_df)],
1207-
improvement_threshold,
1208-
regression_threshold,
1209-
)
1210-
if cold_summary is not None:
1211-
summary_fields.append(f"**Cold run (geomean)**: {cold_summary}")
1233+
summary_fields.append(f"**Parquet (geomean)**: {parquet_performance}")
12121234

12131235
if verdict is not None:
12141236
shifts = f"Parquet (control) {verdict['environment_shift']}"

‎scripts/tests/test_benchmark_reporting.py‎

Lines changed: 85 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -82,15 +82,33 @@ def stored_custom_row(
8282
}
8383

8484

85+
def git_repo_with_one_commit(path: Path) -> Path:
86+
"""Create a throwaway repository the reporter can walk.
87+
88+
The reporter resolves its baseline from `git rev-list HEAD`, so the tests
89+
need a repository of their own rather than whatever checkout they happen to
90+
run inside — a contributor's shallow clone must not change the result.
91+
"""
92+
93+
path.mkdir(parents=True)
94+
identity = ["-c", "user.name=Vortex Test", "-c", "[email protected]"]
95+
subprocess.run(["git", "init", "--quiet"], cwd=path, check=True)
96+
(path / "README.md").write_text("benchmark reporting fixture\n", encoding="utf-8")
97+
subprocess.run(["git", "add", "README.md"], cwd=path, check=True)
98+
subprocess.run(["git", *identity, "commit", "--quiet", "-m", "fixture"], cwd=path, check=True)
99+
return path
100+
101+
85102
def render_report(
86103
tmp_path: Path,
87104
base_rows: list[dict[str, object]],
88105
pr_rows: list[dict[str, object]],
89106
benchmark_name: str,
90107
) -> str:
108+
repo = git_repo_with_one_commit(tmp_path / "repo")
91109
head_commit = subprocess.run(
92110
["git", "rev-parse", "HEAD"],
93-
cwd=REPO_ROOT,
111+
cwd=repo,
94112
check=True,
95113
capture_output=True,
96114
text=True,
@@ -103,6 +121,7 @@ def render_report(
103121

104122
result = subprocess.run(
105123
[sys.executable, str(COMPARE_SCRIPT), str(base_path), str(pr_path), benchmark_name],
124+
cwd=repo,
106125
check=False,
107126
capture_output=True,
108127
text=True,
@@ -429,8 +448,8 @@ def test_comparison_report_retains_sql_analysis(tmp_path: Path) -> None:
429448
report = render_report(tmp_path, base_rows, pr_rows, "TPC-H")
430449

431450
assert "**Verdict**:" in report
432-
assert "**Vortex (hot geomean)**:" in report
433-
assert "**Parquet (hot geomean)**:" in report
451+
assert "**Vortex (geomean)**: hot " in report
452+
assert "**Parquet (geomean)**: hot " in report
434453
assert "How to read Verdict and Engines" in report
435454
assert "<summary>datafusion / vortex-file-compressed / ns " in report
436455
assert "<summary>datafusion / parquet / ns " in report
@@ -518,7 +537,8 @@ def test_report_splits_cold_and_hot_runs(tmp_path: Path) -> None:
518537
"0.03 / 0.10 / -75.0%",
519538
]
520539
assert "**Attributed Vortex impact**: -50.0%" in report
521-
assert "**Cold run (geomean)**: Vortex 2.000x ❌ · Parquet 1.000x ➖" in report
540+
assert "**Vortex (geomean)**: hot 0.500x ✅ · cold 2.000x ❌" in report
541+
assert "**Parquet (geomean)**: hot 1.000x ➖ · cold 1.000x ➖" in report
522542
assert "**Commits**: PR `pr-sha` vs base " in report
523543

524544

@@ -547,7 +567,7 @@ def test_verdict_ignores_a_cold_start_only_regression(tmp_path: Path) -> None:
547567
report = render_report(tmp_path, base_rows, pr_rows, "TPC-H")
548568

549569
assert "**Attributed Vortex impact**: +0.0%" in report
550-
assert "**Cold run (geomean)**: Vortex 4.000x ❌ · Parquet 1.000x ➖" in report
570+
assert "**Vortex (geomean)**: hot 1.000x ➖ · cold 4.000x ❌" in report
551571
assert markdown_row(report, "tpch_q01/datafusion:vortex-file-compressed") == [
552572
"tpch_q01/datafusion:vortex-file-compressed",
553573
"100 / 100 / +0.0%",
@@ -682,3 +702,63 @@ def test_capture_file_sizes_emits_shared_benchmark_rows(tmp_path: Path) -> None:
682702
},
683703
}
684704
]
705+
706+
707+
def test_shallow_checkout_fails_instead_of_reporting_no_baseline(tmp_path: Path) -> None:
708+
"""A shallow clone must fail loudly, not render a baseline-free report.
709+
710+
`git rev-list HEAD` reaches only the tip of a shallow clone, so every
711+
recorded commit looks unreachable. Reporting that as "no baseline yet"
712+
hides a real comparison behind a report that merely looks new.
713+
"""
714+
715+
origin = git_repo_with_one_commit(tmp_path / "origin")
716+
shallow = tmp_path / "shallow"
717+
subprocess.run(
718+
["git", "clone", "--quiet", "--depth", "1", "--no-local", f"file://{origin}", str(shallow)],
719+
check=True,
720+
capture_output=True,
721+
)
722+
base_path = tmp_path / "base.jsonl"
723+
pr_path = tmp_path / "pr.jsonl"
724+
base_path.write_text(
725+
f"{json.dumps(stored_timing_row('recorded', 'tpch_q01/datafusion:parquet', 100))}\n",
726+
encoding="utf-8",
727+
)
728+
pr_path.write_text(
729+
f"{json.dumps(stored_timing_row('pr', 'tpch_q01/datafusion:parquet', 105))}\n",
730+
encoding="utf-8",
731+
)
732+
733+
result = subprocess.run(
734+
[sys.executable, str(COMPARE_SCRIPT), str(base_path), str(pr_path), "TPC-H"],
735+
cwd=shallow,
736+
check=False,
737+
capture_output=True,
738+
text=True,
739+
)
740+
741+
assert result.returncode != 0
742+
assert "shallow checkout" in result.stderr
743+
assert "fetch-depth: 0" in result.stderr
744+
assert "No baseline is available" not in result.stdout
745+
746+
747+
def test_summary_reports_hot_alone_when_no_row_recorded_its_runs(tmp_path: Path) -> None:
748+
# Random Access, String Encoding and Compression report one value per row, so
749+
# there is no cold geomean to sit beside the hot one.
750+
base_rows = [
751+
stored_custom_row("base-sha", "random-access/vortex-tokio-local-disk", "ns", 100.0),
752+
stored_custom_row("base-sha", "random-access/parquet-tokio-local-disk", "ns", 200.0, file_format="parquet"),
753+
]
754+
pr_rows = [
755+
stored_custom_row("pr-sha", "random-access/vortex-tokio-local-disk", "ns", 50.0),
756+
stored_custom_row("pr-sha", "random-access/parquet-tokio-local-disk", "ns", 200.0, file_format="parquet"),
757+
]
758+
759+
report = render_report(tmp_path, base_rows, pr_rows, "Random Access")
760+
761+
summary = report.splitlines()[2]
762+
assert "**Vortex (geomean)**: hot 0.500x ✅" in summary
763+
assert "**Parquet (geomean)**: hot 1.000x ➖" in summary
764+
assert "cold" not in summary

0 commit comments

Comments
 (0)