@@ -82,15 +82,33 @@ def stored_custom_row(
8282 }
8383
8484
85+ def git_repo_with_one_commit (path : Path ) -> Path :
86+ """Create a throwaway repository the reporter can walk.
87+
88+ The reporter resolves its baseline from `git rev-list HEAD`, so the tests
89+ need a repository of their own rather than whatever checkout they happen to
90+ run inside — a contributor's shallow clone must not change the result.
91+ """
92+
93+ path .mkdir (parents = True )
94+ identity = [
"-c" ,
"user.name=Vortex Test" ,
"-c" ,
"[email protected] " ]
95+ subprocess .run (["git" , "init" , "--quiet" ], cwd = path , check = True )
96+ (path / "README.md" ).write_text ("benchmark reporting fixture\n " , encoding = "utf-8" )
97+ subprocess .run (["git" , "add" , "README.md" ], cwd = path , check = True )
98+ subprocess .run (["git" , * identity , "commit" , "--quiet" , "-m" , "fixture" ], cwd = path , check = True )
99+ return path
100+
101+
85102def render_report (
86103 tmp_path : Path ,
87104 base_rows : list [dict [str , object ]],
88105 pr_rows : list [dict [str , object ]],
89106 benchmark_name : str ,
90107) -> str :
108+ repo = git_repo_with_one_commit (tmp_path / "repo" )
91109 head_commit = subprocess .run (
92110 ["git" , "rev-parse" , "HEAD" ],
93- cwd = REPO_ROOT ,
111+ cwd = repo ,
94112 check = True ,
95113 capture_output = True ,
96114 text = True ,
@@ -103,6 +121,7 @@ def render_report(
103121
104122 result = subprocess .run (
105123 [sys .executable , str (COMPARE_SCRIPT ), str (base_path ), str (pr_path ), benchmark_name ],
124+ cwd = repo ,
106125 check = False ,
107126 capture_output = True ,
108127 text = True ,
@@ -429,8 +448,8 @@ def test_comparison_report_retains_sql_analysis(tmp_path: Path) -> None:
429448 report = render_report (tmp_path , base_rows , pr_rows , "TPC-H" )
430449
431450 assert "**Verdict**:" in report
432- assert "**Vortex (hot geomean)**:" in report
433- assert "**Parquet (hot geomean)**:" in report
451+ assert "**Vortex (geomean)**: hot " in report
452+ assert "**Parquet (geomean)**: hot " in report
434453 assert "How to read Verdict and Engines" in report
435454 assert "<summary>datafusion / vortex-file-compressed / ns " in report
436455 assert "<summary>datafusion / parquet / ns " in report
@@ -518,7 +537,8 @@ def test_report_splits_cold_and_hot_runs(tmp_path: Path) -> None:
518537 "0.03 / 0.10 / -75.0%" ,
519538 ]
520539 assert "**Attributed Vortex impact**: -50.0%" in report
521- assert "**Cold run (geomean)**: Vortex 2.000x ❌ · Parquet 1.000x ➖" in report
540+ assert "**Vortex (geomean)**: hot 0.500x ✅ · cold 2.000x ❌" in report
541+ assert "**Parquet (geomean)**: hot 1.000x ➖ · cold 1.000x ➖" in report
522542 assert "**Commits**: PR `pr-sha` vs base " in report
523543
524544
@@ -547,7 +567,7 @@ def test_verdict_ignores_a_cold_start_only_regression(tmp_path: Path) -> None:
547567 report = render_report (tmp_path , base_rows , pr_rows , "TPC-H" )
548568
549569 assert "**Attributed Vortex impact**: +0.0%" in report
550- assert "**Cold run (geomean)**: Vortex 4 .000x ❌ · Parquet 1 .000x ➖ " in report
570+ assert "**Vortex (geomean)**: hot 1 .000x ➖ · cold 4 .000x ❌ " in report
551571 assert markdown_row (report , "tpch_q01/datafusion:vortex-file-compressed" ) == [
552572 "tpch_q01/datafusion:vortex-file-compressed" ,
553573 "100 / 100 / +0.0%" ,
@@ -682,3 +702,63 @@ def test_capture_file_sizes_emits_shared_benchmark_rows(tmp_path: Path) -> None:
682702 },
683703 }
684704 ]
705+
706+
707+ def test_shallow_checkout_fails_instead_of_reporting_no_baseline (tmp_path : Path ) -> None :
708+ """A shallow clone must fail loudly, not render a baseline-free report.
709+
710+ `git rev-list HEAD` reaches only the tip of a shallow clone, so every
711+ recorded commit looks unreachable. Reporting that as "no baseline yet"
712+ hides a real comparison behind a report that merely looks new.
713+ """
714+
715+ origin = git_repo_with_one_commit (tmp_path / "origin" )
716+ shallow = tmp_path / "shallow"
717+ subprocess .run (
718+ ["git" , "clone" , "--quiet" , "--depth" , "1" , "--no-local" , f"file://{ origin } " , str (shallow )],
719+ check = True ,
720+ capture_output = True ,
721+ )
722+ base_path = tmp_path / "base.jsonl"
723+ pr_path = tmp_path / "pr.jsonl"
724+ base_path .write_text (
725+ f"{ json .dumps (stored_timing_row ('recorded' , 'tpch_q01/datafusion:parquet' , 100 ))} \n " ,
726+ encoding = "utf-8" ,
727+ )
728+ pr_path .write_text (
729+ f"{ json .dumps (stored_timing_row ('pr' , 'tpch_q01/datafusion:parquet' , 105 ))} \n " ,
730+ encoding = "utf-8" ,
731+ )
732+
733+ result = subprocess .run (
734+ [sys .executable , str (COMPARE_SCRIPT ), str (base_path ), str (pr_path ), "TPC-H" ],
735+ cwd = shallow ,
736+ check = False ,
737+ capture_output = True ,
738+ text = True ,
739+ )
740+
741+ assert result .returncode != 0
742+ assert "shallow checkout" in result .stderr
743+ assert "fetch-depth: 0" in result .stderr
744+ assert "No baseline is available" not in result .stdout
745+
746+
747+ def test_summary_reports_hot_alone_when_no_row_recorded_its_runs (tmp_path : Path ) -> None :
748+ # Random Access, String Encoding and Compression report one value per row, so
749+ # there is no cold geomean to sit beside the hot one.
750+ base_rows = [
751+ stored_custom_row ("base-sha" , "random-access/vortex-tokio-local-disk" , "ns" , 100.0 ),
752+ stored_custom_row ("base-sha" , "random-access/parquet-tokio-local-disk" , "ns" , 200.0 , file_format = "parquet" ),
753+ ]
754+ pr_rows = [
755+ stored_custom_row ("pr-sha" , "random-access/vortex-tokio-local-disk" , "ns" , 50.0 ),
756+ stored_custom_row ("pr-sha" , "random-access/parquet-tokio-local-disk" , "ns" , 200.0 , file_format = "parquet" ),
757+ ]
758+
759+ report = render_report (tmp_path , base_rows , pr_rows , "Random Access" )
760+
761+ summary = report .splitlines ()[2 ]
762+ assert "**Vortex (geomean)**: hot 0.500x ✅" in summary
763+ assert "**Parquet (geomean)**: hot 1.000x ➖" in summary
764+ assert "cold" not in summary
0 commit comments