Skip to content

Commit 10d2d39

Browse files
mprammerclaude
andauthored
bench: name the data directory when capturing file sizes (#10935)
## Summary `scripts/capture-file-sizes.py` measured every directory under `vortex-bench/data` whose name started with the benchmark's (`clickbench` matched `clickbench_partitioned`, and anything else named `clickbench_*`), so data sitting next to a benchmark's own could leak into its file-size report. The script now takes `--dataset-dir` and measures exactly that directory. ## Changes Each CI matrix entry carries a `dataset_dir`, which defaults to the benchmark's subcommand; ClickBench sets `clickbench_partitioned`. The workflow passes it through. For every benchmark the CI captures today, the script's output is byte-identical to before. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: Martin Prammer <[email protected]> Co-authored-by: Claude <[email protected]>
1 parent 1577edb commit 10d2d39

7 files changed

Lines changed: 86 additions & 44 deletions

File tree

‎.github/workflows/sql-bench-matrix.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,7 @@ jobs:
235235
uv run --no-project scripts/capture-file-sizes.py \
236236
vortex-bench/data \
237237
--benchmark ${{ matrix.subcommand }} \
238+
--dataset-dir ${{ matrix.dataset_dir }} \
238239
--commit ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }} \
239240
-o sizes.json
240241
cat sizes.json >> results.json

‎bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@
108108
id="clickbench-nvme",
109109
benchmark=Benchmark.CLICKBENCH,
110110
name="Clickbench on NVME",
111+
dataset_dir="clickbench_partitioned",
111112
runs={
112113
"pr": DEFAULT,
113114
"pr-compact": COMPACT,

‎bench-orchestrator/bench_orchestrator/ci_matrix/model.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,12 @@ class BenchmarkCase:
3939
iterations: int | None = None
4040
local_dir: str | None = None
4141
remote_key: str | None = None
42+
dataset_dir: str | None = None
43+
44+
@property
45+
def data_dir_name(self) -> str:
46+
"""Return the directory under ``vortex-bench/data`` holding this benchmark's data."""
47+
return self.dataset_dir or self.subcommand
4248

4349
@property
4450
def subcommand(self) -> str:

‎bench-orchestrator/bench_orchestrator/ci_matrix/render.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ def _matrix_entry(benchmark: BenchmarkCase, coverage: Coverage) -> dict[str, obj
4242
"id": benchmark.id,
4343
"subcommand": benchmark.subcommand,
4444
"name": benchmark.name,
45+
"dataset_dir": benchmark.data_dir_name,
4546
"targets": [target.to_dict() for target in coverage.targets],
4647
"data_formats": [fmt.value for fmt in _data_formats(coverage)],
4748
}

‎bench-orchestrator/tests/test_matrix.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,13 @@ def test_resolver_rejects_duplicate_ids() -> None:
168168
_ = render_matrix("develop", catalog)
169169

170170

171+
def test_matrix_entries_name_their_data_directory() -> None:
172+
entries = {entry["id"]: entry for entry in render_matrix("develop", CATALOG)}
173+
174+
assert entries["clickbench-nvme"]["dataset_dir"] == "clickbench_partitioned"
175+
assert entries["tpch-nvme"]["dataset_dir"] == "tpch"
176+
177+
171178
def test_matrix_command() -> None:
172179
result = runner.invoke(cli_module.app, ["matrix", "develop"])
173180
assert result.exit_code == 0

‎scripts/capture-file-sizes.py‎

Lines changed: 39 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ def main() -> None:
1919
parser = argparse.ArgumentParser(description="Capture file sizes from benchmark data directories")
2020
parser.add_argument("data_dir", help="Data directory (e.g., vortex-bench/data)")
2121
parser.add_argument("--benchmark", required=True, help="Benchmark name (e.g., clickbench)")
22+
parser.add_argument(
23+
"--dataset-dir", required=True, help="The benchmark's directory under data_dir (e.g., clickbench_partitioned)"
24+
)
2225
parser.add_argument("--commit", required=True, help="Commit SHA")
2326
parser.add_argument("-o", "--output", required=True, help="Output JSONL file path")
2427
args = parser.parse_args()
@@ -28,16 +31,9 @@ def main() -> None:
2831
print(f"Data directory not found: {data_dir}", file=sys.stderr)
2932
sys.exit(1)
3033

31-
# Find benchmark directories matching the name (handles flavors like clickbench_partitioned)
32-
# Also handles exact match (e.g., tpch)
33-
benchmark_dirs = [
34-
d
35-
for d in data_dir.iterdir()
36-
if d.is_dir() and (d.name == args.benchmark or d.name.startswith(f"{args.benchmark}_"))
37-
]
38-
39-
if not benchmark_dirs:
40-
print(f"No benchmark directories found matching: {args.benchmark}", file=sys.stderr)
34+
benchmark_dir = data_dir / args.dataset_dir
35+
if not benchmark_dir.is_dir():
36+
print(f"Benchmark directory not found: {benchmark_dir}", file=sys.stderr)
4137
sys.exit(1)
4238

4339
# Formats to capture (vortex formats only, not parquet/duckdb)
@@ -49,42 +45,41 @@ def main() -> None:
4945
# Walk subdirectories looking for format directories
5046
# Handle both direct format dirs (clickbench_partitioned/vortex-file-compressed/)
5147
# and scale factor subdirs (tpch/1.0/vortex-file-compressed/)
52-
for benchmark_dir in benchmark_dirs:
53-
for format_dir in benchmark_dir.rglob("*"):
54-
if not format_dir.is_dir():
55-
continue
56-
57-
format_name = format_dir.name
58-
if format_name not in formats_to_capture:
48+
for format_dir in benchmark_dir.rglob("*"):
49+
if not format_dir.is_dir():
50+
continue
51+
52+
format_name = format_dir.name
53+
if format_name not in formats_to_capture:
54+
continue
55+
56+
# Extract scale factor from path (e.g., "1.0" for tpch/1.0/vortex-file-compressed)
57+
# Default to "1.0" if no intermediate directory (e.g., clickbench)
58+
path_between = format_dir.relative_to(benchmark_dir).parent
59+
scale_factor = str(path_between) if str(path_between) != "." else "1.0"
60+
61+
# Capture all files in this format directory
62+
for file_path in format_dir.rglob("*"):
63+
if not file_path.is_file():
5964
continue
6065

61-
# Extract scale factor from path (e.g., "1.0" for tpch/1.0/vortex-file-compressed)
62-
# Default to "1.0" if no intermediate directory (e.g., clickbench)
63-
path_between = format_dir.relative_to(benchmark_dir).parent
64-
scale_factor = str(path_between) if str(path_between) != "." else "1.0"
65-
66-
# Capture all files in this format directory
67-
for file_path in format_dir.rglob("*"):
68-
if not file_path.is_file():
69-
continue
70-
71-
size_bytes = file_path.stat().st_size
72-
relative_path = file_path.relative_to(format_dir)
73-
74-
records.append(
75-
{
76-
"metric": "file_size",
77-
"unit": "bytes",
78-
"value": size_bytes,
79-
"commit_id": args.commit,
80-
"file_size": {
81-
"benchmark": args.benchmark,
82-
"scale_factor": scale_factor,
83-
"format": format_name,
84-
"file": str(relative_path),
85-
},
86-
}
87-
)
66+
size_bytes = file_path.stat().st_size
67+
relative_path = file_path.relative_to(format_dir)
68+
69+
records.append(
70+
{
71+
"metric": "file_size",
72+
"unit": "bytes",
73+
"value": size_bytes,
74+
"commit_id": args.commit,
75+
"file_size": {
76+
"benchmark": args.benchmark,
77+
"scale_factor": scale_factor,
78+
"format": format_name,
79+
"file": str(relative_path),
80+
},
81+
}
82+
)
8883

8984
# Sort for deterministic output
9085
records.sort(

‎scripts/tests/test_benchmark_reporting.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -701,6 +701,8 @@ def test_capture_file_sizes_emits_shared_benchmark_rows(tmp_path: Path) -> None:
701701
str(data_dir),
702702
"--benchmark",
703703
"tpch",
704+
"--dataset-dir",
705+
"tpch",
704706
"--commit",
705707
"deadbeef",
706708
"-o",
@@ -787,3 +789,32 @@ def test_summary_reports_hot_alone_when_no_row_recorded_its_runs(tmp_path: Path)
787789
assert "**Vortex (geomean)**: hot 0.500x ✅" in summary
788790
assert "**Parquet (geomean)**: hot 1.000x ➖" in summary
789791
assert "cold" not in summary
792+
793+
794+
def test_capture_file_sizes_reads_only_the_named_directory(tmp_path: Path) -> None:
795+
data_dir = tmp_path / "data"
796+
for name, size in (("clickbench_partitioned", 42), ("clickbench_partitioned_10", 7)):
797+
format_dir = data_dir / name / "vortex-file-compressed"
798+
format_dir.mkdir(parents=True)
799+
(format_dir / "hits_0.vortex").write_bytes(b"x" * size)
800+
output_path = tmp_path / "sizes.jsonl"
801+
802+
subprocess.run(
803+
[
804+
sys.executable,
805+
str(CAPTURE_SCRIPT),
806+
str(data_dir),
807+
"--benchmark",
808+
"clickbench",
809+
"--dataset-dir",
810+
"clickbench_partitioned",
811+
"--commit",
812+
"deadbeef",
813+
"-o",
814+
str(output_path),
815+
],
816+
check=True,
817+
)
818+
819+
records = [json.loads(line) for line in output_path.read_text(encoding="utf-8").splitlines()]
820+
assert [record["value"] for record in records] == [42]

0 commit comments

Comments
 (0)