|
7 | 7 | validate, and render these declarations. |
8 | 8 | """ |
9 | 9 |
|
| 10 | +from dataclasses import replace |
| 11 | + |
10 | 12 | from ..config import Benchmark, Format |
11 | 13 | from .model import BenchmarkCase, Catalog, Coverage, Storage |
| 14 | +from .render import merge_coverage |
12 | 15 | from .targets import df, duck |
13 | 16 |
|
14 | 17 | PRESETS = { |
15 | 18 | "develop": "Every regular SQL benchmark at full target coverage.", |
16 | 19 | "pr": "The quicker pull-request SQL benchmark matrix (action/bench-sql).", |
17 | 20 | "pr-compact": "Pull-request SQL benchmarks for Vortex Compact plus Parquet controls.", |
18 | | - "pr-all": "The PR and PR Compact benchmark matrices plus extended-only default targets (action/bench-all).", |
| 21 | + "pr-all": "The union of the pr, pr-compact and pr-full matrices (action/bench-all).", |
19 | 22 | "pr-full": "Every regular SQL benchmark at full PR target coverage (action/bench-sql-extended).", |
20 | 23 | "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", |
21 | 24 | } |
|
71 | 74 | STANDARD_WITH_DUCKDB = Coverage(STANDARD_WITH_DUCKDB_TARGETS) |
72 | 75 | DATAFUSION_VORTEX = Coverage(DATAFUSION_VORTEX_TARGETS) |
73 | 76 | COMPACT = Coverage(COMPACT_TARGETS) |
| 77 | +JOB_DUCKDB = Coverage(duck(Format.PARQUET, Format.VORTEX, Format.DUCKDB)) |
74 | 78 |
|
75 | 79 | # Concrete benchmark cases |
76 | 80 |
|
|
82 | 86 | runs={ |
83 | 87 | "pr": DEFAULT, |
84 | 88 | "pr-compact": COMPACT, |
85 | | - "pr-all": STANDARD, |
86 | 89 | "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, |
87 | 90 | "develop": FULL_LOCAL, |
88 | 91 | }, |
|
93 | 96 | name="Clickbench Sorted on NVME", |
94 | 97 | runs={ |
95 | 98 | "pr-compact": COMPACT, |
96 | | - "pr-all": STANDARD, |
97 | 99 | "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, |
98 | 100 | "develop": FULL_LOCAL, |
99 | 101 | }, |
|
107 | 109 | runs={ |
108 | 110 | "pr": DEFAULT, |
109 | 111 | "pr-compact": COMPACT, |
110 | | - "pr-all": STANDARD, |
111 | 112 | "pr-full": FULL_PR, |
112 | 113 | "develop": FULL_LOCAL, |
113 | 114 | }, |
|
124 | 125 | runs={ |
125 | 126 | "pr": DEFAULT, |
126 | 127 | "pr-compact": COMPACT, |
127 | | - "pr-all": STANDARD, |
128 | 128 | "pr-full": STANDARD, |
129 | 129 | "develop": STANDARD, |
130 | 130 | }, |
|
138 | 138 | runs={ |
139 | 139 | "pr": DEFAULT, |
140 | 140 | "pr-compact": COMPACT, |
141 | | - "pr-all": STANDARD, |
142 | 141 | "pr-full": FULL_PR, |
143 | 142 | "develop": FULL_LOCAL, |
144 | 143 | }, |
|
154 | 153 | remote_key="tpch/10.0", |
155 | 154 | runs={ |
156 | 155 | "pr-compact": COMPACT, |
157 | | - "pr-all": COMPACT, |
158 | 156 | "pr-full": STANDARD, |
159 | 157 | "develop": STANDARD, |
160 | 158 | }, |
|
184 | 182 | runs={ |
185 | 183 | "pr": DEFAULT, |
186 | 184 | "pr-compact": COMPACT, |
187 | | - "pr-all": STANDARD, |
188 | 185 | "pr-full": STANDARD_WITH_DUCKDB, |
189 | 186 | "develop": STANDARD_WITH_DUCKDB, |
190 | 187 | }, |
|
197 | 194 | local_dir="vortex-bench/data/statpopgen", |
198 | 195 | runs={ |
199 | 196 | "pr-compact": COMPACT, |
200 | | - "pr-all": STANDARD, |
201 | 197 | "pr-full": STANDARD, |
202 | 198 | "develop": STANDARD, |
203 | 199 | }, |
|
210 | 206 | runs={ |
211 | 207 | "pr": DEFAULT, |
212 | 208 | "pr-compact": COMPACT, |
213 | | - "pr-all": STANDARD, |
214 | 209 | "pr-full": STANDARD, |
215 | 210 | "develop": STANDARD, |
216 | 211 | }, |
|
225 | 220 | remote_key="fineweb", |
226 | 221 | runs={ |
227 | 222 | "pr-compact": COMPACT, |
228 | | - "pr-all": STANDARD, |
229 | 223 | "pr-full": STANDARD, |
230 | 224 | "develop": STANDARD, |
231 | 225 | }, |
|
237 | 231 | scale_factor=1, |
238 | 232 | runs={ |
239 | 233 | "pr": DATAFUSION_VORTEX, |
240 | | - "pr-all": DATAFUSION_VORTEX, |
241 | 234 | "pr-full": DATAFUSION_VORTEX, |
242 | 235 | "develop": DATAFUSION_VORTEX, |
243 | 236 | }, |
|
249 | 242 | iterations=10, |
250 | 243 | runs={ |
251 | 244 | "pr-compact": COMPACT, |
252 | | - "pr-all": COMPACT, |
253 | 245 | "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, |
254 | 246 | "develop": STANDARD_WITH_DUCKDB, |
255 | 247 | }, |
256 | 248 | ), |
| 249 | + BenchmarkCase( |
| 250 | + id="job-nvme", |
| 251 | + benchmark=Benchmark.JOB, |
| 252 | + name="JOB on NVME", |
| 253 | + # DuckDB only: DataFusion spends minutes per pass on the 17x queries' join orders. |
| 254 | + runs={"pr-full": JOB_DUCKDB}, |
| 255 | + ), |
257 | 256 | BenchmarkCase( |
258 | 257 | id="vortex-queries", |
259 | 258 | benchmark=Benchmark.VORTEX_QUERIES, |
|
266 | 265 | ), |
267 | 266 | ) |
268 | 267 |
|
269 | | -CATALOG = Catalog(presets=PRESETS, benchmarks=BENCHMARKS) |
| 268 | +PR_ALL_SOURCES = ("pr", "pr-compact", "pr-full") |
| 269 | + |
| 270 | + |
| 271 | +def _with_pr_all(benchmark: BenchmarkCase) -> BenchmarkCase: |
| 272 | + sources = [benchmark.runs[preset] for preset in PR_ALL_SOURCES if preset in benchmark.runs] |
| 273 | + if not sources: |
| 274 | + return benchmark |
| 275 | + return replace(benchmark, runs={**benchmark.runs, "pr-all": merge_coverage(sources)}) |
| 276 | + |
| 277 | + |
| 278 | +CATALOG = Catalog(presets=PRESETS, benchmarks=tuple(map(_with_pr_all, BENCHMARKS))) |
0 commit comments