Skip to content

Commit c22b4e5

Browse files
joseph-isaacsclaude
joseph-isaacs
and
claude
authored
Reorder bench groups and SQL file size charts (#225)
- Groups: Compression and Compression Size, then SQL File Size, then Random Access and the query groups as before. String Encoding and String Encoding Size move to the end, after the groups sorted by name. - SQL File Size charts: clickbench, then tpch, then tpcds, then the other benchmarks by name, each by numeric scale factor. - TPC-H, TPC-DS and nested TPC-H charts always name their scale factor, so `tpch sf=1.0` sits beside `tpch sf=10.0`. - Bump the group payload cache version, since the cached order and names change. Claude-Session: https://claude.ai/code/session_01UeniB9quRiv5fCx8AhzH9s Signed-off-by: Joe Isaacs <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 26ddd51 commit c22b4e5

6 files changed

Lines changed: 70 additions & 16 deletions

File tree

‎src/bench/lib/data-cache.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,9 @@ describe('data-cache wrappers', () => {
6969
expect(BENCH_DATA_TAG).toBe('bench-data');
7070
expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400);
7171
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([
72-
'data-cache:group-charts:v13:n100',
72+
'data-cache:group-charts:v14:n100',
7373
]);
74-
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v13']);
74+
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v14']);
7575
for (const call of cacheCalls) {
7676
expect(call.options.tags).toEqual([BENCH_DATA_TAG]);
7777
expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS);

‎src/bench/lib/data-cache.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_
5757
* code can serve payloads the old code computed until the next ingest.
5858
* It is independent of the producer-facing benchmark schema version.
5959
*/
60-
const GROUP_PAYLOAD_CACHE_VERSION = 'v13';
60+
const GROUP_PAYLOAD_CACHE_VERSION = 'v14';
6161

6262
// The default last-100 group bundle, keyed by group slug. The slug is the cache
6363
// key (an `unstable_cache` argument), so one wrapper covers every group. A

‎src/bench/lib/group-charts.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -648,7 +648,7 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', ()
648648
...defined(OTHER_FAMILIES[3]),
649649
parseCommitWindow('all'),
650650
);
651-
expect(sf1?.display_name).toBe('tpch');
651+
expect(sf1?.display_name).toBe('tpch sf=1.0');
652652
expect(sf1?.unit_kind).toBe('bytes');
653653
expect(sf1?.series).toEqual({
654654
'vortex-compact': [null, null, null, null, 88, 86],

‎src/bench/lib/groups.test.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,8 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg
642642
[develop, 'tpch', '10.0', 'vortex-compact', 'lineitem.vortex', 1000],
643643
[nightly, 'tpch', '100', 'vortex-file-compressed', 'lineitem.vortex', 40000],
644644
[develop, 'clickbench', '1.0', 'vortex-file-compressed', 'hits_0.vortex', 1000],
645+
[develop, 'tpcds', '1.0', 'vortex-file-compressed', 'store_sales.vortex', 500],
646+
[develop, 'appian', '1.0', 'vortex-file-compressed', 'data.vortex', 70],
645647
] as const) {
646648
await getPool().query(
647649
`INSERT INTO file_sizes
@@ -655,11 +657,15 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg
655657
expect(groups.map((group) => group.name)).toEqual(['SQL File Size']);
656658
const [group] = groups;
657659
expect(groupKeyFromSlug(defined(group).slug)).toEqual({ k: 'FileSizeGroup' });
660+
// Clickbench, TPC-H and TPC-DS lead; the rest follow by name. TPC-H always names its scale
661+
// factor, while Clickbench and Appian have none.
658662
expect(defined(group).charts.map((chart) => chart.name)).toEqual([
659663
'clickbench',
660-
'tpch',
664+
'tpch sf=1.0',
661665
'tpch sf=10.0',
662666
'tpch sf=100',
667+
'tpcds sf=1.0',
668+
'appian',
663669
]);
664670
expect(chartKeyFromSlug(defined(defined(group).charts[3]).slug)).toEqual({
665671
k: 'FileSize',

‎src/bench/lib/queries.test.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,29 @@ describe('compareGroupSortKey canonical group ordering (no DB)', () => {
371371
]);
372372
});
373373

374+
it('puts compression and SQL file size before random access and string encoding last', () => {
375+
const sorted = [
376+
'String Encoding Size',
377+
'Clickbench',
378+
'cohere-large-10m / partitioned',
379+
'String Encoding',
380+
'Random Access (NVMe)',
381+
'SQL File Size',
382+
'Compression Size',
383+
'Compression',
384+
].sort(compareGroupSortKey);
385+
expect(sorted).toEqual([
386+
'Compression',
387+
'Compression Size',
388+
'SQL File Size',
389+
'Random Access (NVMe)',
390+
'Clickbench',
391+
'cohere-large-10m / partitioned',
392+
'String Encoding',
393+
'String Encoding Size',
394+
]);
395+
});
396+
374397
it('places the appian group directly after both fineweb groups', () => {
375398
const sorted = [
376399
'fineweb [s3]',

‎src/bench/lib/queries.ts‎

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -438,12 +438,18 @@ function chartDisplayName(key: ChartKey): string {
438438
}
439439
}
440440

441+
/** Benchmarks run at several scale factors, so their file-size charts always name it. */
442+
const SCALED_FILE_SIZE_BENCHMARKS: ReadonlySet<string> = new Set(['tpch', 'tpcds', 'tpch-nested']);
443+
441444
/**
442-
* A file-size chart's name: the benchmark, with its scale factor unless it is the default `1.0`.
443-
* The monorepo's capture script records `1.0` for a benchmark with no scale-factor directory.
445+
* A file-size chart's name: the benchmark, with its scale factor unless it is the default `1.0`
446+
* of a benchmark with no scale factors. The monorepo's capture script records `1.0` for a
447+
* benchmark with no scale-factor directory.
444448
*/
445449
function fileSizeChartName(benchmark: string, scaleFactor: string): string {
446-
return scaleFactor === '1.0' ? benchmark : `${benchmark} sf=${scaleFactor}`;
450+
return scaleFactor === '1.0' && !SCALED_FILE_SIZE_BENCHMARKS.has(benchmark)
451+
? benchmark
452+
: `${benchmark} sf=${scaleFactor}`;
447453
}
448454

449455
/** A chart's base `unit_kind`: sizes are bytes, everything else is a time. */
@@ -1236,12 +1242,13 @@ export interface GroupChartsResponse {
12361242
charts: NamedChartResponse[];
12371243
}
12381244

1239-
/** Canonical group order for the v4 site. Unknown groups sort last by name. */
1245+
/**
1246+
* Canonical group order for the v4 site. Unknown groups follow by name, then
1247+
* [`GROUP_ORDER_LAST`].
1248+
*/
12401249
const GROUP_ORDER: readonly string[] = [
12411250
'Compression',
12421251
'Compression Size',
1243-
'String Encoding',
1244-
'String Encoding Size',
12451252
'SQL File Size',
12461253
'Random Access (NVMe)',
12471254
'Random Access (S3)',
@@ -1264,14 +1271,21 @@ const GROUP_ORDER: readonly string[] = [
12641271
'appian [nvme]',
12651272
];
12661273

1274+
/** Groups that sort after every other group, in this order: the string-bench groups. */
1275+
const GROUP_ORDER_LAST: readonly string[] = ['String Encoding', 'String Encoding Size'];
1276+
12671277
/**
1268-
* Sort key for a group name against [`GROUP_ORDER`]: names in the list sort by
1269-
* position; names not in the list sort after, with an alphabetical tiebreak.
1270-
* Mirrors the Rust `group_sort_key` `(usize, &str)` tuple.
1278+
* Sort key for a group name: names in [`GROUP_ORDER`] sort by position, then
1279+
* names in neither list alphabetically, then names in [`GROUP_ORDER_LAST`] by
1280+
* position. Mirrors the Rust `group_sort_key` `(usize, &str)` tuple.
12711281
*/
12721282
function groupSortKey(name: string): [number, string] {
12731283
const pos = GROUP_ORDER.indexOf(name);
1274-
return [pos === -1 ? GROUP_ORDER.length : pos, name];
1284+
if (pos !== -1) {
1285+
return [pos, name];
1286+
}
1287+
const last = GROUP_ORDER_LAST.indexOf(name);
1288+
return [last === -1 ? GROUP_ORDER.length : GROUP_ORDER.length + 1 + last, name];
12751289
}
12761290

12771291
/** Comparator over [`groupSortKey`] applying the canonical group ordering. */
@@ -1659,9 +1673,19 @@ async function collectVectorSearchGroups(): Promise<Group[]> {
16591673

16601674
type FileSizeScopeRow = { benchmark: string; scale_factor: string };
16611675

1676+
/** Benchmarks whose file-size charts lead the group, in this order; the rest follow by name. */
1677+
const FILE_SIZE_CHART_ORDER: readonly string[] = ['clickbench', 'tpch', 'tpcds'];
1678+
1679+
/** A benchmark's position in the file-size group: its `FILE_SIZE_CHART_ORDER` index, else after. */
1680+
function fileSizeChartRank(benchmark: string): number {
1681+
const pos = FILE_SIZE_CHART_ORDER.indexOf(benchmark);
1682+
return pos === -1 ? FILE_SIZE_CHART_ORDER.length : pos;
1683+
}
1684+
16621685
/**
16631686
* The `SQL File Size` group: one chart per benchmark and scale factor, or `[]` if the table is
1664-
* empty. Charts order by benchmark, then by numeric scale factor, so `10.0` follows `1.0`.
1687+
* empty. Charts order by [`FILE_SIZE_CHART_ORDER`], then by benchmark name, then by numeric
1688+
* scale factor, so `10.0` follows `1.0`.
16651689
*/
16661690
async function collectFileSizeGroup(): Promise<Group[]> {
16671691
// A recursive skip scan of `idx_file_sizes_baseline`: one descent per scope instead of a scan
@@ -1691,6 +1715,7 @@ async function collectFileSizeGroup(): Promise<Group[]> {
16911715
}
16921716
rows.sort(
16931717
(a, b) =>
1718+
fileSizeChartRank(a.benchmark) - fileSizeChartRank(b.benchmark) ||
16941719
compareCodeUnits(a.benchmark, b.benchmark) ||
16951720
Number(a.scale_factor) - Number(b.scale_factor) ||
16961721
compareCodeUnits(a.scale_factor, b.scale_factor),

0 commit comments

Comments
 (0)