Skip to content

Commit cc890a7

Browse files
committed
Merge upstream main and resolve Parquet config conflicts
2 parents ac49e2e + 6fc7baa commit cc890a7

93 files changed

Lines changed: 3904 additions & 705 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎benchmarks/bench.sh‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,12 @@ h2o_big_join: h2oai benchmark with large dataset (1e9 rows) fo
139139
h2o_small_window: Extended h2oai benchmark with small dataset (1e7 rows) for window, default file format is csv
140140
h2o_medium_window: Extended h2oai benchmark with medium dataset (1e8 rows) for window, default file format is csv
141141
h2o_big_window: Extended h2oai benchmark with large dataset (1e9 rows) for window, default file format is csv
142+
h2o_small_window_sorted: Window Top-N over a declared-sorted h2o input, small dataset (1e7 rows), default file format is csv
143+
h2o_medium_window_sorted: Window Top-N over a declared-sorted h2o input, medium dataset (1e8 rows), default file format is csv
144+
h2o_big_window_sorted: Window Top-N over a declared-sorted h2o input, large dataset (1e9 rows), default file format is csv
145+
h2o_small_window_sorted_parquet: Window Top-N over a declared-sorted h2o input, small dataset (1e7 rows), source file format is parquet
146+
h2o_medium_window_sorted_parquet: Window Top-N over a declared-sorted h2o input, medium dataset (1e8 rows), source file format is parquet
147+
h2o_big_window_sorted_parquet: Window Top-N over a declared-sorted h2o input, large dataset (1e9 rows), source file format is parquet
142148
h2o_small_parquet: h2oai benchmark with small dataset (1e7 rows) for groupby, file format is parquet
143149
h2o_medium_parquet: h2oai benchmark with medium dataset (1e8 rows) for groupby, file format is parquet
144150
h2o_big_parquet: h2oai benchmark with large dataset (1e9 rows) for groupby, file format is parquet
@@ -309,6 +315,26 @@ main() {
309315
h2o_big_window)
310316
data_h2o_join "BIG" "CSV"
311317
;;
318+
# the sorted window subgroup derives its data from the same
319+
# source, then sorts it inside its load SQL
320+
h2o_small_window_sorted)
321+
data_h2o_join "SMALL" "CSV"
322+
;;
323+
h2o_medium_window_sorted)
324+
data_h2o_join "MEDIUM" "CSV"
325+
;;
326+
h2o_big_window_sorted)
327+
data_h2o_join "BIG" "CSV"
328+
;;
329+
h2o_small_window_sorted_parquet)
330+
data_h2o_join "SMALL" "PARQUET"
331+
;;
332+
h2o_medium_window_sorted_parquet)
333+
data_h2o_join "MEDIUM" "PARQUET"
334+
;;
335+
h2o_big_window_sorted_parquet)
336+
data_h2o_join "BIG" "PARQUET"
337+
;;
312338
h2o_small_parquet)
313339
data_h2o "SMALL" "PARQUET"
314340
;;
@@ -534,6 +560,24 @@ main() {
534560
h2o_big_window)
535561
run_h2o_window "BIG" "CSV" "window"
536562
;;
563+
h2o_small_window_sorted)
564+
run_h2o_window_sorted "small" "csv"
565+
;;
566+
h2o_medium_window_sorted)
567+
run_h2o_window_sorted "medium" "csv"
568+
;;
569+
h2o_big_window_sorted)
570+
run_h2o_window_sorted "big" "csv"
571+
;;
572+
h2o_small_window_sorted_parquet)
573+
run_h2o_window_sorted "small" "parquet"
574+
;;
575+
h2o_medium_window_sorted_parquet)
576+
run_h2o_window_sorted "medium" "parquet"
577+
;;
578+
h2o_big_window_sorted_parquet)
579+
run_h2o_window_sorted "big" "parquet"
580+
;;
537581
h2o_small_parquet)
538582
run_h2o "SMALL" "PARQUET"
539583
;;
@@ -1266,6 +1310,38 @@ run_h2o_window() {
12661310
h2o_runner "$1" "$2" "window"
12671311
}
12681312

1313+
# Runs the h2o window_sorted subgroup: window Top-N over an input that declares
1314+
# the ordering the window requires.
1315+
#
1316+
# The `window` subgroup registers `x` with no declared ordering, so
1317+
# `output_ordering()` is None and any plan that depends on a declared ordering is
1318+
# unreachable from it, however the data happens to sit on disk. This subgroup's
1319+
# load SQL writes a sorted copy and registers it `WITH ORDER`, then asserts a
1320+
# per-partition top-K operator is actually in the plan so a silent fallback to
1321+
# window-plus-filter cannot masquerade as a result.
1322+
#
1323+
# The sort happens in the untimed `load` step, so it stays out of the
1324+
# measurement. Data comes from the same source as the window subgroup, so the
1325+
# data step is data_h2o_join.
1326+
#
1327+
# H2O_FILE_TYPE selects the *source* format and so only affects that untimed
1328+
# load: the measured query always reads the sorted Parquet copy, whatever the
1329+
# source was. A Parquet source still makes the load markedly cheaper than
1330+
# re-parsing a multi-GB CSV, which is why the `_parquet` entries exist; they are
1331+
# expected to produce the same measured numbers as their CSV counterparts, not
1332+
# different ones. Each entry pairs with the data step that generates its format.
1333+
run_h2o_window_sorted() {
1334+
SIZE=${1:-"small"}
1335+
FILE_TYPE=${2:-"csv"}
1336+
echo "Running h2o window_sorted benchmark (size=${SIZE}, source format=${FILE_TYPE})..."
1337+
debug_run env BENCH_NAME=h2o \
1338+
BENCH_SUBGROUP=window_sorted \
1339+
H2O_BENCH_SIZE="${SIZE}" \
1340+
H2O_FILE_TYPE="${FILE_TYPE}" \
1341+
${QUERY:+BENCH_QUERY="${QUERY}"} \
1342+
bash -c "$SQL_CARGO_COMMAND"
1343+
}
1344+
12691345
# Runs the external aggregation benchmark
12701346
run_external_aggr() {
12711347
# Use TPC-H SF1 dataset
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# ROW_NUMBER top-2 per partition over the declared-sorted input, swept across
2+
# partition cardinality. This mirrors the `window` subgroup q14-q17 sweep, with
3+
# the partition key materialized as a column so the ordering can be declared.
4+
5+
subgroup window_sorted
6+
7+
template sql_benchmarks/h2o/window_sorted.benchmark.template
8+
QPAD=01
9+
FN=ROW_NUMBER
10+
PK_MOD=100
11+
OB_EXPR=v2
12+
OB_BOUND=count(DISTINCT ob) > 1000
13+
NAME=window_sorted_q01_row_number_100_partitions
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# ROW_NUMBER top-2, ~1K partitions.
2+
3+
subgroup window_sorted
4+
5+
template sql_benchmarks/h2o/window_sorted.benchmark.template
6+
QPAD=02
7+
FN=ROW_NUMBER
8+
PK_MOD=1000
9+
OB_EXPR=v2
10+
OB_BOUND=count(DISTINCT ob) > 1000
11+
NAME=window_sorted_q02_row_number_1k_partitions
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# ROW_NUMBER top-2, ~10K partitions.
2+
3+
subgroup window_sorted
4+
5+
template sql_benchmarks/h2o/window_sorted.benchmark.template
6+
QPAD=03
7+
FN=ROW_NUMBER
8+
PK_MOD=10000
9+
OB_EXPR=v2
10+
OB_BOUND=count(DISTINCT ob) > 1000
11+
NAME=window_sorted_q03_row_number_10k_partitions
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# ROW_NUMBER top-2, ~100K partitions. Fewest rows per partition on the sweep,
2+
# so per-partition bookkeeping is amortized over the least work.
3+
4+
subgroup window_sorted
5+
6+
template sql_benchmarks/h2o/window_sorted.benchmark.template
7+
QPAD=04
8+
FN=ROW_NUMBER
9+
PK_MOD=100000
10+
OB_EXPR=v2
11+
OB_BOUND=count(DISTINCT ob) > 1000
12+
NAME=window_sorted_q04_row_number_100k_partitions
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# RANK top-2 per partition, mirroring the ROW_NUMBER sweep above. RANK retains
2+
# boundary ties, so `WHERE rnk <= 2` can keep more than 2 rows per partition and
3+
# the operator has to track the tied set rather than a fixed row count. `v2` is
4+
# near-unique, so these four measure RANK with the tie machinery idle; q09-q10
5+
# drive it.
6+
7+
subgroup window_sorted
8+
9+
template sql_benchmarks/h2o/window_sorted.benchmark.template
10+
QPAD=05
11+
FN=RANK
12+
PK_MOD=100
13+
OB_EXPR=v2
14+
OB_BOUND=count(DISTINCT ob) > 1000
15+
NAME=window_sorted_q05_rank_100_partitions
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# RANK top-2, ~1K partitions.
2+
3+
subgroup window_sorted
4+
5+
template sql_benchmarks/h2o/window_sorted.benchmark.template
6+
QPAD=06
7+
FN=RANK
8+
PK_MOD=1000
9+
OB_EXPR=v2
10+
OB_BOUND=count(DISTINCT ob) > 1000
11+
NAME=window_sorted_q06_rank_1k_partitions
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# RANK top-2, ~10K partitions.
2+
3+
subgroup window_sorted
4+
5+
template sql_benchmarks/h2o/window_sorted.benchmark.template
6+
QPAD=07
7+
FN=RANK
8+
PK_MOD=10000
9+
OB_EXPR=v2
10+
OB_BOUND=count(DISTINCT ob) > 1000
11+
NAME=window_sorted_q07_rank_10k_partitions
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# RANK top-2, ~100K partitions.
2+
3+
subgroup window_sorted
4+
5+
template sql_benchmarks/h2o/window_sorted.benchmark.template
6+
QPAD=08
7+
FN=RANK
8+
PK_MOD=100000
9+
OB_EXPR=v2
10+
OB_BOUND=count(DISTINCT ob) > 1000
11+
NAME=window_sorted_q08_rank_100k_partitions
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# RANK top-2 with heavy ties: 10 distinct ORDER BY values, so a whole partition
2+
# slice ties at rank 1 and every tied row survives `rnk <= 2`. The tied set, not
3+
# the top-K bound, dominates the work.
4+
#
5+
# The cast is load-bearing. `v2` is Float64, so `v2 % 10` is float modulo and
6+
# stays near-unique — it produces no ties whatsoever. Truncating to an integer
7+
# first is what actually collapses the domain to 10 values.
8+
9+
subgroup window_sorted
10+
11+
template sql_benchmarks/h2o/window_sorted.benchmark.template
12+
QPAD=09
13+
FN=RANK
14+
PK_MOD=1000
15+
OB_EXPR=CAST(v2 AS BIGINT) % 10
16+
OB_BOUND=count(DISTINCT ob) = 10
17+
NAME=window_sorted_q09_rank_1k_partitions_heavy_ties

0 commit comments

Comments
 (0)