@@ -109,6 +109,7 @@ clickbench_extended: ClickBench \"inspired\" queries against a single parquet
109109# Sort Pushdown Benchmarks
110110sort_pushdown: Sort pushdown baseline (no WITH ORDER) on TPC-H data (SF=1)
111111sort_pushdown_sorted: Sort pushdown with WITH ORDER — tests sort elimination on non-overlapping files
112+ sort_pushdown_inexact: Sort pushdown Inexact path — tests RG reorder by statistics for TopK queries
112113
113114# Sorted Data Benchmarks (ORDER BY Optimization)
114115clickbench_sorted: ClickBench queries on pre-sorted data using prefer_existing_sort (tests sort elimination optimization)
@@ -316,6 +317,9 @@ main() {
316317 sort_pushdown|sort_pushdown_sorted)
317318 data_sort_pushdown
318319 ;;
320+ sort_pushdown_inexact)
321+ data_sort_pushdown_inexact
322+ ;;
319323 sort_tpch)
320324 # same data as for tpch
321325 data_tpch " 1" " parquet"
@@ -522,6 +526,9 @@ main() {
522526 sort_pushdown_sorted)
523527 run_sort_pushdown_sorted
524528 ;;
529+ sort_pushdown_inexact)
530+ run_sort_pushdown_inexact
531+ ;;
525532 sort_tpch)
526533 run_sort_tpch " 1"
527534 ;;
@@ -1137,6 +1144,62 @@ run_sort_pushdown_sorted() {
11371144 debug_run $CARGO_COMMAND --bin dfbench -- sort-pushdown --sorted --iterations 5 --path " ${SORT_PUSHDOWN_DIR} " --queries-path " ${SCRIPT_DIR} /queries/sort_pushdown" -o " ${RESULTS_FILE} " ${QUERY_ARG} ${LATENCY_ARG}
11381145}
11391146
1147+ # Generates data for sort pushdown Inexact benchmark.
1148+ #
1149+ # Produces a single large lineitem parquet file where row groups have
1150+ # NON-OVERLAPPING but OUT-OF-ORDER l_orderkey ranges (each RG internally
1151+ # sorted, RGs shuffled). This simulates append-heavy workloads where data
1152+ # is written in batches at different times.
1153+ data_sort_pushdown_inexact () {
1154+ INEXACT_DIR=" ${DATA_DIR} /sort_pushdown_inexact/lineitem"
1155+ if [ -d " ${INEXACT_DIR} " ] && [ " $( ls -A ${INEXACT_DIR} /* .parquet 2> /dev/null) " ]; then
1156+ echo " Sort pushdown Inexact data already exists at ${INEXACT_DIR} "
1157+ return
1158+ fi
1159+
1160+ echo " Generating sort pushdown Inexact benchmark data (single file, shuffled RGs)..."
1161+
1162+ # Re-use the sort_pushdown data as the source (generate if missing)
1163+ data_sort_pushdown
1164+
1165+ mkdir -p " ${INEXACT_DIR} "
1166+ SRC_DIR=" ${DATA_DIR} /sort_pushdown/lineitem"
1167+
1168+ # Use datafusion-cli to bucket rows into 64 groups by a deterministic
1169+ # scrambler, then sort within each bucket by orderkey. This produces
1170+ # ~64 RG-sized segments where each has a tight orderkey range but the
1171+ # segments appear in scrambled (non-sorted) order in the file.
1172+ (cd " ${SCRIPT_DIR} /.." && cargo run --release -p datafusion-cli -- -c "
1173+ CREATE EXTERNAL TABLE src
1174+ STORED AS PARQUET
1175+ LOCATION '${SRC_DIR} ';
1176+
1177+ COPY (
1178+ SELECT * FROM src
1179+ ORDER BY
1180+ (l_orderkey * 1664525 + 1013904223) % 64,
1181+ l_orderkey
1182+ )
1183+ TO '${INEXACT_DIR} /shuffled.parquet'
1184+ STORED AS PARQUET
1185+ OPTIONS ('format.max_row_group_size' '100000');
1186+ " )
1187+
1188+ echo " Sort pushdown Inexact data generated at ${INEXACT_DIR} "
1189+ ls -la " ${INEXACT_DIR} "
1190+ }
1191+
1192+ # Runs the sort pushdown Inexact benchmark (tests RG reorder by statistics).
1193+ # Enables pushdown_filters so TopK's dynamic filter is pushed to the parquet
1194+ # reader for late materialization (only needed for Inexact path).
1195+ run_sort_pushdown_inexact () {
1196+ INEXACT_DIR=" ${DATA_DIR} /sort_pushdown_inexact"
1197+ RESULTS_FILE=" ${RESULTS_DIR} /sort_pushdown_inexact.json"
1198+ echo " Running sort pushdown Inexact benchmark (row group reorder by statistics)..."
1199+ DATAFUSION_EXECUTION_PARQUET_PUSHDOWN_FILTERS=true \
1200+ debug_run $CARGO_COMMAND --bin dfbench -- sort-pushdown --sorted --iterations 5 --path " ${INEXACT_DIR} " --queries-path " ${SCRIPT_DIR} /queries/sort_pushdown_inexact" -o " ${RESULTS_FILE} " ${QUERY_ARG} ${LATENCY_ARG}
1201+ }
1202+
11401203# Runs the sort integration benchmark
11411204run_sort_tpch () {
11421205 SCALE_FACTOR=$1
0 commit comments