@@ -316,9 +316,6 @@ main() {
316316 sort_pushdown|sort_pushdown_sorted)
317317 data_sort_pushdown
318318 ;;
319- sort_pushdown_inexact)
320- data_sort_pushdown_inexact
321- ;;
322319 sort_tpch)
323320 # same data as for tpch
324321 data_tpch " 1" " parquet"
@@ -525,9 +522,6 @@ main() {
525522 sort_pushdown_sorted)
526523 run_sort_pushdown_sorted
527524 ;;
528- sort_pushdown_inexact)
529- run_sort_pushdown_inexact
530- ;;
531525 sort_tpch)
532526 run_sort_tpch " 1"
533527 ;;
@@ -1143,77 +1137,6 @@ run_sort_pushdown_sorted() {
11431137 debug_run $CARGO_COMMAND --bin dfbench -- sort-pushdown --sorted --iterations 5 --path " ${SORT_PUSHDOWN_DIR} " --queries-path " ${SCRIPT_DIR} /queries/sort_pushdown" -o " ${RESULTS_FILE} " ${QUERY_ARG} ${LATENCY_ARG}
11441138}
11451139
1146- # Generates data for sort pushdown Inexact benchmark.
1147- #
1148- # Unlike sort_pushdown (Exact path), this benchmark targets the Inexact path
1149- # where row group reorder by statistics is beneficial. It produces a single
1150- # large lineitem parquet file where row groups have NON-OVERLAPPING but
1151- # OUT-OF-ORDER l_orderkey ranges (each RG internally sorted, RGs shuffled).
1152- #
1153- # This simulates append-heavy workloads where data is written in batches at
1154- # different times, producing segments with tight value ranges but in arbitrary
1155- # row-group order.
1156- data_sort_pushdown_inexact () {
1157- INEXACT_DIR=" ${DATA_DIR} /sort_pushdown_inexact/lineitem"
1158- if [ -d " ${INEXACT_DIR} " ] && [ " $( ls -A ${INEXACT_DIR} /* .parquet 2> /dev/null) " ]; then
1159- echo " Sort pushdown Inexact data already exists at ${INEXACT_DIR} "
1160- return
1161- fi
1162-
1163- echo " Generating sort pushdown Inexact benchmark data (single file, shuffled RGs)..."
1164-
1165- # Re-use the sort_pushdown data as the source (generate if missing)
1166- data_sort_pushdown
1167-
1168- mkdir -p " ${INEXACT_DIR} "
1169- SRC_DIR=" ${DATA_DIR} /sort_pushdown/lineitem"
1170-
1171- # Use datafusion-cli to:
1172- # 1. Read the 3 source files as one table
1173- # 2. Shuffle row order by hashing l_orderkey (produces deterministic but
1174- # non-sorted output — each RG will cover a scattered subset of orderkeys
1175- # with no overlap between RGs once row_group_size is small)
1176- # 3. Write a single parquet file with small max_row_group_size so we get
1177- # many RGs per file.
1178- #
1179- # Note: with pure hash-shuffle, each RG still covers the full orderkey range.
1180- # To get tight RGs with non-overlapping ranges we bucket by hash first then
1181- # sort within bucket.
1182- (cd " ${SCRIPT_DIR} /.." && cargo run --release -p datafusion-cli -- -c "
1183- CREATE EXTERNAL TABLE src
1184- STORED AS PARQUET
1185- LOCATION '${SRC_DIR} ';
1186-
1187- -- Bucket rows: split orderkey range into 64 chunks, scramble chunk
1188- -- order, but sort within each chunk. This produces ~64 RG-sized
1189- -- segments where each segment has a tight orderkey range but the
1190- -- segments appear in scrambled (non-sorted) order in the file.
1191- -- We use (l_orderkey * 1664525 + 1013904223) % 64 as a deterministic
1192- -- scrambler (linear congruential generator) so bucket order is
1193- -- effectively random but reproducible.
1194- COPY (
1195- SELECT * FROM src
1196- ORDER BY
1197- (l_orderkey * 1664525 + 1013904223) % 64,
1198- l_orderkey
1199- )
1200- TO '${INEXACT_DIR} /shuffled.parquet'
1201- STORED AS PARQUET
1202- OPTIONS ('format.max_row_group_size' '100000');
1203- " )
1204-
1205- echo " Sort pushdown Inexact data generated at ${INEXACT_DIR} "
1206- ls -la " ${INEXACT_DIR} "
1207- }
1208-
1209- # Runs the sort pushdown Inexact benchmark (tests RG reorder by statistics).
1210- run_sort_pushdown_inexact () {
1211- INEXACT_DIR=" ${DATA_DIR} /sort_pushdown_inexact"
1212- RESULTS_FILE=" ${RESULTS_DIR} /sort_pushdown_inexact.json"
1213- echo " Running sort pushdown Inexact benchmark (row group reorder by statistics)..."
1214- debug_run $CARGO_COMMAND --bin dfbench -- sort-pushdown --sorted --iterations 5 --path " ${INEXACT_DIR} " --queries-path " ${SCRIPT_DIR} /queries/sort_pushdown_inexact" -o " ${RESULTS_FILE} " ${QUERY_ARG} ${LATENCY_ARG}
1215- }
1216-
12171140# Runs the sort integration benchmark
12181141run_sort_tpch () {
12191142 SCALE_FACTOR=$1
0 commit comments