Skip to content

Commit 15b13e6

Browse files
committed
chore: remove benchmark from this PR (tracked in apache#21582)
1 parent f7e5c92 commit 15b13e6

6 files changed

Lines changed: 1 addition & 115 deletions

File tree

‎benchmarks/bench.sh‎

Lines changed: 0 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -316,9 +316,6 @@ main() {
316316
sort_pushdown|sort_pushdown_sorted)
317317
data_sort_pushdown
318318
;;
319-
sort_pushdown_inexact)
320-
data_sort_pushdown_inexact
321-
;;
322319
sort_tpch)
323320
# same data as for tpch
324321
data_tpch "1" "parquet"
@@ -525,9 +522,6 @@ main() {
525522
sort_pushdown_sorted)
526523
run_sort_pushdown_sorted
527524
;;
528-
sort_pushdown_inexact)
529-
run_sort_pushdown_inexact
530-
;;
531525
sort_tpch)
532526
run_sort_tpch "1"
533527
;;
@@ -1143,77 +1137,6 @@ run_sort_pushdown_sorted() {
11431137
debug_run $CARGO_COMMAND --bin dfbench -- sort-pushdown --sorted --iterations 5 --path "${SORT_PUSHDOWN_DIR}" --queries-path "${SCRIPT_DIR}/queries/sort_pushdown" -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
11441138
}
11451139

1146-
# Generates data for sort pushdown Inexact benchmark.
1147-
#
1148-
# Unlike sort_pushdown (Exact path), this benchmark targets the Inexact path
1149-
# where row group reorder by statistics is beneficial. It produces a single
1150-
# large lineitem parquet file where row groups have NON-OVERLAPPING but
1151-
# OUT-OF-ORDER l_orderkey ranges (each RG internally sorted, RGs shuffled).
1152-
#
1153-
# This simulates append-heavy workloads where data is written in batches at
1154-
# different times, producing segments with tight value ranges but in arbitrary
1155-
# row-group order.
1156-
data_sort_pushdown_inexact() {
1157-
INEXACT_DIR="${DATA_DIR}/sort_pushdown_inexact/lineitem"
1158-
if [ -d "${INEXACT_DIR}" ] && [ "$(ls -A ${INEXACT_DIR}/*.parquet 2>/dev/null)" ]; then
1159-
echo "Sort pushdown Inexact data already exists at ${INEXACT_DIR}"
1160-
return
1161-
fi
1162-
1163-
echo "Generating sort pushdown Inexact benchmark data (single file, shuffled RGs)..."
1164-
1165-
# Re-use the sort_pushdown data as the source (generate if missing)
1166-
data_sort_pushdown
1167-
1168-
mkdir -p "${INEXACT_DIR}"
1169-
SRC_DIR="${DATA_DIR}/sort_pushdown/lineitem"
1170-
1171-
# Use datafusion-cli to:
1172-
# 1. Read the 3 source files as one table
1173-
# 2. Shuffle row order by hashing l_orderkey (produces deterministic but
1174-
# non-sorted output — each RG will cover a scattered subset of orderkeys
1175-
# with no overlap between RGs once row_group_size is small)
1176-
# 3. Write a single parquet file with small max_row_group_size so we get
1177-
# many RGs per file.
1178-
#
1179-
# Note: with pure hash-shuffle, each RG still covers the full orderkey range.
1180-
# To get tight RGs with non-overlapping ranges we bucket by hash first then
1181-
# sort within bucket.
1182-
(cd "${SCRIPT_DIR}/.." && cargo run --release -p datafusion-cli -- -c "
1183-
CREATE EXTERNAL TABLE src
1184-
STORED AS PARQUET
1185-
LOCATION '${SRC_DIR}';
1186-
1187-
-- Bucket rows: split orderkey range into 64 chunks, scramble chunk
1188-
-- order, but sort within each chunk. This produces ~64 RG-sized
1189-
-- segments where each segment has a tight orderkey range but the
1190-
-- segments appear in scrambled (non-sorted) order in the file.
1191-
-- We use (l_orderkey * 1664525 + 1013904223) % 64 as a deterministic
1192-
-- scrambler (linear congruential generator) so bucket order is
1193-
-- effectively random but reproducible.
1194-
COPY (
1195-
SELECT * FROM src
1196-
ORDER BY
1197-
(l_orderkey * 1664525 + 1013904223) % 64,
1198-
l_orderkey
1199-
)
1200-
TO '${INEXACT_DIR}/shuffled.parquet'
1201-
STORED AS PARQUET
1202-
OPTIONS ('format.max_row_group_size' '100000');
1203-
")
1204-
1205-
echo "Sort pushdown Inexact data generated at ${INEXACT_DIR}"
1206-
ls -la "${INEXACT_DIR}"
1207-
}
1208-
1209-
# Runs the sort pushdown Inexact benchmark (tests RG reorder by statistics).
1210-
run_sort_pushdown_inexact() {
1211-
INEXACT_DIR="${DATA_DIR}/sort_pushdown_inexact"
1212-
RESULTS_FILE="${RESULTS_DIR}/sort_pushdown_inexact.json"
1213-
echo "Running sort pushdown Inexact benchmark (row group reorder by statistics)..."
1214-
debug_run $CARGO_COMMAND --bin dfbench -- sort-pushdown --sorted --iterations 5 --path "${INEXACT_DIR}" --queries-path "${SCRIPT_DIR}/queries/sort_pushdown_inexact" -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
1215-
}
1216-
12171140
# Runs the sort integration benchmark
12181141
run_sort_tpch() {
12191142
SCALE_FACTOR=$1

‎benchmarks/queries/sort_pushdown_inexact/q1.sql‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎benchmarks/queries/sort_pushdown_inexact/q2.sql‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎benchmarks/queries/sort_pushdown_inexact/q3.sql‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎benchmarks/queries/sort_pushdown_inexact/q4.sql‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎benchmarks/src/sort_pushdown.rs‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -159,14 +159,7 @@ impl RunOpt {
159159
async fn benchmark_query(&self, query_id: usize) -> Result<Vec<QueryResult>> {
160160
let sql = self.load_query(query_id)?;
161161

162-
let mut config = self.common.config()?;
163-
// Enable parquet filter pushdown + late materialization. This is
164-
// essential for the Inexact sort pushdown path: TopK's dynamic
165-
// filter is pushed to the parquet reader, so only sort-column
166-
// rows pass the filter's Decode non-sort columns are skipped for
167-
// rows that don't pass the filter — this is where RG reorder's
168-
// tight-threshold-first strategy pays off for wide-row queries.
169-
config.options_mut().execution.parquet.pushdown_filters = true;
162+
let config = self.common.config()?;
170163
let rt = self.common.build_runtime()?;
171164
let state = SessionStateBuilder::new()
172165
.with_config(config)

0 commit comments

Comments
 (0)