|
| 1 | +#!/bin/bash |
| 2 | +# SPDX-License-Identifier: Apache-2.0 |
| 3 | +# SPDX-FileCopyrightText: Copyright the Vortex contributors |
| 4 | +# |
| 5 | +# Runs SQL benchmarks (datafusion-bench, duckdb-bench, lance-bench) for the given targets. |
| 6 | +# This script is used by the sql-benchmarks.yml workflow. |
| 7 | +# |
| 8 | +# Usage: |
| 9 | +# run-sql-bench.sh <subcommand> <targets> [options] |
| 10 | +# |
| 11 | +# Arguments: |
| 12 | +# subcommand The benchmark subcommand (e.g., tpch, clickbench, tpcds) |
| 13 | +# targets Comma-separated list of engine:format pairs |
| 14 | +# (e.g., "datafusion:parquet,datafusion:vortex,duckdb:parquet") |
| 15 | +# |
| 16 | +# Options: |
| 17 | +# --scale-factor <sf> Scale factor for the benchmark (e.g., 1.0, 10.0) |
| 18 | +# --remote-storage <url> Remote storage URL (e.g., s3://bucket/path/) |
| 19 | +# If provided, runs in remote mode (no lance support). |
| 20 | +# --benchmark-id <id> Benchmark ID for error messages (e.g., tpch-s3) |
| 21 | + |
| 22 | +set -Eeu -o pipefail |
| 23 | + |
| 24 | +subcommand="$1" |
| 25 | +targets="$2" |
| 26 | +shift 2 |
| 27 | + |
| 28 | +scale_factor="" |
| 29 | +remote_storage="" |
| 30 | +benchmark_id="" |
| 31 | + |
| 32 | +while [[ $# -gt 0 ]]; do |
| 33 | + case "$1" in |
| 34 | + --scale-factor) |
| 35 | + scale_factor="$2" |
| 36 | + shift 2 |
| 37 | + ;; |
| 38 | + --remote-storage) |
| 39 | + remote_storage="$2" |
| 40 | + shift 2 |
| 41 | + ;; |
| 42 | + --benchmark-id) |
| 43 | + benchmark_id="$2" |
| 44 | + shift 2 |
| 45 | + ;; |
| 46 | + *) |
| 47 | + echo "Unknown option: $1" >&2 |
| 48 | + exit 1 |
| 49 | + ;; |
| 50 | + esac |
| 51 | +done |
| 52 | + |
| 53 | +is_remote=false |
| 54 | +if [[ -n "$remote_storage" ]]; then |
| 55 | + is_remote=true |
| 56 | +fi |
| 57 | + |
| 58 | +# Lance on remote storage is not supported. The infrastructure to generate and upload lance files |
| 59 | +# to S3 does not exist. If you need lance on S3, you must first implement: |
| 60 | +# 1. Lance data generation in data-gen (or a separate step) |
| 61 | +# 2. Lance data upload to S3 before this step runs |
| 62 | +if $is_remote && echo "$targets" | grep -q 'lance'; then |
| 63 | + echo "ERROR: Lance format is not supported for remote storage benchmarks." |
| 64 | + echo "Remove 'datafusion:lance' from targets for benchmark '${benchmark_id:-unknown}'." |
| 65 | + exit 1 |
| 66 | +fi |
| 67 | + |
| 68 | +# Extract formats for each engine from the targets string. |
| 69 | +# Example input: "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet" |
| 70 | +# |
| 71 | +# Pipeline: split by comma -> filter by engine prefix -> remove prefix -> rejoin with commas |
| 72 | +# |
| 73 | +# Lance is filtered out of df_formats because it uses a separate binary (lance-bench). |
| 74 | +# |
| 75 | +# The `|| true` is needed because some benchmarks don't use all engines (e.g., statpopgen only has |
| 76 | +# duckdb targets). grep returns exit code 1 when no matches are found. Both greps must be in the |
| 77 | +# subshell so that `|| true` covers the case where grep -v receives empty input. |
| 78 | +df_formats=$(echo "$targets" | tr ',' '\n' | (grep '^datafusion:' | grep -v ':lance$' || true) | sed 's/datafusion://' | tr '\n' ',' | sed 's/,$//') |
| 79 | +ddb_formats=$(echo "$targets" | tr ',' '\n' | (grep '^duckdb:' || true) | sed 's/duckdb://' | tr '\n' ',' | sed 's/,$//') |
| 80 | +has_lance=$(echo "$targets" | grep -q 'datafusion:lance' && echo "true" || echo "false") |
| 81 | + |
| 82 | +# Build options string. |
| 83 | +opts="" |
| 84 | +if $is_remote; then |
| 85 | + opts="--opt remote-data-dir=$remote_storage" |
| 86 | +fi |
| 87 | +if [[ -n "$scale_factor" ]]; then |
| 88 | + if [[ -n "$opts" ]]; then |
| 89 | + opts="--opt scale-factor=$scale_factor $opts" |
| 90 | + else |
| 91 | + opts="--opt scale-factor=$scale_factor" |
| 92 | + fi |
| 93 | +fi |
| 94 | + |
| 95 | +touch results.json |
| 96 | + |
| 97 | +if [[ -n "$df_formats" ]]; then |
| 98 | + # shellcheck disable=SC2086 |
| 99 | + target/release_debug/datafusion-bench "$subcommand" \ |
| 100 | + -d gh-json \ |
| 101 | + --formats "$df_formats" \ |
| 102 | + $opts \ |
| 103 | + -o df-results.json |
| 104 | + |
| 105 | + cat df-results.json >> results.json |
| 106 | +fi |
| 107 | + |
| 108 | +if [[ -n "$ddb_formats" ]]; then |
| 109 | + # shellcheck disable=SC2086 |
| 110 | + target/release_debug/duckdb-bench "$subcommand" \ |
| 111 | + -d gh-json \ |
| 112 | + --formats "$ddb_formats" \ |
| 113 | + $opts \ |
| 114 | + --delete-duckdb-database \ |
| 115 | + -o ddb-results.json |
| 116 | + |
| 117 | + cat ddb-results.json >> results.json |
| 118 | +fi |
| 119 | + |
| 120 | +# Lance-bench only runs for local benchmarks. |
| 121 | +if ! $is_remote && [[ "$has_lance" == "true" ]] && [[ -f "target/release_debug/lance-bench" ]]; then |
| 122 | + # shellcheck disable=SC2086 |
| 123 | + target/release_debug/lance-bench "$subcommand" \ |
| 124 | + -d gh-json \ |
| 125 | + $opts \ |
| 126 | + -o lance-results.json |
| 127 | + |
| 128 | + cat lance-results.json >> results.json |
| 129 | +fi |
0 commit comments