|
| 1 | +#!/bin/bash |
| 2 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | +# All rights reserved. |
| 4 | +# |
| 5 | +# This source code is licensed under the BSD-style license found in the |
| 6 | +# LICENSE file in the root directory of this source tree. |
| 7 | + |
| 8 | +# Run the CUDA perf benchmark for one (model, quant) pair, reusing the |
| 9 | +# .pte/.ptd artifacts and the already-built runner left behind by |
| 10 | +# test_model_e2e.sh in the same CI job. |
| 11 | +# |
| 12 | +# Only the (model, quant) pairs in the perf allowlist below are benchmarked. |
| 13 | +# Anything else exits 0 after printing BENCHMARK_SKIP, so the caller can tell |
| 14 | +# "not covered by perf" apart from a real failure. The optional |
| 15 | +# models/quantizations filters (from workflow_dispatch) narrow the benchmark |
| 16 | +# stage further without touching the accuracy stage. |
| 17 | +# |
| 18 | +# Usage: |
| 19 | +# cuda_benchmark_from_e2e.sh <hf_model> <quant> <model_dir> <results_dir> |
| 20 | +# [num_runs] [git_sha] [run_id] [run_url] [models_filter] [quants_filter] |
| 21 | +# |
| 22 | +# Outputs (in <results_dir>): |
| 23 | +# benchmark_results.json, benchmark_results_v3.json, metadata.json |
| 24 | + |
| 25 | +set -euo pipefail |
| 26 | + |
| 27 | +HF_MODEL="${1:?hf_model required, e.g. openai/whisper-small}" |
| 28 | +QUANT_NAME="${2:?quant required, e.g. non-quantized}" |
| 29 | +MODEL_DIR="${3:?model_dir required}" |
| 30 | +RESULTS_DIR="${4:?results_dir required}" |
| 31 | +NUM_RUNS="${5:-50}" |
| 32 | +GIT_SHA="${6:-unknown}" |
| 33 | +WORKFLOW_RUN_ID="${7:-0}" |
| 34 | +WORKFLOW_RUN_URL="${8:-}" |
| 35 | +MODELS_FILTER="${9:-}" |
| 36 | +QUANTS_FILTER="${10:-}" |
| 37 | + |
| 38 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 39 | +EXECUTORCH_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" |
| 40 | + |
| 41 | +# (model, quant) pairs covered by perf tracking. Keep in sync with the |
| 42 | +# test-model-cuda-e2e matrix in .github/workflows/cuda.yml. |
| 43 | +is_bench_pair() { |
| 44 | + case "${HF_MODEL} ${QUANT_NAME}" in |
| 45 | + "mistralai/Voxtral-Mini-3B-2507 "* | \ |
| 46 | + "openai/whisper-small non-quantized" | \ |
| 47 | + "openai/whisper-medium non-quantized" | \ |
| 48 | + "openai/whisper-large-v3-turbo quantized-"* | \ |
| 49 | + "google/gemma-3-4b-it non-quantized" | \ |
| 50 | + "google/gemma-3-4b-it quantized-int4-tile-packed" | \ |
| 51 | + "nvidia/parakeet-tdt non-quantized" | \ |
| 52 | + "nvidia/parakeet-tdt quantized-int4-tile-packed" | \ |
| 53 | + "SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4 quantized-int4-tile-packed") |
| 54 | + return 0 |
| 55 | + ;; |
| 56 | + *) |
| 57 | + return 1 |
| 58 | + ;; |
| 59 | + esac |
| 60 | +} |
| 61 | + |
| 62 | +if ! is_bench_pair; then |
| 63 | + echo "BENCHMARK_SKIP: ${HF_MODEL} ${QUANT_NAME} is not in the perf allowlist" |
| 64 | + exit 0 |
| 65 | +fi |
| 66 | + |
| 67 | +# workflow_dispatch filters: comma-separated HF model IDs / quant names. |
| 68 | +# Empty means "benchmark every allowlisted pair". |
| 69 | +case ",${MODELS_FILTER}," in |
| 70 | +*",${HF_MODEL},"*) ;; |
| 71 | +*) |
| 72 | + if [ -n "${MODELS_FILTER}" ]; then |
| 73 | + echo "BENCHMARK_SKIP: ${HF_MODEL} not in dispatch models filter" |
| 74 | + exit 0 |
| 75 | + fi |
| 76 | + ;; |
| 77 | +esac |
| 78 | +case ",${QUANTS_FILTER}," in |
| 79 | +*",${QUANT_NAME},"*) ;; |
| 80 | +*) |
| 81 | + if [ -n "${QUANTS_FILTER}" ]; then |
| 82 | + echo "BENCHMARK_SKIP: ${QUANT_NAME} not in dispatch quantizations filter" |
| 83 | + exit 0 |
| 84 | + fi |
| 85 | + ;; |
| 86 | +esac |
| 87 | + |
| 88 | +cd "${EXECUTORCH_ROOT}" |
| 89 | +export LD_LIBRARY_PATH="/opt/conda/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" |
| 90 | + |
| 91 | +# Runner command mapping, ported from the retired cuda-perf.yml so the |
| 92 | +# benchmark prompts and flags (and therefore the numbers) stay identical. |
| 93 | +case "${HF_MODEL}" in |
| 94 | +mistralai/Voxtral-Mini-3B-2507) |
| 95 | + RUNNER="cmake-out/examples/models/voxtral/voxtral_runner" |
| 96 | + PREPROCESSOR="${MODEL_DIR}/voxtral_preprocessor.pte" |
| 97 | + TOKENIZER="${MODEL_DIR}/tekken.json" |
| 98 | + AUDIO="${MODEL_DIR}/poem.wav" |
| 99 | + RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path $TOKENIZER --audio_path $AUDIO --processor_path $PREPROCESSOR --temperature 0" |
| 100 | + MODEL_NAME="voxtral_${QUANT_NAME}" |
| 101 | + ;; |
| 102 | +openai/whisper-*) |
| 103 | + RUNNER="cmake-out/examples/models/whisper/whisper_runner" |
| 104 | + PREPROCESSOR="${MODEL_DIR}/whisper_preprocessor.pte" |
| 105 | + AUDIO="${MODEL_DIR}/output.wav" |
| 106 | + RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path ${MODEL_DIR}/ --audio_path $AUDIO --processor_path $PREPROCESSOR --temperature 0" |
| 107 | + MODEL_NAME="${HF_MODEL#openai/}_${QUANT_NAME}" |
| 108 | + ;; |
| 109 | +google/gemma-3-4b-it) |
| 110 | + RUNNER="cmake-out/examples/models/gemma3/gemma3_e2e_runner" |
| 111 | + IMAGE="docs/source/_static/img/et-logo.png" |
| 112 | + RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path ${MODEL_DIR}/ --image_path $IMAGE --temperature 0" |
| 113 | + MODEL_NAME="gemma3_${QUANT_NAME}" |
| 114 | + ;; |
| 115 | +nvidia/parakeet-tdt) |
| 116 | + RUNNER="cmake-out/examples/models/parakeet/parakeet_runner" |
| 117 | + AUDIO="${MODEL_DIR}/test_audio.wav" |
| 118 | + TOKENIZER="${MODEL_DIR}/tokenizer.model" |
| 119 | + RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --audio_path $AUDIO --tokenizer_path $TOKENIZER" |
| 120 | + MODEL_NAME="parakeet_${QUANT_NAME}" |
| 121 | + ;; |
| 122 | +SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4) |
| 123 | + RUNNER="cmake-out/examples/models/qwen3_5_moe/qwen3_5_moe_runner" |
| 124 | + TOKENIZER="${MODEL_DIR}/tokenizer.json" |
| 125 | + # A checked-in long prompt (>1000 tokens). A static, meaningful prompt |
| 126 | + # avoids the degenerate / repetitive outputs that can result from |
| 127 | + # synthetic prompts built by repeating the same sentence. |
| 128 | + PROMPT_FILE=".ci/scripts/cuda_perf_prompts/qwen3_5_moe_long_prompt.txt" |
| 129 | + RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path $TOKENIZER --prompt_file $PROMPT_FILE --max_new_tokens 512 --temperature 0" |
| 130 | + MODEL_NAME="qwen3_5_moe_${QUANT_NAME}" |
| 131 | + ;; |
| 132 | +*) |
| 133 | + echo "BENCHMARK_SKIP: no runner mapping for '${HF_MODEL}'" |
| 134 | + exit 0 |
| 135 | + ;; |
| 136 | +esac |
| 137 | + |
| 138 | +echo "::group::Running benchmark for ${HF_MODEL} (${QUANT_NAME}) with ${NUM_RUNS} runs" |
| 139 | + |
| 140 | +GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1) |
| 141 | +echo "Detected GPU: $GPU_NAME" |
| 142 | +CUDA_DRIVER_VERSION=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -1) |
| 143 | +echo "CUDA Driver Version: $CUDA_DRIVER_VERSION" |
| 144 | + |
| 145 | +mkdir -p "${RESULTS_DIR}" |
| 146 | + |
| 147 | +# Run benchmark using cuda_benchmark.py |
| 148 | +python .ci/scripts/cuda_benchmark.py \ |
| 149 | + --runner_command "$RUNNER_CMD" \ |
| 150 | + --model_name "$MODEL_NAME" \ |
| 151 | + --num_runs "${NUM_RUNS}" \ |
| 152 | + --output_json "${RESULTS_DIR}/benchmark_results.json" \ |
| 153 | + --output_v3 "${RESULTS_DIR}/benchmark_results_v3.json" \ |
| 154 | + --model "${HF_MODEL}" \ |
| 155 | + --quantization "${QUANT_NAME}" \ |
| 156 | + --git_sha "${GIT_SHA}" \ |
| 157 | + --workflow_run_id "${WORKFLOW_RUN_ID}" \ |
| 158 | + --workflow_run_url "${WORKFLOW_RUN_URL}" \ |
| 159 | + --gpu_name "$GPU_NAME" \ |
| 160 | + --cuda_driver_version "$CUDA_DRIVER_VERSION" |
| 161 | + |
| 162 | +# Save additional metadata (written via a serializer, never string-built). |
| 163 | +BENCH_MODEL="${HF_MODEL}" BENCH_QUANT="${QUANT_NAME}" BENCH_NUM_RUNS="${NUM_RUNS}" \ |
| 164 | + BENCH_RUNNER="${RUNNER}" BENCH_SHA="${GIT_SHA}" BENCH_RUN_ID="${WORKFLOW_RUN_ID}" \ |
| 165 | + BENCH_RUN_URL="${WORKFLOW_RUN_URL}" BENCH_GPU="${GPU_NAME}" \ |
| 166 | + RESULTS_DIR="${RESULTS_DIR}" python3 - <<'PY' |
| 167 | +import datetime |
| 168 | +import json |
| 169 | +import os |
| 170 | +
|
| 171 | +metadata = { |
| 172 | + "model": os.environ["BENCH_MODEL"], |
| 173 | + "quantization": os.environ["BENCH_QUANT"], |
| 174 | + "num_runs": int(os.environ["BENCH_NUM_RUNS"]), |
| 175 | + "runner": os.environ["BENCH_RUNNER"], |
| 176 | + "timestamp": datetime.datetime.now(datetime.timezone.utc).strftime( |
| 177 | + "%Y-%m-%dT%H:%M:%SZ" |
| 178 | + ), |
| 179 | + "git_sha": os.environ["BENCH_SHA"], |
| 180 | + "workflow_run_id": os.environ["BENCH_RUN_ID"], |
| 181 | + "workflow_run_url": os.environ["BENCH_RUN_URL"], |
| 182 | + "gpu_name": os.environ["BENCH_GPU"], |
| 183 | +} |
| 184 | +with open(os.path.join(os.environ.get("RESULTS_DIR", "."), "metadata.json"), "w") as f: |
| 185 | + json.dump(metadata, f, indent=2) |
| 186 | +PY |
| 187 | +echo "::endgroup::" |
0 commit comments