Skip to content

Commit d635299

Browse files
authored
Merge branch 'main' into perf/cache-coreml-quantizer-patterns
2 parents b172a36 + d036d50 commit d635299

849 files changed

Lines changed: 90250 additions & 15207 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2bb2a9c522e7f944bf98c02c9b85fd959fa0bf0b
1+
e91899ca45f99811049ac7d539820f92a9acddde
Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
# Run the CUDA perf benchmark for one (model, quant) pair, reusing the
9+
# .pte/.ptd artifacts and the already-built runner left behind by
10+
# test_model_e2e.sh in the same CI job.
11+
#
12+
# Only the (model, quant) pairs in the perf allowlist below are benchmarked.
13+
# Anything else exits 0 after printing BENCHMARK_SKIP, so the caller can tell
14+
# "not covered by perf" apart from a real failure. The optional
15+
# models/quantizations filters (from workflow_dispatch) narrow the benchmark
16+
# stage further without touching the accuracy stage.
17+
#
18+
# Usage:
19+
# cuda_benchmark_from_e2e.sh <hf_model> <quant> <model_dir> <results_dir>
20+
# [num_runs] [git_sha] [run_id] [run_url] [models_filter] [quants_filter]
21+
#
22+
# Outputs (in <results_dir>):
23+
# benchmark_results.json, benchmark_results_v3.json, metadata.json
24+
25+
set -euo pipefail
26+
27+
HF_MODEL="${1:?hf_model required, e.g. openai/whisper-small}"
28+
QUANT_NAME="${2:?quant required, e.g. non-quantized}"
29+
MODEL_DIR="${3:?model_dir required}"
30+
RESULTS_DIR="${4:?results_dir required}"
31+
NUM_RUNS="${5:-50}"
32+
GIT_SHA="${6:-unknown}"
33+
WORKFLOW_RUN_ID="${7:-0}"
34+
WORKFLOW_RUN_URL="${8:-}"
35+
MODELS_FILTER="${9:-}"
36+
QUANTS_FILTER="${10:-}"
37+
38+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
39+
EXECUTORCH_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
40+
41+
# (model, quant) pairs covered by perf tracking. Keep in sync with the
42+
# test-model-cuda-e2e matrix in .github/workflows/cuda.yml.
43+
is_bench_pair() {
44+
case "${HF_MODEL} ${QUANT_NAME}" in
45+
"mistralai/Voxtral-Mini-3B-2507 "* | \
46+
"openai/whisper-small non-quantized" | \
47+
"openai/whisper-medium non-quantized" | \
48+
"openai/whisper-large-v3-turbo quantized-"* | \
49+
"google/gemma-3-4b-it non-quantized" | \
50+
"google/gemma-3-4b-it quantized-int4-tile-packed" | \
51+
"nvidia/parakeet-tdt non-quantized" | \
52+
"nvidia/parakeet-tdt quantized-int4-tile-packed" | \
53+
"SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4 quantized-int4-tile-packed")
54+
return 0
55+
;;
56+
*)
57+
return 1
58+
;;
59+
esac
60+
}
61+
62+
if ! is_bench_pair; then
63+
echo "BENCHMARK_SKIP: ${HF_MODEL} ${QUANT_NAME} is not in the perf allowlist"
64+
exit 0
65+
fi
66+
67+
# workflow_dispatch filters: comma-separated HF model IDs / quant names.
68+
# Empty means "benchmark every allowlisted pair".
69+
case ",${MODELS_FILTER}," in
70+
*",${HF_MODEL},"*) ;;
71+
*)
72+
if [ -n "${MODELS_FILTER}" ]; then
73+
echo "BENCHMARK_SKIP: ${HF_MODEL} not in dispatch models filter"
74+
exit 0
75+
fi
76+
;;
77+
esac
78+
case ",${QUANTS_FILTER}," in
79+
*",${QUANT_NAME},"*) ;;
80+
*)
81+
if [ -n "${QUANTS_FILTER}" ]; then
82+
echo "BENCHMARK_SKIP: ${QUANT_NAME} not in dispatch quantizations filter"
83+
exit 0
84+
fi
85+
;;
86+
esac
87+
88+
cd "${EXECUTORCH_ROOT}"
89+
export LD_LIBRARY_PATH="/opt/conda/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
90+
91+
# Runner command mapping, ported from the retired cuda-perf.yml so the
92+
# benchmark prompts and flags (and therefore the numbers) stay identical.
93+
case "${HF_MODEL}" in
94+
mistralai/Voxtral-Mini-3B-2507)
95+
RUNNER="cmake-out/examples/models/voxtral/voxtral_runner"
96+
PREPROCESSOR="${MODEL_DIR}/voxtral_preprocessor.pte"
97+
TOKENIZER="${MODEL_DIR}/tekken.json"
98+
AUDIO="${MODEL_DIR}/poem.wav"
99+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path $TOKENIZER --audio_path $AUDIO --processor_path $PREPROCESSOR --temperature 0"
100+
MODEL_NAME="voxtral_${QUANT_NAME}"
101+
;;
102+
openai/whisper-*)
103+
RUNNER="cmake-out/examples/models/whisper/whisper_runner"
104+
PREPROCESSOR="${MODEL_DIR}/whisper_preprocessor.pte"
105+
AUDIO="${MODEL_DIR}/output.wav"
106+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path ${MODEL_DIR}/ --audio_path $AUDIO --processor_path $PREPROCESSOR --temperature 0"
107+
MODEL_NAME="${HF_MODEL#openai/}_${QUANT_NAME}"
108+
;;
109+
google/gemma-3-4b-it)
110+
RUNNER="cmake-out/examples/models/gemma3/gemma3_e2e_runner"
111+
IMAGE="docs/source/_static/img/et-logo.png"
112+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path ${MODEL_DIR}/ --image_path $IMAGE --temperature 0"
113+
MODEL_NAME="gemma3_${QUANT_NAME}"
114+
;;
115+
nvidia/parakeet-tdt)
116+
RUNNER="cmake-out/examples/models/parakeet/parakeet_runner"
117+
AUDIO="${MODEL_DIR}/test_audio.wav"
118+
TOKENIZER="${MODEL_DIR}/tokenizer.model"
119+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --audio_path $AUDIO --tokenizer_path $TOKENIZER"
120+
MODEL_NAME="parakeet_${QUANT_NAME}"
121+
;;
122+
SocialLocalMobile/Qwen3.5-35B-A3B-HQQ-INT4)
123+
RUNNER="cmake-out/examples/models/qwen3_5_moe/qwen3_5_moe_runner"
124+
TOKENIZER="${MODEL_DIR}/tokenizer.json"
125+
# A checked-in long prompt (>1000 tokens). A static, meaningful prompt
126+
# avoids the degenerate / repetitive outputs that can result from
127+
# synthetic prompts built by repeating the same sentence.
128+
PROMPT_FILE=".ci/scripts/cuda_perf_prompts/qwen3_5_moe_long_prompt.txt"
129+
RUNNER_CMD="$RUNNER --model_path ${MODEL_DIR}/model.pte --data_path ${MODEL_DIR}/aoti_cuda_blob.ptd --tokenizer_path $TOKENIZER --prompt_file $PROMPT_FILE --max_new_tokens 512 --temperature 0"
130+
MODEL_NAME="qwen3_5_moe_${QUANT_NAME}"
131+
;;
132+
*)
133+
echo "BENCHMARK_SKIP: no runner mapping for '${HF_MODEL}'"
134+
exit 0
135+
;;
136+
esac
137+
138+
echo "::group::Running benchmark for ${HF_MODEL} (${QUANT_NAME}) with ${NUM_RUNS} runs"
139+
140+
GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)
141+
echo "Detected GPU: $GPU_NAME"
142+
CUDA_DRIVER_VERSION=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -1)
143+
echo "CUDA Driver Version: $CUDA_DRIVER_VERSION"
144+
145+
mkdir -p "${RESULTS_DIR}"
146+
147+
# Run benchmark using cuda_benchmark.py
148+
python .ci/scripts/cuda_benchmark.py \
149+
--runner_command "$RUNNER_CMD" \
150+
--model_name "$MODEL_NAME" \
151+
--num_runs "${NUM_RUNS}" \
152+
--output_json "${RESULTS_DIR}/benchmark_results.json" \
153+
--output_v3 "${RESULTS_DIR}/benchmark_results_v3.json" \
154+
--model "${HF_MODEL}" \
155+
--quantization "${QUANT_NAME}" \
156+
--git_sha "${GIT_SHA}" \
157+
--workflow_run_id "${WORKFLOW_RUN_ID}" \
158+
--workflow_run_url "${WORKFLOW_RUN_URL}" \
159+
--gpu_name "$GPU_NAME" \
160+
--cuda_driver_version "$CUDA_DRIVER_VERSION"
161+
162+
# Save additional metadata (written via a serializer, never string-built).
163+
BENCH_MODEL="${HF_MODEL}" BENCH_QUANT="${QUANT_NAME}" BENCH_NUM_RUNS="${NUM_RUNS}" \
164+
BENCH_RUNNER="${RUNNER}" BENCH_SHA="${GIT_SHA}" BENCH_RUN_ID="${WORKFLOW_RUN_ID}" \
165+
BENCH_RUN_URL="${WORKFLOW_RUN_URL}" BENCH_GPU="${GPU_NAME}" \
166+
RESULTS_DIR="${RESULTS_DIR}" python3 - <<'PY'
167+
import datetime
168+
import json
169+
import os
170+
171+
metadata = {
172+
"model": os.environ["BENCH_MODEL"],
173+
"quantization": os.environ["BENCH_QUANT"],
174+
"num_runs": int(os.environ["BENCH_NUM_RUNS"]),
175+
"runner": os.environ["BENCH_RUNNER"],
176+
"timestamp": datetime.datetime.now(datetime.timezone.utc).strftime(
177+
"%Y-%m-%dT%H:%M:%SZ"
178+
),
179+
"git_sha": os.environ["BENCH_SHA"],
180+
"workflow_run_id": os.environ["BENCH_RUN_ID"],
181+
"workflow_run_url": os.environ["BENCH_RUN_URL"],
182+
"gpu_name": os.environ["BENCH_GPU"],
183+
}
184+
with open(os.path.join(os.environ.get("RESULTS_DIR", "."), "metadata.json"), "w") as f:
185+
json.dump(metadata, f, indent=2)
186+
PY
187+
echo "::endgroup::"

‎.ci/scripts/gather_test_models.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,6 @@ def model_should_run_on_target_os(model: str, target_os: str) -> bool:
113113
A helper function to decide whether a model should be tested on a target os (linux/macos).
114114
For example, a big model can be disabled in macos due to the limited macos resources.
115115
"""
116-
# yolo26 was contributed as an OpenVINO/XNNPACK example and has never passed on
117-
# the portable backend, so gathering it only ever produces a permanently red job.
118-
# TODO(#21621): drop this skip once yolo26 exports cleanly on portable.
119-
if model == "yolo26":
120-
return False
121116
if target_os == "macos":
122117
# Disabled in macos due to limited resources, and should stay that way even if
123118
# we otherwise re-enable.
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# Installs the NVIDIA display driver on a Windows GPU runner. The runner images carry the
8+
# GPU but no driver, so without this nvcuda.dll is missing and every CUDA call fails with
9+
# cudaErrorInsufficientDriver. The same driver, from the same place, that PyTorch's Windows
10+
# CUDA smoke tests install first (.ci/pytorch/windows/internal/driver_update.bat in
11+
# pytorch/pytorch), so both projects test against one driver.
12+
$ErrorActionPreference = "Stop"
13+
$version = "580.88"
14+
$installer = Join-Path $env:RUNNER_TEMP "$version-data-center-tesla-desktop-win10-win11-64bit-dch-international.exe"
15+
$url = "https://ossci-windows.s3.amazonaws.com/$(Split-Path $installer -Leaf)"
16+
17+
$nvcuda = Join-Path $env:SystemRoot "System32\nvcuda.dll"
18+
if (Test-Path $nvcuda) {
19+
Write-Host "NVIDIA driver already present: $((Get-Item $nvcuda).VersionInfo.FileVersion)"
20+
exit 0
21+
}
22+
curl.exe --retry 3 -fsSL $url --output $installer
23+
if ($LASTEXITCODE -ne 0) { throw "downloading the NVIDIA driver from $url failed" }
24+
$process = Start-Process -FilePath $installer -ArgumentList "-s", "-noreboot" -Wait -PassThru
25+
Remove-Item $installer -ErrorAction SilentlyContinue
26+
if ($process.ExitCode -ne 0) { throw "the NVIDIA driver installer exited with $($process.ExitCode)" }
27+
if (-not (Test-Path $nvcuda)) { throw "the NVIDIA driver installed but $nvcuda is missing" }
28+
Write-Host "NVIDIA driver $version installed: $((Get-Item $nvcuda).VersionInfo.FileVersion)"

‎.ci/scripts/setup-vulkan-linux-deps.sh‎

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -67,35 +67,37 @@ install_vulkan_loader() {
6767
# libvulkan.so.1 (the Khronos loader that volk dlopen()s at runtime) is not part
6868
# of the NVIDIA driver and is absent from the CUDA builder image; vulkan-tools
6969
# provides vulkaninfo for the device sanity check. Both ship as native el8 RPMs.
70+
# The NVIDIA ICD also needs EGL and X11 client libraries on headless runners.
7071
if command -v dnf >/dev/null 2>&1; then
71-
_maybe_sudo dnf install -y vulkan-loader vulkan-tools
72+
_maybe_sudo dnf install -y vulkan-loader vulkan-tools libglvnd-egl libX11 libXext
7273
fi
7374
}
7475

7576
_find_nvidia_vulkan_library() {
76-
# NVIDIA implements its Vulkan ICD inside libGLX_nvidia.so.0. The NVIDIA
77-
# container runtime mounts this library into the container (it is pulled from
78-
# the driver's ldcache when NVIDIA_DRIVER_CAPABILITIES includes graphics/all),
79-
# so prefer ldconfig and fall back to the usual mount locations.
80-
local lib cand
81-
lib="$(ldconfig -p 2>/dev/null | awk '/libGLX_nvidia\.so\.0/ {print $NF; exit}')"
82-
if [ -z "${lib}" ]; then
83-
for cand in /usr/lib64/libGLX_nvidia.so.0 \
84-
/usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.0 \
85-
/usr/lib/libGLX_nvidia.so.0; do
77+
# NVIDIA provides EGL and GLX Vulkan ICDs. Prefer EGL on headless runners;
78+
# the GLX entry point can fail to initialize without an X server.
79+
local soname lib cand
80+
for soname in libEGL_nvidia.so.0 libGLX_nvidia.so.0; do
81+
lib="$(ldconfig -p 2>/dev/null | awk -v name="${soname}" '$1 == name {print $NF; exit}')"
82+
if [ -n "${lib}" ]; then
83+
printf '%s' "${lib}"
84+
return
85+
fi
86+
for cand in "/usr/lib64/${soname}" \
87+
"/usr/lib/x86_64-linux-gnu/${soname}" \
88+
"/usr/lib/${soname}"; do
8689
if [ -e "${cand}" ]; then
87-
lib="${cand}"
88-
break
90+
printf '%s' "${cand}"
91+
return
8992
fi
9093
done
91-
fi
92-
printf '%s' "${lib}"
94+
done
9395
}
9496

9597
_vulkan_has_real_device() {
9698
# True if the loader enumerates a hardware GPU. vulkaninfo can exit non-zero
9799
# for unrelated reasons (no display/WSI), so key off the reported deviceType.
98-
command -v vulkaninfo >/dev/null 2>&1 || return 0
100+
command -v vulkaninfo >/dev/null 2>&1 || return 1
99101
vulkaninfo --summary 2>/dev/null |
100102
grep -qE 'PHYSICAL_DEVICE_TYPE_(DISCRETE|INTEGRATED|VIRTUAL)_GPU'
101103
}
@@ -131,25 +133,24 @@ JSON
131133
echo "Real NVIDIA GPU selected; pinned Vulkan ICD to ${nvidia_lib}"
132134
return
133135
fi
134-
echo "WARNING: ${nvidia_lib} present but no GPU enumerated; using SwiftShader."
135-
# Surface why the NVIDIA driver did not enumerate (e.g. a missing dependency
136-
# of libGLX_nvidia, or no render node) so the fallback is diagnosable in CI.
136+
echo "ERROR: ${nvidia_lib} present but no GPU enumerated."
137+
# Surface missing ICD dependencies and device-enumeration errors.
138+
ldd "${nvidia_lib}" || true
137139
if command -v vulkaninfo >/dev/null 2>&1; then
138140
echo "--- NVIDIA Vulkan ICD diagnostic ---"
139141
VK_LOADER_DEBUG=warn vulkaninfo --summary 2>&1 | head -40 || true
140142
echo "--- end diagnostic ---"
141143
fi
142-
unset VK_ICD_FILENAMES
143144
else
144-
echo "WARNING: no NVIDIA Vulkan driver library found; using SwiftShader."
145+
echo "ERROR: no NVIDIA Vulkan driver library found."
145146
fi
146-
install_swiftshader
147+
return 1
147148
}
148149

149150
VULKAN_SDK_VERSION="1.4.321.1"
150151

151152
# The no-argument default installs SwiftShader so the existing CPU-runner CI is
152-
# unchanged. Pass "real-gpu" to prefer a real system ICD when one is present.
153+
# unchanged. Pass "real-gpu" to require a real system ICD.
153154
case "${1:-swiftshader}" in
154155
real-gpu)
155156
# Do not download the LunarG SDK here: its prebuilt glslc cannot run on the

‎.ci/scripts/test-rocm-aoti.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ ROCM_EXCLUDED_TESTS=(
147147
--ignore=backends/cuda/tests/test_triton_sdpa_splitk.py
148148
# This test process core-dumps on ROCm instead of reporting a failure.
149149
--ignore=backends/cuda/tests/test_triton_sdpa_nan.py
150+
# CUDA-graph autotune timing is NVIDIA-only; the CUDA backend turns it off
151+
# on ROCm.
152+
--ignore=backends/cuda/tests/test_cuda_graph_timing.py
150153
)
151154
python -m pytest -v -o 'addopts=' \
152155
"${ROCM_EXCLUDED_TESTS[@]}" \

‎.ci/scripts/test_backend.sh‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,8 @@ if [[ "$FLOW" == *qnn* ]]; then
5454
fi
5555

5656
if [[ "$FLOW" == *vulkan* ]]; then
57-
# Setup the Vulkan SDK and select an ICD: use the real system GPU ICD when one
58-
# is present (real-GPU runner), otherwise fall back to SwiftShader (CPU
59-
# runner). The Vulkan loader searches both standard ICD directories.
60-
if ls /etc/vulkan/icd.d/*.json /usr/share/vulkan/icd.d/*.json \
61-
>/dev/null 2>&1; then
62-
source .ci/scripts/setup-vulkan-linux-deps.sh "real-gpu"
63-
else
64-
source .ci/scripts/setup-vulkan-linux-deps.sh "swiftshader"
65-
fi
57+
# CPU runners can have Mesa ICDs installed without a usable hardware GPU.
58+
source .ci/scripts/setup-vulkan-linux-deps.sh "swiftshader"
6659

6760
EXTRA_BUILD_ARGS+=" -DEXECUTORCH_BUILD_VULKAN=ON"
6861
fi
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# Builds and runs the CUDA backend's runtime C++ tests on a Windows GPU runner, the
8+
# counterpart of the unittest-cuda-runtime job in cuda.yml. The Windows runners carry
9+
# data-center GPUs in TCC mode, which have no memory pools, so this is also what covers the
10+
# allocator's synchronous fallback; the Linux job covers the pool path.
11+
$ErrorActionPreference = "Stop"
12+
13+
# Windows PowerShell 5.1 does not stop on a failing native command.
14+
function Invoke-Native {
15+
param([Parameter(Mandatory = $true)][scriptblock]$Command)
16+
& $Command
17+
if ($LASTEXITCODE -ne 0) {
18+
throw "exit code ${LASTEXITCODE}: $Command"
19+
}
20+
}
21+
22+
& "$PSScriptRoot\install_nvidia_driver_windows.ps1"
23+
24+
$cudaNvcc = Join-Path $env:CUDA_HOME "bin\nvcc.exe"
25+
if (-not (Test-Path $cudaNvcc)) { throw "CUDA compiler not found at '$cudaNvcc'" }
26+
$env:CUDACXX = $cudaNvcc
27+
$env:PATH = "$env:CUDA_HOME\bin\x64;$env:CUDA_HOME\bin;$env:PATH"
28+
# The runner's GPU has no memory pools, which is what makes this job cover the
29+
# allocator's fallback; fail rather than skip those tests if it ever has them.
30+
$env:EXECUTORCH_CUDA_TEST_REQUIRE_NO_MEMORY_POOLS = "1"
31+
32+
$tests = @(
33+
"test_cuda_allocator",
34+
"test_cuda_mutable_state",
35+
"test_cuda_weight_cache",
36+
"test_cuda_sort_rand",
37+
"test_cuda_guard",
38+
"test_cuda_stream_guard"
39+
)
40+
$numCores = [Math]::Max([Environment]::ProcessorCount - 1, 1)
41+
Invoke-Native {
42+
cmake --preset llm-release-cuda -DEXECUTORCH_BUILD_TESTS=ON -DCMAKE_CXX_STANDARD=20 `
43+
-T "cuda=$env:CUDA_HOME" "-DCMAKE_CUDA_COMPILER=$cudaNvcc" "-DCUDAToolkit_ROOT=$env:CUDA_HOME"
44+
}
45+
Invoke-Native { cmake --build cmake-out --config Release -j $numCores --target $tests }
46+
foreach ($test in $tests) {
47+
Invoke-Native { ctest --test-dir cmake-out -C Release -R "^$test`$" --output-on-failure -V }
48+
}

0 commit comments

Comments
 (0)