version: 2.1

orbs:

codecov: codecov/[email protected]

node: circleci/[email protected] # Add this line to declare the node orb

win: circleci/[email protected] # Add Windows orb

commands:

setup_google_dns:

steps:

- run:

name: "Configure Google DNS"

command: |

# Backup original resolv.conf

sudo cp /etc/resolv.conf /etc/resolv.conf.backup

# Add both local and Google DNS servers

echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf

echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf

echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf

wait_for_service:

description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)"

parameters:

url:

type: string

timeout:

type: string

default: "60"

steps:

- run:

name: "Wait for << parameters.url >>"

command: |

TIMEOUT=<< parameters.timeout >>

URL="<< parameters.url >>"

ELAPSED=0

echo "Waiting up to ${TIMEOUT}s for ${URL} ..."

if echo "$URL" | grep -q '^tcp://'; then

HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1)

PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2)

while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do

sleep 2; ELAPSED=$((ELAPSED+2))

if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi

done

else

while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do

sleep 2; ELAPSED=$((ELAPSED+2))

if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi

done

fi

echo "Service ready after ${ELAPSED}s"

install_helm:

steps:

- run:

name: Install Helm v3.17.3

command: |

curl -sSLf -o /tmp/helm.tar.gz \

https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz

echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c -

sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm

rm -f /tmp/helm.tar.gz

install_kind:

steps:

- run:

name: Install Kind v0.20.0

command: |

curl -sSLf -o /tmp/kind \

https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64

echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c -

chmod +x /tmp/kind

sudo mv /tmp/kind /usr/local/bin/kind

install_uv:

description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH."

steps:

- run:

name: Install uv (pinned 0.10.9)

command: |

curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh

echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -

env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh

rm -f /tmp/uv-install.sh

echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"

export PATH="$HOME/.local/bin:$PATH"

start_postgres:

description: "Start a postgres-db container on port 5432 and wait until it accepts connections."

parameters:

db_name:

type: string

default: circle_test

steps:

- run:

name: Start PostgreSQL

command: |

docker run -d \

--name postgres-db \

-e POSTGRES_USER=postgres \

-e POSTGRES_PASSWORD=postgres \

-e POSTGRES_DB=<< parameters.db_name >> \

-p 5432:5432 \

postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26

- wait_for_service:

url: tcp://localhost:5432

timeout: "60"

start_redis:

description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys."

steps:

- run:

name: Start Redis

command: |

docker run -d \

--name redis-cache \

-p 6379:6379 \

redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6

- wait_for_service:

url: tcp://localhost:6379

timeout: "60"

setup_litellm_enterprise_pip:

steps:

- run:

name: "Install local version of litellm-enterprise"

command: |

# litellm-enterprise is a uv workspace member and is already installed

# by the main `uv sync --all-groups --all-extras`. Do NOT run

# `uv sync --package litellm-enterprise` here — that overwrites the

# shared .venv and strips out dev/test deps (pytest, prisma, etc.).

uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)"

setup_litellm_test_deps:

steps:

- checkout

- setup_google_dns

- install_uv

- restore_cache:

keys:

- v1-uv-cache-{{ checksum "uv.lock" }}

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- setup_litellm_enterprise_pip

- save_cache:

paths:

- ~/.cache/uv

key: v1-uv-cache-{{ checksum "uv.lock" }}

jobs:

# Add Windows testing job

using_litellm_on_windows:

executor:

name: win/default

shell: powershell.exe

working_directory: ~/project

steps:

- checkout

- run:

name: Install Python

command: |

choco install python --version=3.11.0 -y --no-progress --force

refreshenv

python --version

environment:

CHOCOLATEY_CONFIRM_ALL: "true"

- run:

name: Install Dependencies

command: |

$installer = Join-Path $env:TEMP "uv-install.ps1"

Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer

$expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d"

$actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower()

if ($actual -ne $expected) {

throw "uv installer hash mismatch: expected $expected got $actual"

}

& $installer

Remove-Item $installer

$uvBin = Join-Path $HOME ".local\bin"

$env:Path = "$uvBin;$env:Path"

if (!(Test-Path $PROFILE)) {

New-Item -ItemType File -Force -Path $PROFILE | Out-Null

}

if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) {

Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`""

}

uv sync --frozen --group dev --python (Get-Command python).Source

- run:

name: Run Windows-specific test

command: |

uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v

local_testing_part1:

docker:

- &python312_image

image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c

auth:

username: ${DOCKERHUB_USERNAME}

password: ${DOCKERHUB_PASSWORD}

working_directory: ~/project

parallelism: 4

steps:

- checkout

- setup_google_dns

- restore_cache:

keys:

- v1-uv-cache-{{ checksum "uv.lock" }}

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- setup_litellm_enterprise_pip

- save_cache:

paths:

- ~/.cache/uv

key: v1-uv-cache-{{ checksum "uv.lock" }}

- run:

name: Run prisma ./docker/entrypoint.sh

command: |

set +e

chmod +x docker/entrypoint.sh

./docker/entrypoint.sh

set -e

# Run pytest and generate JUnit XML report

- run:

name: Run tests (Part 1 - A-M)

command: |

mkdir test-results

# Discover test files (A-M)

TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[a-mA-M]*.py")

echo "$TEST_FILES" | circleci tests run \

--split-by=timings \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv \

--cov=litellm \

--cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=20 \

-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \

-n 4 \

--timeout=300 \

--timeout_method=thread"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml local_testing_part1_coverage.xml

mv .coverage local_testing_part1_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- local_testing_part1_coverage.xml

- local_testing_part1_coverage

local_testing_part2:

docker:

- *python312_image

working_directory: ~/project

parallelism: 4

steps:

- checkout

- setup_google_dns

- restore_cache:

keys:

- v1-uv-cache-{{ checksum "uv.lock" }}

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- setup_litellm_enterprise_pip

- save_cache:

paths:

- ~/.cache/uv

key: v1-uv-cache-{{ checksum "uv.lock" }}

- run:

name: Run prisma ./docker/entrypoint.sh

command: |

set +e

chmod +x docker/entrypoint.sh

./docker/entrypoint.sh

set -e

# Run pytest and generate JUnit XML report

- run:

name: Run tests (Part 2 - N-Z)

command: |

mkdir test-results

# Discover test files (N-Z)

TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_[n-zN-Z]*.py")

echo "$TEST_FILES" | circleci tests run \

--split-by=timings \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv \

--cov=litellm \

--cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=20 \

-k \"not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache\" \

-n 4 \

--timeout=300 \

--timeout_method=thread"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml local_testing_part2_coverage.xml

mv .coverage local_testing_part2_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- local_testing_part2_coverage.xml

- local_testing_part2_coverage

langfuse_logging_unit_tests:

docker:

- *python312_image

working_directory: ~/project

resource_class: medium

steps:

- checkout

- setup_google_dns

- restore_cache:

keys:

- v1-uv-cache-{{ checksum "uv.lock" }}

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- setup_litellm_enterprise_pip

- save_cache:

paths:

- ~/.cache/uv

key: v1-uv-cache-{{ checksum "uv.lock" }}

- run:

name: Run prisma ./docker/entrypoint.sh

command: |

set +e

chmod +x docker/entrypoint.sh

./docker/entrypoint.sh

set -e

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-v -x \

--junitxml=test-results/junit.xml \

--durations=5 \

-k \"langfuse\""

no_output_timeout: 15m

# Store test results

- store_test_results:

path: test-results

auth_ui_unit_tests:

docker:

- *python312_image

- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84

environment:

POSTGRES_USER: postgres

POSTGRES_PASSWORD: postgres

POSTGRES_DB: litellm_test

working_directory: ~/project

environment:

DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- save_cache:

paths:

- ~/.cache/uv

key: v1-uv-cache-{{ checksum "uv.lock" }}

- wait_for_service:

url: tcp://localhost:5432

timeout: "60"

- run:

name: Seed DB schema via prisma db push

command: |

set +e

uv run --no-sync litellm --skip_server_startup --use_prisma_db_push

set -e

- run:

name: Generate Prisma Client

command: uv run --no-sync python -m prisma generate

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-v -x \

--junitxml=test-results/junit.xml \

--durations=5 \

-n 2"

no_output_timeout: 15m

# Store test results

- store_test_results:

path: test-results

litellm_router_testing: # Runs all tests with the "router" keyword

docker:

- *python312_image

working_directory: ~/project

resource_class: large

parallelism: 4

steps:

- checkout

- setup_google_dns

- install_uv

- restore_cache:

keys:

- v1-uv-cache-{{ checksum "uv.lock" }}

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- save_cache:

paths:

- ~/.cache/uv

key: v1-uv-cache-{{ checksum "uv.lock" }}

# Run pytest and generate JUnit XML report

- setup_litellm_enterprise_pip

- run:

name: Run tests

command: |

TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--split-by=timings \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-v \

-k 'router' \

-n 4 \

--junitxml=test-results/junit.xml \

--durations=5 \

--timeout=300 --timeout_method=thread"

no_output_timeout: 15m

# Store test results

- store_test_results:

path: test-results

litellm_router_unit_testing: # Runs all tests with the "router" keyword

docker:

- *python312_image

working_directory: ~/project

resource_class: large

steps:

- checkout

- setup_google_dns

- install_uv

- restore_cache:

keys:

- v1-uv-cache-{{ checksum "uv.lock" }}

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- save_cache:

paths:

- ~/.cache/uv

key: v1-uv-cache-{{ checksum "uv.lock" }}

# Run pytest and generate JUnit XML report

- setup_litellm_enterprise_pip

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-v -x \

--junitxml=test-results/junit.xml \

--durations=5 \

-n 4"

no_output_timeout: 15m

# Store test results

- store_test_results:

path: test-results

litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword

docker:

- *python312_image

working_directory: ~/project

resource_class: medium

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

# Run pytest and generate JUnit XML report

- setup_litellm_enterprise_pip

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-v -x \

--junitxml=test-results/junit.xml \

--durations=5 \

-k \"assistants\""

no_output_timeout: 15m

# Store test results

- store_test_results:

path: test-results

llm_translation_testing:

docker:

- *python312_image

working_directory: ~/project

resource_class: xlarge

steps:

- checkout

- setup_google_dns

- install_uv

- restore_cache:

keys:

- v1-uv-cache-{{ checksum "uv.lock" }}

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- save_cache:

paths:

- ~/.cache/uv

key: v1-uv-cache-{{ checksum "uv.lock" }}

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

# Add --timeout to kill hanging tests after 120s (2 min)

# Add --durations=20 to show 20 slowest tests for debugging

# Subdirectories with dedicated jobs (maintain this list as new jobs are added)

mkdir -p test-results

# Glob excludes the realtime/ subdirectory since it has its own job

TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-v \

--junitxml=test-results/junit.xml \

--durations=20 \

-n 4 \

--timeout=120 --timeout_method=thread \

--retries 2 --retry-delay 5 \

--max-worker-restart=5"

no_output_timeout: 15m

# Store test results

- store_test_results:

path: test-results

realtime_translation_testing:

docker:

- *python312_image

working_directory: ~/project

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

# Run pytest and generate JUnit XML report

- run:

name: Run realtime tests

command: |

# Add --timeout to kill hanging tests after 120s (2 min)

# Add --durations=20 to show 20 slowest tests for debugging

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv \

--cov=litellm --cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=20 \

-n 4 \

--timeout=120 --timeout_method=thread"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml realtime_translation_coverage.xml

mv .coverage realtime_translation_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- realtime_translation_coverage.xml

- realtime_translation_coverage

agent_testing:

docker:

- *python312_image

working_directory: ~/project

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv -x -s \

--cov=litellm --cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=5"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml agent_coverage.xml

mv .coverage agent_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- agent_coverage.xml

- agent_coverage

guardrails_testing:

docker:

- *python312_image

working_directory: ~/project

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

export LITELLM_LOG=WARNING

TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv \

--cov=litellm --cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=5 \

-n 2 \

--timeout=120 --timeout_method=thread"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml guardrails_coverage.xml

mv .coverage guardrails_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- guardrails_coverage.xml

- guardrails_coverage

google_generate_content_endpoint_testing:

docker:

- *python312_image

working_directory: ~/project

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv -x -s \

--cov=litellm --cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=5 \

--retries 3 --retry-delay 5"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml google_generate_content_endpoint_coverage.xml

mv .coverage google_generate_content_endpoint_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- google_generate_content_endpoint_coverage.xml

- google_generate_content_endpoint_coverage

llm_responses_api_testing:

docker:

- *python312_image

working_directory: ~/project

resource_class: large

steps:

- checkout

- setup_google_dns

- install_uv

- restore_cache:

keys:

- v1-uv-cache-{{ checksum "uv.lock" }}

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- save_cache:

paths:

- ~/.cache/uv

key: v1-uv-cache-{{ checksum "uv.lock" }}

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-v -x \

--junitxml=test-results/junit.xml \

--durations=5 \

-n 8"

no_output_timeout: 15m

# Store test results

- store_test_results:

path: test-results

ocr_testing:

docker:

- *python312_image

working_directory: ~/project

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv -x \

--cov=litellm --cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=5 \

-n 4"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml ocr_coverage.xml

mv .coverage ocr_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- ocr_coverage.xml

- ocr_coverage

search_testing:

docker:

- *python312_image

working_directory: ~/project

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv -x \

--cov=litellm --cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=5 \

-n 4"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml search_coverage.xml

mv .coverage search_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- search_coverage.xml

- search_coverage

litellm_mapped_enterprise_tests:

docker:

- *python312_image

working_directory: ~/project

resource_class: large

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

- setup_litellm_enterprise_pip

- run:

name: Run enterprise tests

command: |

uv run --no-sync python -m prisma generate

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-v -x \

--junitxml=test-results/junit-enterprise.xml \

--durations=10 \

-n 4"

no_output_timeout: 15m

# Store test results

- store_test_results:

path: test-results

batches_testing:

docker:

- *python312_image

working_directory: ~/project

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv -x -s \

--cov=litellm --cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=5 \

-n 2"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml batches_coverage.xml

mv .coverage batches_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- batches_coverage.xml

- batches_coverage

litellm_utils_testing:

docker:

- *python312_image

working_directory: ~/project

steps:

- checkout

- setup_google_dns

- install_uv

- run:

name: Install Dependencies

command: |

uv sync --frozen --all-groups --all-extras --python 3.12

# Run pytest and generate JUnit XML report

- run:

name: Run tests

command: |

mkdir -p test-results

TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py")

echo "$TEST_FILES" | circleci tests run \

--verbose \

--command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \

-vv -x -s \

--cov=litellm --cov-report=xml \

--junitxml=test-results/junit.xml \

--durations=5 \

-n 2"

no_output_timeout: 15m

- run:

name: Rename the coverage files

command: |

mv coverage.xml litellm_utils_coverage.xml

mv .coverage litellm_utils_coverage

# Store test results

- store_test_results:

path: test-results

- persist_to_workspace:

root: .

paths:

- litellm_utils_coverage.xml

- litellm_utils_coverage

pass_through_unit_testing:

docker:

- *python312_image

working_directory: ~/project

steps: