Skip to content

Commit 0a57ff7

Browse files
committed
feat(vllm): switch from Gemma to Qwen3-Coder for better tool calling support
- Replace Gemma 3 12B with Qwen3-Coder-30B-A3B (AWQ 4-bit) - Change quantization from bitsandbytes to awq (more efficient) - Update tool parser from hermes to qwen3_coder (native support) - Increase context window from 16k to 32k tokens - Remove --trust-remote-code flag (not needed for Qwen) - VRAM usage: ~17GB (fits comfortably in 24GB GPU) Based on anomalyco/opencode#1122 Qwen models have better tool calling compatibility with OpenCode 🤖 Generated with Claude Code
1 parent 417a29b commit 0a57ff7

1 file changed

Lines changed: 8 additions & 9 deletions

File tree

‎overlays/prod/vllm/values.yaml‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,12 @@ vllm-stack:
1212
type: Recreate
1313

1414
modelSpec:
15-
- name: "gemma-3-12b"
15+
- name: "qwen3-coder-30b"
1616
# vLLM OpenAI-compatible server image
1717
repository: "vllm/vllm-openai"
1818
tag: "v0.11.0" # Latest stable release (Oct 2025)
19-
# Hugging Face model ID - BitsAndBytes 4-bit quantized (fits in 24GB GPU)
20-
modelURL: "unsloth/gemma-3-12b-it-unsloth-bnb-4bit"
19+
# Hugging Face model ID - AWQ 4-bit quantized (fits in 24GB GPU)
20+
modelURL: "cpatonn/Qwen3-Coder-30B-A3B-Instruct-AWQ-4bit"
2121
replicaCount: 1
2222

2323
# HuggingFace token from 1Password secret
@@ -49,24 +49,23 @@ vllm-stack:
4949

5050
# vLLM engine configuration
5151
vllmConfig:
52-
# Context window (can be larger with quantized model)
53-
maxModelLen: 16384
52+
# Context window (Qwen3 supports 262k natively, using 32k for stability)
53+
maxModelLen: 32768
5454

55-
# BitsAndBytes works with bfloat16 (required by Gemma 3)
55+
# AWQ works with bfloat16
5656
dtype: "bfloat16"
5757

5858
# Additional performance flags
5959
extraArgs:
6060
- "--quantization"
61-
- "bitsandbytes" # Enable BitsAndBytes quantization
61+
- "awq" # AWQ 4-bit quantization
6262
- "--enable-chunked-prefill" # Performance optimization
6363
- "--enable-prefix-caching" # Cache common prefixes
6464
- "--gpu-memory-utilization"
6565
- "0.90" # Use 90% of available VRAM
6666
- "--enable-auto-tool-choice" # Enable automatic tool choice
6767
- "--tool-call-parser"
68-
- "hermes" # Tool call parser for Gemma
69-
- "--trust-remote-code" # Trust remote code for chat template
68+
- "qwen3_coder" # Tool call parser for Qwen3-Coder
7069

7170
# Node affinity: schedule only on node-4 (GPU node with storage)
7271
nodeSelector:

0 commit comments

Comments
 (0)