Skip to content

Instantly share code, notes, and snippets.

@rafaself
Created October 23, 2026 14:18
Show Gist options
  • Select an option

  • Save rafaself/4fa16e5ab83eb5f1ff1ded7904d616de to your computer and use it in GitHub Desktop.

Select an option

Save rafaself/4fa16e5ab83eb5f1ff1ded7904d616de to your computer and use it in GitHub Desktop.
A Simple vLLM Chat with a minimal CLI chatbot demonstrating LLM inference, conversation history, system prompts, and output validation.
from vllm import LLM, SamplingParams
# Model configuration
MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
llm = LLM(model=MODEL_NAME)
# Inference configuration
llm_config = {
"temperature": 0.2,
"max_tokens": 16,
"top_p": 0.5,
}
sampling_params = SamplingParams(**llm_config)
# Allowed responses
ALLOWED_RESPONSES = {"yes", "no", "maybe"}
# Initialize conversation history
conversation = [
{
"role": "system",
"content": (
"You're a boring person. "
"You must answer using only one word: yes, no, or maybe."
),
}
]
print("Chat started! Type 'exit' or 'quit' to leave.\n")
# Chat loop
while True:
try:
user_message = input("User: ").strip()
except (KeyboardInterrupt, EOFError):
print("\nGoodbye!")
break
# Exit condition
if user_message.lower() in {"exit", "quit"}:
print("Goodbye!")
break
# Ignore empty messages
if not user_message:
continue
# Store user message
conversation.append({
"role": "user",
"content": user_message,
})
# LLM inference
outputs = llm.chat(
conversation,
sampling_params,
use_tqdm=False,
)
bot_message = outputs[0].outputs[0].text
# Normalize generated response
bot_message = bot_message.strip().lower().rstrip("!?.")
# Validate response
if bot_message not in ALLOWED_RESPONSES:
print("Assistant: I couldn't answer this time. Try again!\n")
# Roll back the unsuccessful turn
conversation.pop()
continue
# Store assistant response
conversation.append({
"role": "assistant",
"content": bot_message,
})
print(f"Assistant: {bot_message}\n")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment