Created
October 23, 2026 14:18
-
-
Save rafaself/4fa16e5ab83eb5f1ff1ded7904d616de to your computer and use it in GitHub Desktop.
A Simple vLLM Chat with a minimal CLI chatbot demonstrating LLM inference, conversation history, system prompts, and output validation.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from vllm import LLM, SamplingParams | |
| # Model configuration | |
| MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct" | |
| llm = LLM(model=MODEL_NAME) | |
| # Inference configuration | |
| llm_config = { | |
| "temperature": 0.2, | |
| "max_tokens": 16, | |
| "top_p": 0.5, | |
| } | |
| sampling_params = SamplingParams(**llm_config) | |
| # Allowed responses | |
| ALLOWED_RESPONSES = {"yes", "no", "maybe"} | |
| # Initialize conversation history | |
| conversation = [ | |
| { | |
| "role": "system", | |
| "content": ( | |
| "You're a boring person. " | |
| "You must answer using only one word: yes, no, or maybe." | |
| ), | |
| } | |
| ] | |
| print("Chat started! Type 'exit' or 'quit' to leave.\n") | |
| # Chat loop | |
| while True: | |
| try: | |
| user_message = input("User: ").strip() | |
| except (KeyboardInterrupt, EOFError): | |
| print("\nGoodbye!") | |
| break | |
| # Exit condition | |
| if user_message.lower() in {"exit", "quit"}: | |
| print("Goodbye!") | |
| break | |
| # Ignore empty messages | |
| if not user_message: | |
| continue | |
| # Store user message | |
| conversation.append({ | |
| "role": "user", | |
| "content": user_message, | |
| }) | |
| # LLM inference | |
| outputs = llm.chat( | |
| conversation, | |
| sampling_params, | |
| use_tqdm=False, | |
| ) | |
| bot_message = outputs[0].outputs[0].text | |
| # Normalize generated response | |
| bot_message = bot_message.strip().lower().rstrip("!?.") | |
| # Validate response | |
| if bot_message not in ALLOWED_RESPONSES: | |
| print("Assistant: I couldn't answer this time. Try again!\n") | |
| # Roll back the unsuccessful turn | |
| conversation.pop() | |
| continue | |
| # Store assistant response | |
| conversation.append({ | |
| "role": "assistant", | |
| "content": bot_message, | |
| }) | |
| print(f"Assistant: {bot_message}\n") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment