[BUG] Streaming ChatAgent silently drops tool calls whose name is not a registered internal tool (hallucinated names and external_tools) #4366

import sys, camel
print(sys.version, sys.platform)
print(camel.__version__)
# 3.13.15 (main, Aug 25 2026, 13:50:12) [Clang 22.1.3] darwin
# 0.2.91a7

import os
import warnings

warnings.filterwarnings("ignore")
# model.run is fully mocked below; the key only passes the constructor check
os.environ.setdefault("OPENAI_API_KEY", "sk-dummy-not-used")

from unittest.mock import MagicMock

from openai.types.chat.chat_completion import ChatCompletion, Choice
from openai.types.chat.chat_completion_chunk import (
    ChatCompletionChunk,
    ChoiceDelta,
)
from openai.types.chat.chat_completion_chunk import (
    Choice as ChunkChoice,
    ChoiceDeltaToolCall,
    ChoiceDeltaToolCallFunction,
)
from openai.types.chat.chat_completion_message import ChatCompletionMessage
from openai.types.chat.chat_completion_message_tool_call import (
    ChatCompletionMessageToolCall,
    Function,
)

from camel.agents import ChatAgent
from camel.models import ModelFactory
from camel.toolkits import FunctionTool
from camel.types import ModelPlatformType, ModelType


class _DummyTokenCounter:
    r"""Offline stand-in so no tiktoken download happens."""

    def count_tokens_from_messages(self, messages, **kwargs):
        return 0

    def count_tokens(self, text):
        return 0

    def encode(self, text):
        return []


def make_model(stream):
    model = ModelFactory.create(
        model_platform=ModelPlatformType.OPENAI,
        model_type=ModelType.GPT_5_MINI,
        model_config_dict={"stream": stream},
    )
    model._token_counter = _DummyTokenCounter()
    return model


def tool_call_chunks(name, args_json):
    r"""Stream a single tool call: the name first, then the arguments."""
    return [
        ChatCompletionChunk(
            id="m1",
            choices=[
                ChunkChoice(
                    delta=ChoiceDelta(
                        role="assistant",
                        tool_calls=[
                            ChoiceDeltaToolCall(
                                index=0,
                                id="call_1",
                                type="function",
                                function=ChoiceDeltaToolCallFunction(
                                    name=name, arguments=""
                                ),
                            )
                        ],
                    ),
                    index=0,
                    finish_reason=None,
                )
            ],
            created=1234567890,
            model="gpt-5-mini",
            object="chat.completion.chunk",
        ),
        ChatCompletionChunk(
            id="m1",
            choices=[
                ChunkChoice(
                    delta=ChoiceDelta(
                        tool_calls=[
                            ChoiceDeltaToolCall(
                                index=0,
                                function=ChoiceDeltaToolCallFunction(
                                    arguments=args_json
                                ),
                            )
                        ]
                    ),
                    index=0,
                    finish_reason="tool_calls",
                )
            ],
            created=1234567890,
            model="gpt-5-mini",
            object="chat.completion.chunk",
        ),
    ]


def tool_call_completion(name, args_json):
    return ChatCompletion(
        id="m1",
        choices=[
            Choice(
                index=0,
                finish_reason="tool_calls",
                message=ChatCompletionMessage(
                    role="assistant",
                    content="",
                    tool_calls=[
                        ChatCompletionMessageToolCall(
                            id="call_1",
                            type="function",
                            function=Function(name=name, arguments=args_json),
                        )
                    ],
                ),
            )
        ],
        created=1234567890,
        model="gpt-5-mini",
        object="chat.completion",
    )


def plain_completion(text):
    return ChatCompletion(
        id="m2",
        choices=[
            Choice(
                index=0,
                finish_reason="stop",
                message=ChatCompletionMessage(role="assistant", content=text),
            )
        ],
        created=1234567891,
        model="gpt-5-mini",
        object="chat.completion",
    )


def memory_of(agent):
    messages = agent.memory.get_context()[0]
    return [
        (
            m["role"],
            (m.get("tool_calls") or [{}])[0].get("function", {}).get("name"),
            m.get("content"),
        )
        for m in messages
    ]


def add(a: int, b: int) -> int:
    """Add two integers."""
    return a + b


def ext_tool(x: int) -> int:
    """External tool: doubles x."""
    return 2 * x


def stream_step(tool_name, args_json, external_tools=None):
    model = make_model(stream=True)

    def mock_run(*a, **k):
        def gen():
            for chunk in tool_call_chunks(tool_name, args_json):
                yield chunk

        return gen()

    model.run = MagicMock(side_effect=mock_run)
    agent = ChatAgent(
        system_message="You are a helpful assistant.",
        model=model,
        tools=[add],
        external_tools=external_tools,
        stream_accumulate=False,
    )
    schemas = [s["function"]["name"] for s in agent._get_full_tool_schemas()]
    resp = agent.step("Please call the tool.")
    finals = [r for r in resp]
    print(
        f"streaming {tool_name!r}: yielded {len(finals)} responses; "
        f"resp.msgs={resp.msgs}; resp.info={resp.info}"
    )
    print(f"  schemas sent to model: {schemas}")
    print(f"  memory: {memory_of(agent)}")


def nonstream_step(tool_name, args_json, external_tools=None):
    model = make_model(stream=False)
    calls = {"n": 0}

    def mock_run(*a, **k):
        calls["n"] += 1
        if calls["n"] == 1:
            return tool_call_completion(tool_name, args_json)
        return plain_completion("all done")

    model.run = MagicMock(side_effect=mock_run)
    agent = ChatAgent(
        system_message="You are a helpful assistant.",
        model=model,
        tools=[add],
        external_tools=external_tools,
    )
    resp = agent.step("Please call the tool.")
    info = resp.info if isinstance(resp.info, list) else [resp.info]
    print(f"non-streaming {tool_name!r}: info[0]: {info[0]}")


# 1) hallucinated tool name, streaming -> dropped
stream_step("hallucinated_tool", '{"q": 1}')
# 2) same input, non-streaming -> error result recorded, model can recover
nonstream_step("hallucinated_tool", '{"q": 1}')
# 3) external tool, streaming -> schema is sent but the call is dropped
stream_step("ext_tool", '{"x": 5}', external_tools=[FunctionTool(ext_tool)])
# 4) same input, non-streaming -> request returned to the caller
nonstream_step(
    "ext_tool", '{"x": 5}', external_tools=[FunctionTool(ext_tool)]
)