From 517eeaf4b9ad2dfecb852fef78125ff6370ff640 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Sun, 21 Jun 2026 22:09:43 +0000 Subject: [PATCH] fix(structured): harden structured output for local servers and thinking models - Local servers (LM Studio, vLLM) reject the object-form tool_choice langchain sends for function calling. The generic openai_compatible provider now binds the schema as a tool without forcing tool_choice. - A structured call can return no parsed result (a thinking model answering in plain text); fall back to free text with a clear reason instead of an opaque render error. --- tests/test_openai_compatible_provider.py | 28 +++++++++++++++++++++- tests/test_structured_agents.py | 18 ++++++++++++++ tradingagents/agents/utils/structured.py | 5 ++++ tradingagents/llm_clients/openai_client.py | 21 +++++++++++++++- 4 files changed, 70 insertions(+), 2 deletions(-) diff --git a/tests/test_openai_compatible_provider.py b/tests/test_openai_compatible_provider.py index ab253e26a..57c3439c4 100644 --- a/tests/test_openai_compatible_provider.py +++ b/tests/test_openai_compatible_provider.py @@ -36,7 +36,7 @@ def test_keyless_local_uses_placeholder_and_chat_completions(monkeypatch): llm = create_llm_client( provider="openai_compatible", model="qwen2.5", base_url="http://localhost:8000/v1" ).get_llm() - assert type(llm).__name__ == "NormalizedChatOpenAI" + assert type(llm).__name__ == "LocalCompatibleChatOpenAI" assert str(llm.openai_api_base) == "http://localhost:8000/v1" # keyless local servers: a placeholder key is sent key = llm.openai_api_key.get_secret_value() if hasattr(llm.openai_api_key, "get_secret_value") else llm.openai_api_key @@ -72,3 +72,29 @@ def test_env_backend_url_precedence(): assert resolve_backend_url("openai", "https://api.openai.com/v1", env_url="http://proxy/v1") == "http://proxy/v1" assert resolve_backend_url("openai", "https://api.openai.com/v1", env_url=None) == "https://api.openai.com/v1" assert resolve_backend_url("deepseek", None, None) == "https://api.deepseek.com" + + +@pytest.mark.unit +def test_structured_output_suppresses_object_tool_choice(monkeypatch): + # LM Studio / vLLM reject the object-form tool_choice langchain sends for + # function-calling structured output (#1057). The generic provider binds the + # schema as a tool but must not force tool_choice. + from langchain_openai import ChatOpenAI + from pydantic import BaseModel + + class Schema(BaseModel): + x: int + + captured = {} + monkeypatch.setattr( + ChatOpenAI, + "with_structured_output", + lambda self, schema, method=None, **kw: captured.update({"method": method, **kw}) or "BOUND", + ) + llm = create_llm_client( + provider="openai_compatible", model="local-llm-30b", base_url="http://localhost:1234/v1" + ).get_llm() + out = llm.with_structured_output(Schema) + assert out == "BOUND" + assert captured["method"] == "function_calling" + assert captured["tool_choice"] is None # not the object form diff --git a/tests/test_structured_agents.py b/tests/test_structured_agents.py index d80063206..dddf741b3 100644 --- a/tests/test_structured_agents.py +++ b/tests/test_structured_agents.py @@ -121,6 +121,24 @@ def _structured_trader_llm(captured: dict, proposal: TraderProposal | None = Non return llm +@pytest.mark.unit +def test_invoke_structured_falls_back_when_result_is_none(): + # A thinking model can answer in plain text, leaving the parser with None. + # That must fall back to free text, not crash on render(None) (#1051). + from tradingagents.agents.utils.structured import invoke_structured_or_freetext + + structured = MagicMock() + structured.invoke.return_value = None + plain = MagicMock() + plain.invoke.return_value = MagicMock(content="FREETEXT") + + out = invoke_structured_or_freetext( + structured, plain, "prompt", render=lambda r: r.rating, agent_name="t" + ) + assert out == "FREETEXT" + plain.invoke.assert_called_once() + + @pytest.mark.unit class TestTraderAgent: def test_structured_path_produces_rendered_markdown(self): diff --git a/tradingagents/agents/utils/structured.py b/tradingagents/agents/utils/structured.py index 28dac8b55..56019bc1e 100644 --- a/tradingagents/agents/utils/structured.py +++ b/tradingagents/agents/utils/structured.py @@ -63,6 +63,11 @@ def invoke_structured_or_freetext( if structured_llm is not None: try: result = structured_llm.invoke(prompt) + if result is None: + # A thinking model can answer in plain text instead of calling + # the tool, leaving the parser with nothing to return. Treat it + # as a structured miss and fall back, with a clear reason. + raise ValueError("structured output returned no parsed result") return render(result) except Exception as exc: logger.warning( diff --git a/tradingagents/llm_clients/openai_client.py b/tradingagents/llm_clients/openai_client.py index da65f58f9..bf6e55457 100644 --- a/tradingagents/llm_clients/openai_client.py +++ b/tradingagents/llm_clients/openai_client.py @@ -51,6 +51,23 @@ class NormalizedChatOpenAI(ChatOpenAI): return super().with_structured_output(schema, method=method, **kwargs) +class LocalCompatibleChatOpenAI(NormalizedChatOpenAI): + """OpenAI-compatible client for arbitrary local servers (LM Studio, vLLM, + llama.cpp via the generic ``openai_compatible`` provider). + + Their tool-calling support varies, and many reject the object-form + ``tool_choice`` langchain sends for function-calling structured output. Bind + the schema as a tool but don't force tool_choice, so structured output works + across local servers regardless of the model ID's capabilities (#1057). + """ + + def with_structured_output(self, schema, *, method=None, **kwargs): + resolved = method or get_capabilities(self.model_name).preferred_structured_method + if resolved == "function_calling": + kwargs.setdefault("tool_choice", None) + return super().with_structured_output(schema, method=method, **kwargs) + + def _input_to_messages(input_: Any) -> list: """Normalise a langchain LLM input to a list of message objects. @@ -210,7 +227,9 @@ OPENAI_COMPATIBLE_PROVIDERS: dict[str, ProviderSpec] = { "ollama": ProviderSpec(base_url="http://localhost:11434/v1", base_url_env="OLLAMA_BASE_URL", key_optional=True, placeholder_key="ollama"), # Generic endpoint: user supplies base_url; key optional (keyless local). - "openai_compatible": ProviderSpec(require_base_url=True, key_optional=True), + "openai_compatible": ProviderSpec( + require_base_url=True, key_optional=True, chat_class=LocalCompatibleChatOpenAI + ), }