From 030b434585b63a3d9934aaec90ef5cfdd84f87c3 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Sat, 18 Jul 2026 06:28:38 +0000 Subject: [PATCH] fix(agents): stop priming tool calls in schema-only structured agents - with_structured_output binds a single tool (the schema), so a primed model emitted an unknown web_search call and the attempt was discarded for a free-text retry, costing an extra round trip and the typed output - drop the tool-range wording from the no-tool sentiment analyst and state the constraint once via a shared NO_EXTERNAL_TOOLS #1130 --- tests/test_structured_agent_prompts.py | 147 ++++++++++++++++++ .../agents/analysts/sentiment_analyst.py | 7 +- .../agents/managers/portfolio_manager.py | 5 +- .../agents/managers/research_manager.py | 5 +- tradingagents/agents/trader/trader.py | 4 +- tradingagents/agents/utils/structured.py | 10 ++ 6 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 tests/test_structured_agent_prompts.py diff --git a/tests/test_structured_agent_prompts.py b/tests/test_structured_agent_prompts.py new file mode 100644 index 000000000..d876225cc --- /dev/null +++ b/tests/test_structured_agent_prompts.py @@ -0,0 +1,147 @@ +"""Agents on the schema-only structured-output path must not invite tool calls (#1130). + +`with_structured_output` binds exactly one tool (the schema). A prompt that +primes tool use makes models emit an unknown `web_search` call, which discards +the structured attempt and forces a free-text retry — an extra LLM round trip +and the loss of typed output. + +These assert the constraint reaches the *rendered* prompt each agent actually +sends, not merely that the constant is referenced in the module. +""" +from __future__ import annotations + +import inspect +from unittest.mock import MagicMock + +import pytest + +import tradingagents.agents.analysts.sentiment_analyst as sentiment +from tradingagents.agents.managers.portfolio_manager import create_portfolio_manager +from tradingagents.agents.managers.research_manager import create_research_manager +from tradingagents.agents.trader.trader import create_trader +from tradingagents.agents.utils.structured import NO_EXTERNAL_TOOLS + + +def _capturing_llm(captured: dict, result): + """LLM whose structured binding records the prompt it was handed.""" + structured = MagicMock() + structured.invoke.side_effect = lambda prompt: ( + captured.__setitem__("prompt", prompt) or result + ) + llm = MagicMock() + llm.with_structured_output.return_value = structured + return llm + + +def _prompt_text(prompt) -> str: + """Flatten a captured prompt (str, message list, or objects) to text.""" + if isinstance(prompt, str): + return prompt + parts = [] + for m in prompt: + parts.append(m.get("content", "") if isinstance(m, dict) else getattr(m, "content", "")) + return "\n".join(str(p) for p in parts) + + +@pytest.mark.unit +def test_trader_prompt_states_constraint(): + from tradingagents.agents.schemas import TraderAction, TraderProposal + + captured = {} + llm = _capturing_llm(captured, TraderProposal(action=TraderAction.BUY, reasoning="x")) + create_trader(llm)({ + "company_of_interest": "NVDA", + "investment_plan": "**Recommendation**: Buy", + }) + assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"]) + + +@pytest.mark.unit +def test_research_manager_prompt_states_constraint(): + from tradingagents.agents.schemas import PortfolioRating, ResearchPlan + + captured = {} + llm = _capturing_llm( + captured, + ResearchPlan( + recommendation=PortfolioRating.BUY, rationale="x", strategic_actions="y" + ), + ) + create_research_manager(llm)({ + "company_of_interest": "NVDA", + "investment_debate_state": { + "history": "h", "bull_history": "b", "bear_history": "r", + "current_response": "", "judge_decision": "", "count": 1, + }, + }) + assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"]) + + +@pytest.mark.unit +def test_portfolio_manager_prompt_states_constraint(): + from tradingagents.agents.schemas import PortfolioDecision, PortfolioRating + + captured = {} + llm = _capturing_llm( + captured, + PortfolioDecision( + rating=PortfolioRating.HOLD, + executive_summary="x", + investment_thesis="y", + ), + ) + risk = { + "history": "h", "aggressive_history": "a", "conservative_history": "c", + "neutral_history": "n", "current_aggressive_response": "", + "current_conservative_response": "", "current_neutral_response": "", + "latest_speaker": "Neutral", "count": 1, + } + create_portfolio_manager(llm)({ + "company_of_interest": "NVDA", + "risk_debate_state": risk, + "investment_plan": "plan", + "trader_investment_plan": "trader plan", + }) + assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"]) + + +@pytest.mark.unit +def test_sentiment_prompt_states_constraint(monkeypatch): + from tradingagents.agents.schemas import SentimentBand, SentimentReport + + # Pre-fetched sources are stubbed so the prompt builds without network I/O. + monkeypatch.setattr(sentiment, "fetch_stocktwits_messages", lambda *a, **k: "st") + monkeypatch.setattr(sentiment, "fetch_reddit_posts", lambda *a, **k: "rd") + monkeypatch.setattr(sentiment.get_news, "func", lambda *a, **k: "news", raising=False) + + captured = {} + llm = _capturing_llm(captured, SentimentReport( + overall_band=SentimentBand.BULLISH, overall_score=7.5, + confidence="high", narrative="n", + )) + sentiment.create_sentiment_analyst(llm)({ + "company_of_interest": "NVDA", "trade_date": "2026-01-15", + "asset_type": "stock", "messages": [], + }) + text = _prompt_text(captured["prompt"]) + assert NO_EXTERNAL_TOOLS in text + # This agent binds no tools, so tool-range wording must not reappear. + assert "tool-call date ranges" not in text + + +@pytest.mark.unit +def test_tool_using_analysts_keep_their_date_guidance(): + # The analysts that really do call tools keep the wording that anchors their + # tool date ranges (#836) — this fix is scoped to no-tool agents. + import tradingagents.agents.analysts.market_analyst as market + import tradingagents.agents.analysts.news_analyst as news + for module in (market, news): + assert "tool-call date ranges" in inspect.getsource(module) + + +@pytest.mark.unit +def test_constraint_text_is_unambiguous(): + assert "do not call external tools" in NO_EXTERNAL_TOOLS.lower() + # No template braces: it is embedded in ChatPromptTemplate strings, where + # braces would be parsed as input variables. + assert "{" not in NO_EXTERNAL_TOOLS and "}" not in NO_EXTERNAL_TOOLS diff --git a/tradingagents/agents/analysts/sentiment_analyst.py b/tradingagents/agents/analysts/sentiment_analyst.py index 68bb8c5f8..e61dc1ae2 100644 --- a/tradingagents/agents/analysts/sentiment_analyst.py +++ b/tradingagents/agents/analysts/sentiment_analyst.py @@ -36,6 +36,7 @@ from tradingagents.agents.utils.agent_utils import ( get_news, ) from tradingagents.agents.utils.structured import ( + NO_EXTERNAL_TOOLS, bind_structured, invoke_structured_or_freetext, ) @@ -86,7 +87,11 @@ def create_sentiment_analyst(llm): "You are a helpful AI assistant, collaborating with other assistants." " If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable," " prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop." - " Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}" + # No tool-calling here: the data is pre-fetched into the + # prompt, so tool-range wording would only invite a + # hallucinated tool call (#1130). + " Today's date is {current_date}; treat it as 'now' for all analysis. {instrument_context}" + " " + NO_EXTERNAL_TOOLS + "\n{system_message}", ), MessagesPlaceholder(variable_name="messages"), diff --git a/tradingagents/agents/managers/portfolio_manager.py b/tradingagents/agents/managers/portfolio_manager.py index 36cdef549..00d5503cd 100644 --- a/tradingagents/agents/managers/portfolio_manager.py +++ b/tradingagents/agents/managers/portfolio_manager.py @@ -16,6 +16,7 @@ from tradingagents.agents.utils.agent_utils import ( get_language_instruction, ) from tradingagents.agents.utils.structured import ( + NO_EXTERNAL_TOOLS, bind_structured, invoke_structured_or_freetext, ) @@ -61,7 +62,9 @@ def create_portfolio_manager(llm): --- -Be decisive and ground every conclusion in specific evidence from the analysts.{get_language_instruction()}""" +Be decisive and ground every conclusion in specific evidence from the analysts. + +{NO_EXTERNAL_TOOLS}{get_language_instruction()}""" final_trade_decision = invoke_structured_or_freetext( structured_llm, diff --git a/tradingagents/agents/managers/research_manager.py b/tradingagents/agents/managers/research_manager.py index 7d39ed0b7..1aec89572 100644 --- a/tradingagents/agents/managers/research_manager.py +++ b/tradingagents/agents/managers/research_manager.py @@ -8,6 +8,7 @@ from tradingagents.agents.utils.agent_utils import ( get_language_instruction, ) from tradingagents.agents.utils.structured import ( + NO_EXTERNAL_TOOLS, bind_structured, invoke_structured_or_freetext, ) @@ -40,7 +41,9 @@ Commit to a clear stance whenever the debate's strongest arguments warrant one; --- **Debate History:** -{history}""" + get_language_instruction() +{history} + +{NO_EXTERNAL_TOOLS}""" + get_language_instruction() investment_plan = invoke_structured_or_freetext( structured_llm, diff --git a/tradingagents/agents/trader/trader.py b/tradingagents/agents/trader/trader.py index 403a6a57d..6c8ac50e4 100644 --- a/tradingagents/agents/trader/trader.py +++ b/tradingagents/agents/trader/trader.py @@ -12,6 +12,7 @@ from tradingagents.agents.utils.agent_utils import ( get_language_instruction, ) from tradingagents.agents.utils.structured import ( + NO_EXTERNAL_TOOLS, bind_structured, invoke_structured_or_freetext, ) @@ -31,7 +32,8 @@ def create_trader(llm): "content": ( "You are a trading agent analyzing market data to make investment decisions. " "Based on your analysis, provide a specific recommendation to buy, sell, or hold. " - "Anchor your reasoning in the analysts' reports and the research plan." + "Anchor your reasoning in the analysts' reports and the research plan. " + + NO_EXTERNAL_TOOLS + get_language_instruction() ), }, diff --git a/tradingagents/agents/utils/structured.py b/tradingagents/agents/utils/structured.py index 56019bc1e..8e94132c9 100644 --- a/tradingagents/agents/utils/structured.py +++ b/tradingagents/agents/utils/structured.py @@ -28,6 +28,16 @@ logger = logging.getLogger(__name__) T = TypeVar("T", bound=BaseModel) +# Schema-only structured output binds exactly one tool (the schema itself), so a +# model that reaches for a search tool emits an unknown tool call and the whole +# structured attempt is discarded for a free-text retry. Agents on this path +# state the constraint explicitly rather than relying on the binding alone +# (#1130). +NO_EXTERNAL_TOOLS = ( + "Use only the evidence provided in this prompt. Do not call external tools " + "or search the web; if something is missing, say so explicitly." +) + def bind_structured(llm: Any, schema: type[T], agent_name: str) -> Any | None: """Return ``llm.with_structured_output(schema)`` or ``None`` if unsupported.