fix(agents): stop priming tool calls in schema-only structured agents

- with_structured_output binds a single tool (the schema), so a primed model emitted
  an unknown web_search call and the attempt was discarded for a free-text retry,
  costing an extra round trip and the typed output
- drop the tool-range wording from the no-tool sentiment analyst and state the
  constraint once via a shared NO_EXTERNAL_TOOLS #1130
This commit is contained in:
Yijia-Xiao
2026-07-18 06:28:38 +00:00
parent 3f6c082695
commit 030b434585
6 changed files with 174 additions and 4 deletions

View File

@@ -0,0 +1,147 @@
"""Agents on the schema-only structured-output path must not invite tool calls (#1130).
`with_structured_output` binds exactly one tool (the schema). A prompt that
primes tool use makes models emit an unknown `web_search` call, which discards
the structured attempt and forces a free-text retry — an extra LLM round trip
and the loss of typed output.
These assert the constraint reaches the *rendered* prompt each agent actually
sends, not merely that the constant is referenced in the module.
"""
from __future__ import annotations
import inspect
from unittest.mock import MagicMock
import pytest
import tradingagents.agents.analysts.sentiment_analyst as sentiment
from tradingagents.agents.managers.portfolio_manager import create_portfolio_manager
from tradingagents.agents.managers.research_manager import create_research_manager
from tradingagents.agents.trader.trader import create_trader
from tradingagents.agents.utils.structured import NO_EXTERNAL_TOOLS
def _capturing_llm(captured: dict, result):
"""LLM whose structured binding records the prompt it was handed."""
structured = MagicMock()
structured.invoke.side_effect = lambda prompt: (
captured.__setitem__("prompt", prompt) or result
)
llm = MagicMock()
llm.with_structured_output.return_value = structured
return llm
def _prompt_text(prompt) -> str:
"""Flatten a captured prompt (str, message list, or objects) to text."""
if isinstance(prompt, str):
return prompt
parts = []
for m in prompt:
parts.append(m.get("content", "") if isinstance(m, dict) else getattr(m, "content", ""))
return "\n".join(str(p) for p in parts)
@pytest.mark.unit
def test_trader_prompt_states_constraint():
from tradingagents.agents.schemas import TraderAction, TraderProposal
captured = {}
llm = _capturing_llm(captured, TraderProposal(action=TraderAction.BUY, reasoning="x"))
create_trader(llm)({
"company_of_interest": "NVDA",
"investment_plan": "**Recommendation**: Buy",
})
assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"])
@pytest.mark.unit
def test_research_manager_prompt_states_constraint():
from tradingagents.agents.schemas import PortfolioRating, ResearchPlan
captured = {}
llm = _capturing_llm(
captured,
ResearchPlan(
recommendation=PortfolioRating.BUY, rationale="x", strategic_actions="y"
),
)
create_research_manager(llm)({
"company_of_interest": "NVDA",
"investment_debate_state": {
"history": "h", "bull_history": "b", "bear_history": "r",
"current_response": "", "judge_decision": "", "count": 1,
},
})
assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"])
@pytest.mark.unit
def test_portfolio_manager_prompt_states_constraint():
from tradingagents.agents.schemas import PortfolioDecision, PortfolioRating
captured = {}
llm = _capturing_llm(
captured,
PortfolioDecision(
rating=PortfolioRating.HOLD,
executive_summary="x",
investment_thesis="y",
),
)
risk = {
"history": "h", "aggressive_history": "a", "conservative_history": "c",
"neutral_history": "n", "current_aggressive_response": "",
"current_conservative_response": "", "current_neutral_response": "",
"latest_speaker": "Neutral", "count": 1,
}
create_portfolio_manager(llm)({
"company_of_interest": "NVDA",
"risk_debate_state": risk,
"investment_plan": "plan",
"trader_investment_plan": "trader plan",
})
assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"])
@pytest.mark.unit
def test_sentiment_prompt_states_constraint(monkeypatch):
from tradingagents.agents.schemas import SentimentBand, SentimentReport
# Pre-fetched sources are stubbed so the prompt builds without network I/O.
monkeypatch.setattr(sentiment, "fetch_stocktwits_messages", lambda *a, **k: "st")
monkeypatch.setattr(sentiment, "fetch_reddit_posts", lambda *a, **k: "rd")
monkeypatch.setattr(sentiment.get_news, "func", lambda *a, **k: "news", raising=False)
captured = {}
llm = _capturing_llm(captured, SentimentReport(
overall_band=SentimentBand.BULLISH, overall_score=7.5,
confidence="high", narrative="n",
))
sentiment.create_sentiment_analyst(llm)({
"company_of_interest": "NVDA", "trade_date": "2026-01-15",
"asset_type": "stock", "messages": [],
})
text = _prompt_text(captured["prompt"])
assert NO_EXTERNAL_TOOLS in text
# This agent binds no tools, so tool-range wording must not reappear.
assert "tool-call date ranges" not in text
@pytest.mark.unit
def test_tool_using_analysts_keep_their_date_guidance():
# The analysts that really do call tools keep the wording that anchors their
# tool date ranges (#836) — this fix is scoped to no-tool agents.
import tradingagents.agents.analysts.market_analyst as market
import tradingagents.agents.analysts.news_analyst as news
for module in (market, news):
assert "tool-call date ranges" in inspect.getsource(module)
@pytest.mark.unit
def test_constraint_text_is_unambiguous():
assert "do not call external tools" in NO_EXTERNAL_TOOLS.lower()
# No template braces: it is embedded in ChatPromptTemplate strings, where
# braces would be parsed as input variables.
assert "{" not in NO_EXTERNAL_TOOLS and "}" not in NO_EXTERNAL_TOOLS

View File

@@ -36,6 +36,7 @@ from tradingagents.agents.utils.agent_utils import (
get_news, get_news,
) )
from tradingagents.agents.utils.structured import ( from tradingagents.agents.utils.structured import (
NO_EXTERNAL_TOOLS,
bind_structured, bind_structured,
invoke_structured_or_freetext, invoke_structured_or_freetext,
) )
@@ -86,7 +87,11 @@ def create_sentiment_analyst(llm):
"You are a helpful AI assistant, collaborating with other assistants." "You are a helpful AI assistant, collaborating with other assistants."
" If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable," " If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable,"
" prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop." " prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop."
" Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}" # No tool-calling here: the data is pre-fetched into the
# prompt, so tool-range wording would only invite a
# hallucinated tool call (#1130).
" Today's date is {current_date}; treat it as 'now' for all analysis. {instrument_context}"
" " + NO_EXTERNAL_TOOLS +
"\n{system_message}", "\n{system_message}",
), ),
MessagesPlaceholder(variable_name="messages"), MessagesPlaceholder(variable_name="messages"),

View File

@@ -16,6 +16,7 @@ from tradingagents.agents.utils.agent_utils import (
get_language_instruction, get_language_instruction,
) )
from tradingagents.agents.utils.structured import ( from tradingagents.agents.utils.structured import (
NO_EXTERNAL_TOOLS,
bind_structured, bind_structured,
invoke_structured_or_freetext, invoke_structured_or_freetext,
) )
@@ -61,7 +62,9 @@ def create_portfolio_manager(llm):
--- ---
Be decisive and ground every conclusion in specific evidence from the analysts.{get_language_instruction()}""" Be decisive and ground every conclusion in specific evidence from the analysts.
{NO_EXTERNAL_TOOLS}{get_language_instruction()}"""
final_trade_decision = invoke_structured_or_freetext( final_trade_decision = invoke_structured_or_freetext(
structured_llm, structured_llm,

View File

@@ -8,6 +8,7 @@ from tradingagents.agents.utils.agent_utils import (
get_language_instruction, get_language_instruction,
) )
from tradingagents.agents.utils.structured import ( from tradingagents.agents.utils.structured import (
NO_EXTERNAL_TOOLS,
bind_structured, bind_structured,
invoke_structured_or_freetext, invoke_structured_or_freetext,
) )
@@ -40,7 +41,9 @@ Commit to a clear stance whenever the debate's strongest arguments warrant one;
--- ---
**Debate History:** **Debate History:**
{history}""" + get_language_instruction() {history}
{NO_EXTERNAL_TOOLS}""" + get_language_instruction()
investment_plan = invoke_structured_or_freetext( investment_plan = invoke_structured_or_freetext(
structured_llm, structured_llm,

View File

@@ -12,6 +12,7 @@ from tradingagents.agents.utils.agent_utils import (
get_language_instruction, get_language_instruction,
) )
from tradingagents.agents.utils.structured import ( from tradingagents.agents.utils.structured import (
NO_EXTERNAL_TOOLS,
bind_structured, bind_structured,
invoke_structured_or_freetext, invoke_structured_or_freetext,
) )
@@ -32,6 +33,7 @@ def create_trader(llm):
"You are a trading agent analyzing market data to make investment decisions. " "You are a trading agent analyzing market data to make investment decisions. "
"Based on your analysis, provide a specific recommendation to buy, sell, or hold. " "Based on your analysis, provide a specific recommendation to buy, sell, or hold. "
"Anchor your reasoning in the analysts' reports and the research plan. " "Anchor your reasoning in the analysts' reports and the research plan. "
+ NO_EXTERNAL_TOOLS
+ get_language_instruction() + get_language_instruction()
), ),
}, },

View File

@@ -28,6 +28,16 @@ logger = logging.getLogger(__name__)
T = TypeVar("T", bound=BaseModel) T = TypeVar("T", bound=BaseModel)
# Schema-only structured output binds exactly one tool (the schema itself), so a
# model that reaches for a search tool emits an unknown tool call and the whole
# structured attempt is discarded for a free-text retry. Agents on this path
# state the constraint explicitly rather than relying on the binding alone
# (#1130).
NO_EXTERNAL_TOOLS = (
"Use only the evidence provided in this prompt. Do not call external tools "
"or search the web; if something is missing, say so explicitly."
)
def bind_structured(llm: Any, schema: type[T], agent_name: str) -> Any | None: def bind_structured(llm: Any, schema: type[T], agent_name: str) -> Any | None:
"""Return ``llm.with_structured_output(schema)`` or ``None`` if unsupported. """Return ``llm.with_structured_output(schema)`` or ``None`` if unsupported.