mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-08-01 19:34:24 +03:00
fix(agents): stop priming tool calls in schema-only structured agents
- with_structured_output binds a single tool (the schema), so a primed model emitted an unknown web_search call and the attempt was discarded for a free-text retry, costing an extra round trip and the typed output - drop the tool-range wording from the no-tool sentiment analyst and state the constraint once via a shared NO_EXTERNAL_TOOLS #1130
This commit is contained in:
147
tests/test_structured_agent_prompts.py
Normal file
147
tests/test_structured_agent_prompts.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""Agents on the schema-only structured-output path must not invite tool calls (#1130).
|
||||
|
||||
`with_structured_output` binds exactly one tool (the schema). A prompt that
|
||||
primes tool use makes models emit an unknown `web_search` call, which discards
|
||||
the structured attempt and forces a free-text retry — an extra LLM round trip
|
||||
and the loss of typed output.
|
||||
|
||||
These assert the constraint reaches the *rendered* prompt each agent actually
|
||||
sends, not merely that the constant is referenced in the module.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import tradingagents.agents.analysts.sentiment_analyst as sentiment
|
||||
from tradingagents.agents.managers.portfolio_manager import create_portfolio_manager
|
||||
from tradingagents.agents.managers.research_manager import create_research_manager
|
||||
from tradingagents.agents.trader.trader import create_trader
|
||||
from tradingagents.agents.utils.structured import NO_EXTERNAL_TOOLS
|
||||
|
||||
|
||||
def _capturing_llm(captured: dict, result):
|
||||
"""LLM whose structured binding records the prompt it was handed."""
|
||||
structured = MagicMock()
|
||||
structured.invoke.side_effect = lambda prompt: (
|
||||
captured.__setitem__("prompt", prompt) or result
|
||||
)
|
||||
llm = MagicMock()
|
||||
llm.with_structured_output.return_value = structured
|
||||
return llm
|
||||
|
||||
|
||||
def _prompt_text(prompt) -> str:
|
||||
"""Flatten a captured prompt (str, message list, or objects) to text."""
|
||||
if isinstance(prompt, str):
|
||||
return prompt
|
||||
parts = []
|
||||
for m in prompt:
|
||||
parts.append(m.get("content", "") if isinstance(m, dict) else getattr(m, "content", ""))
|
||||
return "\n".join(str(p) for p in parts)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_trader_prompt_states_constraint():
|
||||
from tradingagents.agents.schemas import TraderAction, TraderProposal
|
||||
|
||||
captured = {}
|
||||
llm = _capturing_llm(captured, TraderProposal(action=TraderAction.BUY, reasoning="x"))
|
||||
create_trader(llm)({
|
||||
"company_of_interest": "NVDA",
|
||||
"investment_plan": "**Recommendation**: Buy",
|
||||
})
|
||||
assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"])
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_research_manager_prompt_states_constraint():
|
||||
from tradingagents.agents.schemas import PortfolioRating, ResearchPlan
|
||||
|
||||
captured = {}
|
||||
llm = _capturing_llm(
|
||||
captured,
|
||||
ResearchPlan(
|
||||
recommendation=PortfolioRating.BUY, rationale="x", strategic_actions="y"
|
||||
),
|
||||
)
|
||||
create_research_manager(llm)({
|
||||
"company_of_interest": "NVDA",
|
||||
"investment_debate_state": {
|
||||
"history": "h", "bull_history": "b", "bear_history": "r",
|
||||
"current_response": "", "judge_decision": "", "count": 1,
|
||||
},
|
||||
})
|
||||
assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"])
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_portfolio_manager_prompt_states_constraint():
|
||||
from tradingagents.agents.schemas import PortfolioDecision, PortfolioRating
|
||||
|
||||
captured = {}
|
||||
llm = _capturing_llm(
|
||||
captured,
|
||||
PortfolioDecision(
|
||||
rating=PortfolioRating.HOLD,
|
||||
executive_summary="x",
|
||||
investment_thesis="y",
|
||||
),
|
||||
)
|
||||
risk = {
|
||||
"history": "h", "aggressive_history": "a", "conservative_history": "c",
|
||||
"neutral_history": "n", "current_aggressive_response": "",
|
||||
"current_conservative_response": "", "current_neutral_response": "",
|
||||
"latest_speaker": "Neutral", "count": 1,
|
||||
}
|
||||
create_portfolio_manager(llm)({
|
||||
"company_of_interest": "NVDA",
|
||||
"risk_debate_state": risk,
|
||||
"investment_plan": "plan",
|
||||
"trader_investment_plan": "trader plan",
|
||||
})
|
||||
assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"])
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_sentiment_prompt_states_constraint(monkeypatch):
|
||||
from tradingagents.agents.schemas import SentimentBand, SentimentReport
|
||||
|
||||
# Pre-fetched sources are stubbed so the prompt builds without network I/O.
|
||||
monkeypatch.setattr(sentiment, "fetch_stocktwits_messages", lambda *a, **k: "st")
|
||||
monkeypatch.setattr(sentiment, "fetch_reddit_posts", lambda *a, **k: "rd")
|
||||
monkeypatch.setattr(sentiment.get_news, "func", lambda *a, **k: "news", raising=False)
|
||||
|
||||
captured = {}
|
||||
llm = _capturing_llm(captured, SentimentReport(
|
||||
overall_band=SentimentBand.BULLISH, overall_score=7.5,
|
||||
confidence="high", narrative="n",
|
||||
))
|
||||
sentiment.create_sentiment_analyst(llm)({
|
||||
"company_of_interest": "NVDA", "trade_date": "2026-01-15",
|
||||
"asset_type": "stock", "messages": [],
|
||||
})
|
||||
text = _prompt_text(captured["prompt"])
|
||||
assert NO_EXTERNAL_TOOLS in text
|
||||
# This agent binds no tools, so tool-range wording must not reappear.
|
||||
assert "tool-call date ranges" not in text
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_tool_using_analysts_keep_their_date_guidance():
|
||||
# The analysts that really do call tools keep the wording that anchors their
|
||||
# tool date ranges (#836) — this fix is scoped to no-tool agents.
|
||||
import tradingagents.agents.analysts.market_analyst as market
|
||||
import tradingagents.agents.analysts.news_analyst as news
|
||||
for module in (market, news):
|
||||
assert "tool-call date ranges" in inspect.getsource(module)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_constraint_text_is_unambiguous():
|
||||
assert "do not call external tools" in NO_EXTERNAL_TOOLS.lower()
|
||||
# No template braces: it is embedded in ChatPromptTemplate strings, where
|
||||
# braces would be parsed as input variables.
|
||||
assert "{" not in NO_EXTERNAL_TOOLS and "}" not in NO_EXTERNAL_TOOLS
|
||||
@@ -36,6 +36,7 @@ from tradingagents.agents.utils.agent_utils import (
|
||||
get_news,
|
||||
)
|
||||
from tradingagents.agents.utils.structured import (
|
||||
NO_EXTERNAL_TOOLS,
|
||||
bind_structured,
|
||||
invoke_structured_or_freetext,
|
||||
)
|
||||
@@ -86,7 +87,11 @@ def create_sentiment_analyst(llm):
|
||||
"You are a helpful AI assistant, collaborating with other assistants."
|
||||
" If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable,"
|
||||
" prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop."
|
||||
" Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}"
|
||||
# No tool-calling here: the data is pre-fetched into the
|
||||
# prompt, so tool-range wording would only invite a
|
||||
# hallucinated tool call (#1130).
|
||||
" Today's date is {current_date}; treat it as 'now' for all analysis. {instrument_context}"
|
||||
" " + NO_EXTERNAL_TOOLS +
|
||||
"\n{system_message}",
|
||||
),
|
||||
MessagesPlaceholder(variable_name="messages"),
|
||||
|
||||
@@ -16,6 +16,7 @@ from tradingagents.agents.utils.agent_utils import (
|
||||
get_language_instruction,
|
||||
)
|
||||
from tradingagents.agents.utils.structured import (
|
||||
NO_EXTERNAL_TOOLS,
|
||||
bind_structured,
|
||||
invoke_structured_or_freetext,
|
||||
)
|
||||
@@ -61,7 +62,9 @@ def create_portfolio_manager(llm):
|
||||
|
||||
---
|
||||
|
||||
Be decisive and ground every conclusion in specific evidence from the analysts.{get_language_instruction()}"""
|
||||
Be decisive and ground every conclusion in specific evidence from the analysts.
|
||||
|
||||
{NO_EXTERNAL_TOOLS}{get_language_instruction()}"""
|
||||
|
||||
final_trade_decision = invoke_structured_or_freetext(
|
||||
structured_llm,
|
||||
|
||||
@@ -8,6 +8,7 @@ from tradingagents.agents.utils.agent_utils import (
|
||||
get_language_instruction,
|
||||
)
|
||||
from tradingagents.agents.utils.structured import (
|
||||
NO_EXTERNAL_TOOLS,
|
||||
bind_structured,
|
||||
invoke_structured_or_freetext,
|
||||
)
|
||||
@@ -40,7 +41,9 @@ Commit to a clear stance whenever the debate's strongest arguments warrant one;
|
||||
---
|
||||
|
||||
**Debate History:**
|
||||
{history}""" + get_language_instruction()
|
||||
{history}
|
||||
|
||||
{NO_EXTERNAL_TOOLS}""" + get_language_instruction()
|
||||
|
||||
investment_plan = invoke_structured_or_freetext(
|
||||
structured_llm,
|
||||
|
||||
@@ -12,6 +12,7 @@ from tradingagents.agents.utils.agent_utils import (
|
||||
get_language_instruction,
|
||||
)
|
||||
from tradingagents.agents.utils.structured import (
|
||||
NO_EXTERNAL_TOOLS,
|
||||
bind_structured,
|
||||
invoke_structured_or_freetext,
|
||||
)
|
||||
@@ -32,6 +33,7 @@ def create_trader(llm):
|
||||
"You are a trading agent analyzing market data to make investment decisions. "
|
||||
"Based on your analysis, provide a specific recommendation to buy, sell, or hold. "
|
||||
"Anchor your reasoning in the analysts' reports and the research plan. "
|
||||
+ NO_EXTERNAL_TOOLS
|
||||
+ get_language_instruction()
|
||||
),
|
||||
},
|
||||
|
||||
@@ -28,6 +28,16 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
T = TypeVar("T", bound=BaseModel)
|
||||
|
||||
# Schema-only structured output binds exactly one tool (the schema itself), so a
|
||||
# model that reaches for a search tool emits an unknown tool call and the whole
|
||||
# structured attempt is discarded for a free-text retry. Agents on this path
|
||||
# state the constraint explicitly rather than relying on the binding alone
|
||||
# (#1130).
|
||||
NO_EXTERNAL_TOOLS = (
|
||||
"Use only the evidence provided in this prompt. Do not call external tools "
|
||||
"or search the web; if something is missing, say so explicitly."
|
||||
)
|
||||
|
||||
|
||||
def bind_structured(llm: Any, schema: type[T], agent_name: str) -> Any | None:
|
||||
"""Return ``llm.with_structured_output(schema)`` or ``None`` if unsupported.
|
||||
|
||||
Reference in New Issue
Block a user