mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-08-01 19:34:24 +03:00
fix(agents): stop priming tool calls in schema-only structured agents
- with_structured_output binds a single tool (the schema), so a primed model emitted an unknown web_search call and the attempt was discarded for a free-text retry, costing an extra round trip and the typed output - drop the tool-range wording from the no-tool sentiment analyst and state the constraint once via a shared NO_EXTERNAL_TOOLS #1130
This commit is contained in:
147
tests/test_structured_agent_prompts.py
Normal file
147
tests/test_structured_agent_prompts.py
Normal file
@@ -0,0 +1,147 @@
|
|||||||
|
"""Agents on the schema-only structured-output path must not invite tool calls (#1130).
|
||||||
|
|
||||||
|
`with_structured_output` binds exactly one tool (the schema). A prompt that
|
||||||
|
primes tool use makes models emit an unknown `web_search` call, which discards
|
||||||
|
the structured attempt and forces a free-text retry — an extra LLM round trip
|
||||||
|
and the loss of typed output.
|
||||||
|
|
||||||
|
These assert the constraint reaches the *rendered* prompt each agent actually
|
||||||
|
sends, not merely that the constant is referenced in the module.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import inspect
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import tradingagents.agents.analysts.sentiment_analyst as sentiment
|
||||||
|
from tradingagents.agents.managers.portfolio_manager import create_portfolio_manager
|
||||||
|
from tradingagents.agents.managers.research_manager import create_research_manager
|
||||||
|
from tradingagents.agents.trader.trader import create_trader
|
||||||
|
from tradingagents.agents.utils.structured import NO_EXTERNAL_TOOLS
|
||||||
|
|
||||||
|
|
||||||
|
def _capturing_llm(captured: dict, result):
|
||||||
|
"""LLM whose structured binding records the prompt it was handed."""
|
||||||
|
structured = MagicMock()
|
||||||
|
structured.invoke.side_effect = lambda prompt: (
|
||||||
|
captured.__setitem__("prompt", prompt) or result
|
||||||
|
)
|
||||||
|
llm = MagicMock()
|
||||||
|
llm.with_structured_output.return_value = structured
|
||||||
|
return llm
|
||||||
|
|
||||||
|
|
||||||
|
def _prompt_text(prompt) -> str:
|
||||||
|
"""Flatten a captured prompt (str, message list, or objects) to text."""
|
||||||
|
if isinstance(prompt, str):
|
||||||
|
return prompt
|
||||||
|
parts = []
|
||||||
|
for m in prompt:
|
||||||
|
parts.append(m.get("content", "") if isinstance(m, dict) else getattr(m, "content", ""))
|
||||||
|
return "\n".join(str(p) for p in parts)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_trader_prompt_states_constraint():
|
||||||
|
from tradingagents.agents.schemas import TraderAction, TraderProposal
|
||||||
|
|
||||||
|
captured = {}
|
||||||
|
llm = _capturing_llm(captured, TraderProposal(action=TraderAction.BUY, reasoning="x"))
|
||||||
|
create_trader(llm)({
|
||||||
|
"company_of_interest": "NVDA",
|
||||||
|
"investment_plan": "**Recommendation**: Buy",
|
||||||
|
})
|
||||||
|
assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_research_manager_prompt_states_constraint():
|
||||||
|
from tradingagents.agents.schemas import PortfolioRating, ResearchPlan
|
||||||
|
|
||||||
|
captured = {}
|
||||||
|
llm = _capturing_llm(
|
||||||
|
captured,
|
||||||
|
ResearchPlan(
|
||||||
|
recommendation=PortfolioRating.BUY, rationale="x", strategic_actions="y"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
create_research_manager(llm)({
|
||||||
|
"company_of_interest": "NVDA",
|
||||||
|
"investment_debate_state": {
|
||||||
|
"history": "h", "bull_history": "b", "bear_history": "r",
|
||||||
|
"current_response": "", "judge_decision": "", "count": 1,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_portfolio_manager_prompt_states_constraint():
|
||||||
|
from tradingagents.agents.schemas import PortfolioDecision, PortfolioRating
|
||||||
|
|
||||||
|
captured = {}
|
||||||
|
llm = _capturing_llm(
|
||||||
|
captured,
|
||||||
|
PortfolioDecision(
|
||||||
|
rating=PortfolioRating.HOLD,
|
||||||
|
executive_summary="x",
|
||||||
|
investment_thesis="y",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
risk = {
|
||||||
|
"history": "h", "aggressive_history": "a", "conservative_history": "c",
|
||||||
|
"neutral_history": "n", "current_aggressive_response": "",
|
||||||
|
"current_conservative_response": "", "current_neutral_response": "",
|
||||||
|
"latest_speaker": "Neutral", "count": 1,
|
||||||
|
}
|
||||||
|
create_portfolio_manager(llm)({
|
||||||
|
"company_of_interest": "NVDA",
|
||||||
|
"risk_debate_state": risk,
|
||||||
|
"investment_plan": "plan",
|
||||||
|
"trader_investment_plan": "trader plan",
|
||||||
|
})
|
||||||
|
assert NO_EXTERNAL_TOOLS in _prompt_text(captured["prompt"])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_sentiment_prompt_states_constraint(monkeypatch):
|
||||||
|
from tradingagents.agents.schemas import SentimentBand, SentimentReport
|
||||||
|
|
||||||
|
# Pre-fetched sources are stubbed so the prompt builds without network I/O.
|
||||||
|
monkeypatch.setattr(sentiment, "fetch_stocktwits_messages", lambda *a, **k: "st")
|
||||||
|
monkeypatch.setattr(sentiment, "fetch_reddit_posts", lambda *a, **k: "rd")
|
||||||
|
monkeypatch.setattr(sentiment.get_news, "func", lambda *a, **k: "news", raising=False)
|
||||||
|
|
||||||
|
captured = {}
|
||||||
|
llm = _capturing_llm(captured, SentimentReport(
|
||||||
|
overall_band=SentimentBand.BULLISH, overall_score=7.5,
|
||||||
|
confidence="high", narrative="n",
|
||||||
|
))
|
||||||
|
sentiment.create_sentiment_analyst(llm)({
|
||||||
|
"company_of_interest": "NVDA", "trade_date": "2026-01-15",
|
||||||
|
"asset_type": "stock", "messages": [],
|
||||||
|
})
|
||||||
|
text = _prompt_text(captured["prompt"])
|
||||||
|
assert NO_EXTERNAL_TOOLS in text
|
||||||
|
# This agent binds no tools, so tool-range wording must not reappear.
|
||||||
|
assert "tool-call date ranges" not in text
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_tool_using_analysts_keep_their_date_guidance():
|
||||||
|
# The analysts that really do call tools keep the wording that anchors their
|
||||||
|
# tool date ranges (#836) — this fix is scoped to no-tool agents.
|
||||||
|
import tradingagents.agents.analysts.market_analyst as market
|
||||||
|
import tradingagents.agents.analysts.news_analyst as news
|
||||||
|
for module in (market, news):
|
||||||
|
assert "tool-call date ranges" in inspect.getsource(module)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_constraint_text_is_unambiguous():
|
||||||
|
assert "do not call external tools" in NO_EXTERNAL_TOOLS.lower()
|
||||||
|
# No template braces: it is embedded in ChatPromptTemplate strings, where
|
||||||
|
# braces would be parsed as input variables.
|
||||||
|
assert "{" not in NO_EXTERNAL_TOOLS and "}" not in NO_EXTERNAL_TOOLS
|
||||||
@@ -36,6 +36,7 @@ from tradingagents.agents.utils.agent_utils import (
|
|||||||
get_news,
|
get_news,
|
||||||
)
|
)
|
||||||
from tradingagents.agents.utils.structured import (
|
from tradingagents.agents.utils.structured import (
|
||||||
|
NO_EXTERNAL_TOOLS,
|
||||||
bind_structured,
|
bind_structured,
|
||||||
invoke_structured_or_freetext,
|
invoke_structured_or_freetext,
|
||||||
)
|
)
|
||||||
@@ -86,7 +87,11 @@ def create_sentiment_analyst(llm):
|
|||||||
"You are a helpful AI assistant, collaborating with other assistants."
|
"You are a helpful AI assistant, collaborating with other assistants."
|
||||||
" If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable,"
|
" If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable,"
|
||||||
" prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop."
|
" prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop."
|
||||||
" Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}"
|
# No tool-calling here: the data is pre-fetched into the
|
||||||
|
# prompt, so tool-range wording would only invite a
|
||||||
|
# hallucinated tool call (#1130).
|
||||||
|
" Today's date is {current_date}; treat it as 'now' for all analysis. {instrument_context}"
|
||||||
|
" " + NO_EXTERNAL_TOOLS +
|
||||||
"\n{system_message}",
|
"\n{system_message}",
|
||||||
),
|
),
|
||||||
MessagesPlaceholder(variable_name="messages"),
|
MessagesPlaceholder(variable_name="messages"),
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from tradingagents.agents.utils.agent_utils import (
|
|||||||
get_language_instruction,
|
get_language_instruction,
|
||||||
)
|
)
|
||||||
from tradingagents.agents.utils.structured import (
|
from tradingagents.agents.utils.structured import (
|
||||||
|
NO_EXTERNAL_TOOLS,
|
||||||
bind_structured,
|
bind_structured,
|
||||||
invoke_structured_or_freetext,
|
invoke_structured_or_freetext,
|
||||||
)
|
)
|
||||||
@@ -61,7 +62,9 @@ def create_portfolio_manager(llm):
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
Be decisive and ground every conclusion in specific evidence from the analysts.{get_language_instruction()}"""
|
Be decisive and ground every conclusion in specific evidence from the analysts.
|
||||||
|
|
||||||
|
{NO_EXTERNAL_TOOLS}{get_language_instruction()}"""
|
||||||
|
|
||||||
final_trade_decision = invoke_structured_or_freetext(
|
final_trade_decision = invoke_structured_or_freetext(
|
||||||
structured_llm,
|
structured_llm,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from tradingagents.agents.utils.agent_utils import (
|
|||||||
get_language_instruction,
|
get_language_instruction,
|
||||||
)
|
)
|
||||||
from tradingagents.agents.utils.structured import (
|
from tradingagents.agents.utils.structured import (
|
||||||
|
NO_EXTERNAL_TOOLS,
|
||||||
bind_structured,
|
bind_structured,
|
||||||
invoke_structured_or_freetext,
|
invoke_structured_or_freetext,
|
||||||
)
|
)
|
||||||
@@ -40,7 +41,9 @@ Commit to a clear stance whenever the debate's strongest arguments warrant one;
|
|||||||
---
|
---
|
||||||
|
|
||||||
**Debate History:**
|
**Debate History:**
|
||||||
{history}""" + get_language_instruction()
|
{history}
|
||||||
|
|
||||||
|
{NO_EXTERNAL_TOOLS}""" + get_language_instruction()
|
||||||
|
|
||||||
investment_plan = invoke_structured_or_freetext(
|
investment_plan = invoke_structured_or_freetext(
|
||||||
structured_llm,
|
structured_llm,
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from tradingagents.agents.utils.agent_utils import (
|
|||||||
get_language_instruction,
|
get_language_instruction,
|
||||||
)
|
)
|
||||||
from tradingagents.agents.utils.structured import (
|
from tradingagents.agents.utils.structured import (
|
||||||
|
NO_EXTERNAL_TOOLS,
|
||||||
bind_structured,
|
bind_structured,
|
||||||
invoke_structured_or_freetext,
|
invoke_structured_or_freetext,
|
||||||
)
|
)
|
||||||
@@ -31,7 +32,8 @@ def create_trader(llm):
|
|||||||
"content": (
|
"content": (
|
||||||
"You are a trading agent analyzing market data to make investment decisions. "
|
"You are a trading agent analyzing market data to make investment decisions. "
|
||||||
"Based on your analysis, provide a specific recommendation to buy, sell, or hold. "
|
"Based on your analysis, provide a specific recommendation to buy, sell, or hold. "
|
||||||
"Anchor your reasoning in the analysts' reports and the research plan."
|
"Anchor your reasoning in the analysts' reports and the research plan. "
|
||||||
|
+ NO_EXTERNAL_TOOLS
|
||||||
+ get_language_instruction()
|
+ get_language_instruction()
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -28,6 +28,16 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
T = TypeVar("T", bound=BaseModel)
|
T = TypeVar("T", bound=BaseModel)
|
||||||
|
|
||||||
|
# Schema-only structured output binds exactly one tool (the schema itself), so a
|
||||||
|
# model that reaches for a search tool emits an unknown tool call and the whole
|
||||||
|
# structured attempt is discarded for a free-text retry. Agents on this path
|
||||||
|
# state the constraint explicitly rather than relying on the binding alone
|
||||||
|
# (#1130).
|
||||||
|
NO_EXTERNAL_TOOLS = (
|
||||||
|
"Use only the evidence provided in this prompt. Do not call external tools "
|
||||||
|
"or search the web; if something is missing, say so explicitly."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def bind_structured(llm: Any, schema: type[T], agent_name: str) -> Any | None:
|
def bind_structured(llm: Any, schema: type[T], agent_name: str) -> Any | None:
|
||||||
"""Return ``llm.with_structured_output(schema)`` or ``None`` if unsupported.
|
"""Return ``llm.with_structured_output(schema)`` or ``None`` if unsupported.
|
||||||
|
|||||||
Reference in New Issue
Block a user