mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-19 11:15:24 +03:00
fix(agents): give every prompt honest inputs
- the fundamentals brief reached the model as a Python tuple - analysts no longer emit a trade call that nothing reads - a report that was not produced says so instead of appearing as a blank section
This commit is contained in:
87
tests/test_prompt_integrity.py
Normal file
87
tests/test_prompt_integrity.py
Normal file
@@ -0,0 +1,87 @@
|
||||
"""What the agents are actually told.
|
||||
|
||||
Three problems the audit found: one analyst's brief reached the model as a Python
|
||||
tuple, every analyst was asked for a trade call that nothing reads, and a report
|
||||
that was never produced was presented as an empty labelled section, which invites
|
||||
the next agent to fill it in from nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
ANALYSTS = ["market_analyst", "sentiment_analyst", "news_analyst", "fundamentals_analyst"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("name", ANALYSTS)
|
||||
def test_an_analyst_brief_is_text_not_a_python_object(name):
|
||||
"""A trailing comma made one brief a tuple, so the model was handed its repr
|
||||
(quotes, parens and all) instead of the instruction."""
|
||||
import ast
|
||||
import inspect
|
||||
|
||||
mod = importlib.import_module(f"tradingagents.agents.analysts.{name}")
|
||||
tree = ast.parse(inspect.getsource(mod))
|
||||
briefs = [node.value for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Assign)
|
||||
and getattr(node.targets[0], "id", "") == "system_message"]
|
||||
assert briefs, f"{name} has no system_message"
|
||||
for brief in briefs:
|
||||
assert not isinstance(brief, ast.Tuple), "the brief is a tuple, not text"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("name", ANALYSTS)
|
||||
def test_an_analyst_is_not_asked_for_a_trade_call_nothing_reads(name):
|
||||
"""The stop signal is never consumed, and asking for it makes an analyst
|
||||
open with a direction that then travels as evidence."""
|
||||
import inspect
|
||||
|
||||
mod = importlib.import_module(f"tradingagents.agents.analysts.{name}")
|
||||
assert "FINAL TRANSACTION PROPOSAL" not in inspect.getsource(mod)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("module, factory", [
|
||||
("tradingagents.agents.researchers.bull_researcher", "create_bull_researcher"),
|
||||
("tradingagents.agents.researchers.bear_researcher", "create_bear_researcher"),
|
||||
("tradingagents.agents.risk_mgmt.aggressive_debator", "create_aggressive_debator"),
|
||||
("tradingagents.agents.risk_mgmt.conservative_debator", "create_conservative_debator"),
|
||||
("tradingagents.agents.risk_mgmt.neutral_debator", "create_neutral_debator"),
|
||||
])
|
||||
def test_a_report_that_was_never_produced_says_so(module, factory):
|
||||
"""`--analysts market` leaves three reports empty; presenting them as blank
|
||||
sections invites the model to invent the contents."""
|
||||
from langchain_core.messages import AIMessage
|
||||
|
||||
mod = importlib.import_module(module)
|
||||
seen = []
|
||||
|
||||
class _LLM:
|
||||
def invoke(self, prompt, *a, **k):
|
||||
seen.append(prompt if isinstance(prompt, str) else str(prompt))
|
||||
return AIMessage("argument")
|
||||
|
||||
def with_structured_output(self, *a, **k):
|
||||
raise NotImplementedError
|
||||
|
||||
state = {
|
||||
"company_of_interest": "NVDA", "trade_date": "2026-08-14", "asset_type": "stock",
|
||||
"instrument_context": "", "portfolio_context": "", "past_context": "",
|
||||
"market_report": "RSI 61, price 178.", "sentiment_report": "", "news_report": "",
|
||||
"fundamentals_report": "", "investment_plan": "P", "trader_investment_plan": "T",
|
||||
"investment_debate_state": {"bull_history": "", "bear_history": "", "history": "",
|
||||
"current_response": "", "judge_decision": "", "count": 0},
|
||||
"risk_debate_state": {"history": "", "latest_speaker": "", "count": 0,
|
||||
"aggressive_history": "", "conservative_history": "", "neutral_history": "",
|
||||
"current_aggressive_response": "", "current_conservative_response": "",
|
||||
"current_neutral_response": "", "judge_decision": ""},
|
||||
}
|
||||
getattr(mod, factory)(_LLM())(state)
|
||||
|
||||
prompt = " ".join(seen)
|
||||
assert "not part of this run" in prompt or "not available" in prompt, prompt[:400]
|
||||
assert "RSI 61" in prompt # the report that does exist is still passed through
|
||||
@@ -26,7 +26,7 @@ def create_fundamentals_analyst(llm):
|
||||
"You are a researcher tasked with analyzing fundamental information over the past week about a company. Please write a comprehensive report of the company's fundamental information such as financial documents, company profile, basic company financials, and company financial history to gain a full view of the company's fundamental information to inform traders. Make sure to include as much detail as possible. Provide specific, actionable insights with supporting evidence to help traders make informed decisions."
|
||||
+ " Make sure to append a Markdown table at the end of the report to organize key points in the report, organized and easy to read."
|
||||
+ " Use the available tools: `get_fundamentals` for comprehensive company analysis, `get_balance_sheet`, `get_cashflow`, and `get_income_statement` for specific financial statements."
|
||||
+ get_language_instruction(),
|
||||
+ get_language_instruction()
|
||||
)
|
||||
|
||||
prompt = ChatPromptTemplate.from_messages(
|
||||
@@ -37,8 +37,7 @@ def create_fundamentals_analyst(llm):
|
||||
" Use the provided tools to progress towards answering the question."
|
||||
" If you are unable to fully answer, that's OK; another assistant with different tools"
|
||||
" will help where you left off. Execute what you can to make progress."
|
||||
" If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable,"
|
||||
" prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop."
|
||||
" Report what your tools support; another agent decides the trade."
|
||||
" You have access to the following tools: {tool_names}."
|
||||
" Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}\n"
|
||||
"{system_message}",
|
||||
|
||||
@@ -63,8 +63,7 @@ Write a very detailed and nuanced report of the trends you observe. Provide spec
|
||||
" Use the provided tools to progress towards answering the question."
|
||||
" If you are unable to fully answer, that's OK; another assistant with different tools"
|
||||
" will help where you left off. Execute what you can to make progress."
|
||||
" If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable,"
|
||||
" prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop."
|
||||
" Report what your tools support; another agent decides the trade."
|
||||
" You have access to the following tools: {tool_names}."
|
||||
" Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}\n"
|
||||
"{system_message}",
|
||||
|
||||
@@ -38,8 +38,7 @@ def create_news_analyst(llm):
|
||||
" Use the provided tools to progress towards answering the question."
|
||||
" If you are unable to fully answer, that's OK; another assistant with different tools"
|
||||
" will help where you left off. Execute what you can to make progress."
|
||||
" If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable,"
|
||||
" prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop."
|
||||
" Report what your tools support; another agent decides the trade."
|
||||
" You have access to the following tools: {tool_names}."
|
||||
" Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}\n"
|
||||
"{system_message}",
|
||||
|
||||
@@ -93,8 +93,7 @@ def create_sentiment_analyst(llm):
|
||||
(
|
||||
"system",
|
||||
"You are a helpful AI assistant, collaborating with other assistants."
|
||||
" If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable,"
|
||||
" prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop."
|
||||
" Report what your tools support; another agent decides the trade."
|
||||
# No tool-calling here: the data is pre-fetched into the
|
||||
# prompt, so tool-range wording would only invite a
|
||||
# hallucinated tool call (#1130).
|
||||
|
||||
@@ -2,6 +2,7 @@ from tradingagents.agents.utils.agent_utils import (
|
||||
get_instrument_context_from_state,
|
||||
get_language_instruction,
|
||||
opponent_argument_or_opening,
|
||||
report_or_absent,
|
||||
)
|
||||
|
||||
|
||||
@@ -14,10 +15,10 @@ def create_bear_researcher(llm):
|
||||
current_response = opponent_argument_or_opening(
|
||||
investment_debate_state.get("current_response", ""), "bull analyst"
|
||||
)
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
news_report = state["news_report"]
|
||||
fundamentals_report = state["fundamentals_report"]
|
||||
market_research_report = report_or_absent(state["market_report"], "market")
|
||||
sentiment_report = report_or_absent(state["sentiment_report"], "sentiment")
|
||||
news_report = report_or_absent(state["news_report"], "news")
|
||||
fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals")
|
||||
instrument_context = get_instrument_context_from_state(state)
|
||||
asset_type = state.get("asset_type", "stock")
|
||||
target_label = "stock" if asset_type == "stock" else "asset"
|
||||
|
||||
@@ -2,6 +2,7 @@ from tradingagents.agents.utils.agent_utils import (
|
||||
get_instrument_context_from_state,
|
||||
get_language_instruction,
|
||||
opponent_argument_or_opening,
|
||||
report_or_absent,
|
||||
)
|
||||
|
||||
|
||||
@@ -14,10 +15,10 @@ def create_bull_researcher(llm):
|
||||
current_response = opponent_argument_or_opening(
|
||||
investment_debate_state.get("current_response", ""), "bear analyst"
|
||||
)
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
news_report = state["news_report"]
|
||||
fundamentals_report = state["fundamentals_report"]
|
||||
market_research_report = report_or_absent(state["market_report"], "market")
|
||||
sentiment_report = report_or_absent(state["sentiment_report"], "sentiment")
|
||||
news_report = report_or_absent(state["news_report"], "news")
|
||||
fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals")
|
||||
instrument_context = get_instrument_context_from_state(state)
|
||||
asset_type = state.get("asset_type", "stock")
|
||||
target_label = "stock" if asset_type == "stock" else "asset"
|
||||
|
||||
@@ -3,6 +3,7 @@ from tradingagents.agents.utils.agent_utils import (
|
||||
get_language_instruction,
|
||||
get_portfolio_context_from_state,
|
||||
opponent_argument_or_opening,
|
||||
report_or_absent,
|
||||
)
|
||||
|
||||
|
||||
@@ -19,10 +20,10 @@ def create_aggressive_debator(llm):
|
||||
risk_debate_state.get("current_neutral_response", ""), "neutral analyst"
|
||||
)
|
||||
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
news_report = state["news_report"]
|
||||
fundamentals_report = state["fundamentals_report"]
|
||||
market_research_report = report_or_absent(state["market_report"], "market")
|
||||
sentiment_report = report_or_absent(state["sentiment_report"], "sentiment")
|
||||
news_report = report_or_absent(state["news_report"], "news")
|
||||
fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals")
|
||||
instrument_context = get_instrument_context_from_state(state)
|
||||
portfolio_context = get_portfolio_context_from_state(state)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from tradingagents.agents.utils.agent_utils import (
|
||||
get_language_instruction,
|
||||
get_portfolio_context_from_state,
|
||||
opponent_argument_or_opening,
|
||||
report_or_absent,
|
||||
)
|
||||
|
||||
|
||||
@@ -19,10 +20,10 @@ def create_conservative_debator(llm):
|
||||
risk_debate_state.get("current_neutral_response", ""), "neutral analyst"
|
||||
)
|
||||
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
news_report = state["news_report"]
|
||||
fundamentals_report = state["fundamentals_report"]
|
||||
market_research_report = report_or_absent(state["market_report"], "market")
|
||||
sentiment_report = report_or_absent(state["sentiment_report"], "sentiment")
|
||||
news_report = report_or_absent(state["news_report"], "news")
|
||||
fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals")
|
||||
instrument_context = get_instrument_context_from_state(state)
|
||||
portfolio_context = get_portfolio_context_from_state(state)
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from tradingagents.agents.utils.agent_utils import (
|
||||
get_language_instruction,
|
||||
get_portfolio_context_from_state,
|
||||
opponent_argument_or_opening,
|
||||
report_or_absent,
|
||||
)
|
||||
|
||||
|
||||
@@ -19,10 +20,10 @@ def create_neutral_debator(llm):
|
||||
risk_debate_state.get("current_conservative_response", ""), "conservative analyst"
|
||||
)
|
||||
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
news_report = state["news_report"]
|
||||
fundamentals_report = state["fundamentals_report"]
|
||||
market_research_report = report_or_absent(state["market_report"], "market")
|
||||
sentiment_report = report_or_absent(state["sentiment_report"], "sentiment")
|
||||
news_report = report_or_absent(state["news_report"], "news")
|
||||
fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals")
|
||||
instrument_context = get_instrument_context_from_state(state)
|
||||
portfolio_context = get_portfolio_context_from_state(state)
|
||||
|
||||
|
||||
@@ -201,6 +201,20 @@ def get_instrument_context_from_state(state: Mapping[str, Any]) -> str:
|
||||
)
|
||||
|
||||
|
||||
def report_or_absent(text: str, source: str) -> str:
|
||||
"""An analyst's report, or a marker saying it was never produced.
|
||||
|
||||
A report is empty when its analyst was not selected, refused, or returned
|
||||
nothing. Interpolating that into a labelled section presents an absence as a
|
||||
blank finding, and the reading agent fills it in from nothing, the same way
|
||||
an empty opponent argument used to invite an invented rebuttal (#1176).
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if text:
|
||||
return text
|
||||
return f"(No {source} report in this run: it is not available, not an empty finding.)"
|
||||
|
||||
|
||||
def get_portfolio_context_from_state(state: Mapping[str, Any]) -> str:
|
||||
"""Return the caller's portfolio block, or a notice that none was given.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user