diff --git a/tests/test_prompt_integrity.py b/tests/test_prompt_integrity.py new file mode 100644 index 000000000..e26e72185 --- /dev/null +++ b/tests/test_prompt_integrity.py @@ -0,0 +1,87 @@ +"""What the agents are actually told. + +Three problems the audit found: one analyst's brief reached the model as a Python +tuple, every analyst was asked for a trade call that nothing reads, and a report +that was never produced was presented as an empty labelled section, which invites +the next agent to fill it in from nothing. +""" + +from __future__ import annotations + +import importlib + +import pytest + +ANALYSTS = ["market_analyst", "sentiment_analyst", "news_analyst", "fundamentals_analyst"] + + +@pytest.mark.unit +@pytest.mark.parametrize("name", ANALYSTS) +def test_an_analyst_brief_is_text_not_a_python_object(name): + """A trailing comma made one brief a tuple, so the model was handed its repr + (quotes, parens and all) instead of the instruction.""" + import ast + import inspect + + mod = importlib.import_module(f"tradingagents.agents.analysts.{name}") + tree = ast.parse(inspect.getsource(mod)) + briefs = [node.value for node in ast.walk(tree) + if isinstance(node, ast.Assign) + and getattr(node.targets[0], "id", "") == "system_message"] + assert briefs, f"{name} has no system_message" + for brief in briefs: + assert not isinstance(brief, ast.Tuple), "the brief is a tuple, not text" + + +@pytest.mark.unit +@pytest.mark.parametrize("name", ANALYSTS) +def test_an_analyst_is_not_asked_for_a_trade_call_nothing_reads(name): + """The stop signal is never consumed, and asking for it makes an analyst + open with a direction that then travels as evidence.""" + import inspect + + mod = importlib.import_module(f"tradingagents.agents.analysts.{name}") + assert "FINAL TRANSACTION PROPOSAL" not in inspect.getsource(mod) + + +@pytest.mark.unit +@pytest.mark.parametrize("module, factory", [ + ("tradingagents.agents.researchers.bull_researcher", "create_bull_researcher"), + ("tradingagents.agents.researchers.bear_researcher", "create_bear_researcher"), + ("tradingagents.agents.risk_mgmt.aggressive_debator", "create_aggressive_debator"), + ("tradingagents.agents.risk_mgmt.conservative_debator", "create_conservative_debator"), + ("tradingagents.agents.risk_mgmt.neutral_debator", "create_neutral_debator"), +]) +def test_a_report_that_was_never_produced_says_so(module, factory): + """`--analysts market` leaves three reports empty; presenting them as blank + sections invites the model to invent the contents.""" + from langchain_core.messages import AIMessage + + mod = importlib.import_module(module) + seen = [] + + class _LLM: + def invoke(self, prompt, *a, **k): + seen.append(prompt if isinstance(prompt, str) else str(prompt)) + return AIMessage("argument") + + def with_structured_output(self, *a, **k): + raise NotImplementedError + + state = { + "company_of_interest": "NVDA", "trade_date": "2026-08-14", "asset_type": "stock", + "instrument_context": "", "portfolio_context": "", "past_context": "", + "market_report": "RSI 61, price 178.", "sentiment_report": "", "news_report": "", + "fundamentals_report": "", "investment_plan": "P", "trader_investment_plan": "T", + "investment_debate_state": {"bull_history": "", "bear_history": "", "history": "", + "current_response": "", "judge_decision": "", "count": 0}, + "risk_debate_state": {"history": "", "latest_speaker": "", "count": 0, + "aggressive_history": "", "conservative_history": "", "neutral_history": "", + "current_aggressive_response": "", "current_conservative_response": "", + "current_neutral_response": "", "judge_decision": ""}, + } + getattr(mod, factory)(_LLM())(state) + + prompt = " ".join(seen) + assert "not part of this run" in prompt or "not available" in prompt, prompt[:400] + assert "RSI 61" in prompt # the report that does exist is still passed through diff --git a/tradingagents/agents/analysts/fundamentals_analyst.py b/tradingagents/agents/analysts/fundamentals_analyst.py index 76e6bed4c..7b2b0ebf7 100644 --- a/tradingagents/agents/analysts/fundamentals_analyst.py +++ b/tradingagents/agents/analysts/fundamentals_analyst.py @@ -26,7 +26,7 @@ def create_fundamentals_analyst(llm): "You are a researcher tasked with analyzing fundamental information over the past week about a company. Please write a comprehensive report of the company's fundamental information such as financial documents, company profile, basic company financials, and company financial history to gain a full view of the company's fundamental information to inform traders. Make sure to include as much detail as possible. Provide specific, actionable insights with supporting evidence to help traders make informed decisions." + " Make sure to append a Markdown table at the end of the report to organize key points in the report, organized and easy to read." + " Use the available tools: `get_fundamentals` for comprehensive company analysis, `get_balance_sheet`, `get_cashflow`, and `get_income_statement` for specific financial statements." - + get_language_instruction(), + + get_language_instruction() ) prompt = ChatPromptTemplate.from_messages( @@ -37,8 +37,7 @@ def create_fundamentals_analyst(llm): " Use the provided tools to progress towards answering the question." " If you are unable to fully answer, that's OK; another assistant with different tools" " will help where you left off. Execute what you can to make progress." - " If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable," - " prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop." + " Report what your tools support; another agent decides the trade." " You have access to the following tools: {tool_names}." " Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}\n" "{system_message}", diff --git a/tradingagents/agents/analysts/market_analyst.py b/tradingagents/agents/analysts/market_analyst.py index a8f1ecb8a..dd41c03db 100644 --- a/tradingagents/agents/analysts/market_analyst.py +++ b/tradingagents/agents/analysts/market_analyst.py @@ -63,8 +63,7 @@ Write a very detailed and nuanced report of the trends you observe. Provide spec " Use the provided tools to progress towards answering the question." " If you are unable to fully answer, that's OK; another assistant with different tools" " will help where you left off. Execute what you can to make progress." - " If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable," - " prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop." + " Report what your tools support; another agent decides the trade." " You have access to the following tools: {tool_names}." " Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}\n" "{system_message}", diff --git a/tradingagents/agents/analysts/news_analyst.py b/tradingagents/agents/analysts/news_analyst.py index c2fe20c55..6c7454811 100644 --- a/tradingagents/agents/analysts/news_analyst.py +++ b/tradingagents/agents/analysts/news_analyst.py @@ -38,8 +38,7 @@ def create_news_analyst(llm): " Use the provided tools to progress towards answering the question." " If you are unable to fully answer, that's OK; another assistant with different tools" " will help where you left off. Execute what you can to make progress." - " If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable," - " prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop." + " Report what your tools support; another agent decides the trade." " You have access to the following tools: {tool_names}." " Today's date is {current_date}; treat it as 'now' for all analysis and tool-call date ranges. {instrument_context}\n" "{system_message}", diff --git a/tradingagents/agents/analysts/sentiment_analyst.py b/tradingagents/agents/analysts/sentiment_analyst.py index 351bb996b..1051746ac 100644 --- a/tradingagents/agents/analysts/sentiment_analyst.py +++ b/tradingagents/agents/analysts/sentiment_analyst.py @@ -93,8 +93,7 @@ def create_sentiment_analyst(llm): ( "system", "You are a helpful AI assistant, collaborating with other assistants." - " If you or any other assistant has the FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** or deliverable," - " prefix your response with FINAL TRANSACTION PROPOSAL: **BUY/HOLD/SELL** so the team knows to stop." + " Report what your tools support; another agent decides the trade." # No tool-calling here: the data is pre-fetched into the # prompt, so tool-range wording would only invite a # hallucinated tool call (#1130). diff --git a/tradingagents/agents/researchers/bear_researcher.py b/tradingagents/agents/researchers/bear_researcher.py index 9e3e2bb23..dddbefa52 100644 --- a/tradingagents/agents/researchers/bear_researcher.py +++ b/tradingagents/agents/researchers/bear_researcher.py @@ -2,6 +2,7 @@ from tradingagents.agents.utils.agent_utils import ( get_instrument_context_from_state, get_language_instruction, opponent_argument_or_opening, + report_or_absent, ) @@ -14,10 +15,10 @@ def create_bear_researcher(llm): current_response = opponent_argument_or_opening( investment_debate_state.get("current_response", ""), "bull analyst" ) - market_research_report = state["market_report"] - sentiment_report = state["sentiment_report"] - news_report = state["news_report"] - fundamentals_report = state["fundamentals_report"] + market_research_report = report_or_absent(state["market_report"], "market") + sentiment_report = report_or_absent(state["sentiment_report"], "sentiment") + news_report = report_or_absent(state["news_report"], "news") + fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals") instrument_context = get_instrument_context_from_state(state) asset_type = state.get("asset_type", "stock") target_label = "stock" if asset_type == "stock" else "asset" diff --git a/tradingagents/agents/researchers/bull_researcher.py b/tradingagents/agents/researchers/bull_researcher.py index 6987ac81b..28af77ea2 100644 --- a/tradingagents/agents/researchers/bull_researcher.py +++ b/tradingagents/agents/researchers/bull_researcher.py @@ -2,6 +2,7 @@ from tradingagents.agents.utils.agent_utils import ( get_instrument_context_from_state, get_language_instruction, opponent_argument_or_opening, + report_or_absent, ) @@ -14,10 +15,10 @@ def create_bull_researcher(llm): current_response = opponent_argument_or_opening( investment_debate_state.get("current_response", ""), "bear analyst" ) - market_research_report = state["market_report"] - sentiment_report = state["sentiment_report"] - news_report = state["news_report"] - fundamentals_report = state["fundamentals_report"] + market_research_report = report_or_absent(state["market_report"], "market") + sentiment_report = report_or_absent(state["sentiment_report"], "sentiment") + news_report = report_or_absent(state["news_report"], "news") + fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals") instrument_context = get_instrument_context_from_state(state) asset_type = state.get("asset_type", "stock") target_label = "stock" if asset_type == "stock" else "asset" diff --git a/tradingagents/agents/risk_mgmt/aggressive_debator.py b/tradingagents/agents/risk_mgmt/aggressive_debator.py index d678f9cef..c0555e0b6 100644 --- a/tradingagents/agents/risk_mgmt/aggressive_debator.py +++ b/tradingagents/agents/risk_mgmt/aggressive_debator.py @@ -3,6 +3,7 @@ from tradingagents.agents.utils.agent_utils import ( get_language_instruction, get_portfolio_context_from_state, opponent_argument_or_opening, + report_or_absent, ) @@ -19,10 +20,10 @@ def create_aggressive_debator(llm): risk_debate_state.get("current_neutral_response", ""), "neutral analyst" ) - market_research_report = state["market_report"] - sentiment_report = state["sentiment_report"] - news_report = state["news_report"] - fundamentals_report = state["fundamentals_report"] + market_research_report = report_or_absent(state["market_report"], "market") + sentiment_report = report_or_absent(state["sentiment_report"], "sentiment") + news_report = report_or_absent(state["news_report"], "news") + fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals") instrument_context = get_instrument_context_from_state(state) portfolio_context = get_portfolio_context_from_state(state) diff --git a/tradingagents/agents/risk_mgmt/conservative_debator.py b/tradingagents/agents/risk_mgmt/conservative_debator.py index 0ecf509d4..0a9d67541 100644 --- a/tradingagents/agents/risk_mgmt/conservative_debator.py +++ b/tradingagents/agents/risk_mgmt/conservative_debator.py @@ -3,6 +3,7 @@ from tradingagents.agents.utils.agent_utils import ( get_language_instruction, get_portfolio_context_from_state, opponent_argument_or_opening, + report_or_absent, ) @@ -19,10 +20,10 @@ def create_conservative_debator(llm): risk_debate_state.get("current_neutral_response", ""), "neutral analyst" ) - market_research_report = state["market_report"] - sentiment_report = state["sentiment_report"] - news_report = state["news_report"] - fundamentals_report = state["fundamentals_report"] + market_research_report = report_or_absent(state["market_report"], "market") + sentiment_report = report_or_absent(state["sentiment_report"], "sentiment") + news_report = report_or_absent(state["news_report"], "news") + fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals") instrument_context = get_instrument_context_from_state(state) portfolio_context = get_portfolio_context_from_state(state) diff --git a/tradingagents/agents/risk_mgmt/neutral_debator.py b/tradingagents/agents/risk_mgmt/neutral_debator.py index b3dcbb72c..661b70ab4 100644 --- a/tradingagents/agents/risk_mgmt/neutral_debator.py +++ b/tradingagents/agents/risk_mgmt/neutral_debator.py @@ -3,6 +3,7 @@ from tradingagents.agents.utils.agent_utils import ( get_language_instruction, get_portfolio_context_from_state, opponent_argument_or_opening, + report_or_absent, ) @@ -19,10 +20,10 @@ def create_neutral_debator(llm): risk_debate_state.get("current_conservative_response", ""), "conservative analyst" ) - market_research_report = state["market_report"] - sentiment_report = state["sentiment_report"] - news_report = state["news_report"] - fundamentals_report = state["fundamentals_report"] + market_research_report = report_or_absent(state["market_report"], "market") + sentiment_report = report_or_absent(state["sentiment_report"], "sentiment") + news_report = report_or_absent(state["news_report"], "news") + fundamentals_report = report_or_absent(state["fundamentals_report"], "fundamentals") instrument_context = get_instrument_context_from_state(state) portfolio_context = get_portfolio_context_from_state(state) diff --git a/tradingagents/agents/utils/agent_utils.py b/tradingagents/agents/utils/agent_utils.py index 39b6c3fb1..68d609534 100644 --- a/tradingagents/agents/utils/agent_utils.py +++ b/tradingagents/agents/utils/agent_utils.py @@ -201,6 +201,20 @@ def get_instrument_context_from_state(state: Mapping[str, Any]) -> str: ) +def report_or_absent(text: str, source: str) -> str: + """An analyst's report, or a marker saying it was never produced. + + A report is empty when its analyst was not selected, refused, or returned + nothing. Interpolating that into a labelled section presents an absence as a + blank finding, and the reading agent fills it in from nothing, the same way + an empty opponent argument used to invite an invented rebuttal (#1176). + """ + text = (text or "").strip() + if text: + return text + return f"(No {source} report in this run: it is not available, not an empty finding.)" + + def get_portfolio_context_from_state(state: Mapping[str, Any]) -> str: """Return the caller's portfolio block, or a notice that none was given.