From 539eae8fd6b26de41e8d77a67016baaf78dd6fbc Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Sun, 30 Aug 2026 06:18:59 +0000 Subject: [PATCH] fix(agents): stop debate openers from rebutting a nonexistent argument - the first speaker in each debate round received an empty opponent response, yet the prompt demanded a rebuttal, so models fabricated the other side - substitute an explicit opening marker when an opponent has not spoken, across all five debators (bull, bear, and the three risk analysts) #1176 --- tests/test_debate_opening.py | 111 ++++++++++++++++++ .../agents/researchers/bear_researcher.py | 5 +- .../agents/researchers/bull_researcher.py | 5 +- .../agents/risk_mgmt/aggressive_debator.py | 9 +- .../agents/risk_mgmt/conservative_debator.py | 9 +- .../agents/risk_mgmt/neutral_debator.py | 9 +- tradingagents/agents/utils/agent_utils.py | 14 +++ 7 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 tests/test_debate_opening.py diff --git a/tests/test_debate_opening.py b/tests/test_debate_opening.py new file mode 100644 index 000000000..a20a08b8a --- /dev/null +++ b/tests/test_debate_opening.py @@ -0,0 +1,111 @@ +"""The first speaker in each debate must not rebut a nonexistent argument (#1176). + +Each debate round's opening speaker receives an empty opponent response; the +prompt used to interpolate it into a "refute the opponent" instruction, so models +fabricated the other side's position. All five debators (bull, bear, and the +three risk analysts) now substitute an explicit opening marker when the opponent +has not spoken, and pass a real argument through unchanged. +""" +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from tradingagents.agents.researchers.bear_researcher import create_bear_researcher +from tradingagents.agents.researchers.bull_researcher import create_bull_researcher +from tradingagents.agents.risk_mgmt.aggressive_debator import create_aggressive_debator +from tradingagents.agents.risk_mgmt.conservative_debator import create_conservative_debator +from tradingagents.agents.risk_mgmt.neutral_debator import create_neutral_debator +from tradingagents.agents.utils.agent_utils import opponent_argument_or_opening + +_REPORTS = { + "company_of_interest": "AAPL", "asset_type": "stock", + "market_report": "m", "sentiment_report": "s", + "news_report": "n", "fundamentals_report": "f", +} + + +def _capturing_llm(captured: dict): + llm = MagicMock() + llm.invoke.side_effect = lambda prompt: ( + captured.__setitem__("prompt", prompt) or MagicMock(content="argument") + ) + return llm + + +def _investment_state(current_response): + return { + **_REPORTS, + "count": 0, + "investment_debate_state": { + "history": "", "bull_history": "", "bear_history": "", + "current_response": current_response, "count": 0, + }, + } + + +def _risk_state(**responses): + base = { + "current_aggressive_response": "", "current_conservative_response": "", + "current_neutral_response": "", "history": "", "aggressive_history": "", + "conservative_history": "", "neutral_history": "", "count": 0, + } + base.update(responses) + return {**_REPORTS, "trader_investment_plan": "plan", "risk_debate_state": base} + + +# --- shared helper ---------------------------------------------------------- + +@pytest.mark.unit +def test_helper_marks_empty_and_passes_through(): + assert "has not spoken yet" in opponent_argument_or_opening("", "bear analyst") + assert opponent_argument_or_opening(" real point ", "bear") == "real point" + + +# --- researchers ------------------------------------------------------------ + +@pytest.mark.unit +@pytest.mark.parametrize( + "factory,opponent", + [(create_bull_researcher, "bear"), (create_bear_researcher, "bull")], +) +def test_researcher_opening_has_no_phantom_opponent(factory, opponent): + captured = {} + factory(_capturing_llm(captured))(_investment_state("")) + assert "has not spoken yet" in captured["prompt"] + + +@pytest.mark.unit +def test_researcher_passes_real_opponent_argument(): + captured = {} + state = _investment_state("Bear Analyst: valuation is stretched") + create_bull_researcher(_capturing_llm(captured))(state) + assert "valuation is stretched" in captured["prompt"] + assert "has not spoken yet" not in captured["prompt"] + + +# --- risk debators ---------------------------------------------------------- + +@pytest.mark.unit +@pytest.mark.parametrize( + "factory", [create_aggressive_debator, create_conservative_debator, create_neutral_debator] +) +def test_risk_opening_has_no_phantom_opponent(factory): + captured = {} + factory(_capturing_llm(captured))(_risk_state()) + # Both opponent slots were empty -> two opening markers, no fabricated args. + assert captured["prompt"].count("has not spoken yet") == 2 + + +@pytest.mark.unit +def test_risk_passes_real_opponent_arguments(): + captured = {} + state = _risk_state( + current_conservative_response="Conservative Analyst: trim risk", + current_neutral_response="Neutral Analyst: hold steady", + ) + create_aggressive_debator(_capturing_llm(captured))(state) + assert "trim risk" in captured["prompt"] + assert "hold steady" in captured["prompt"] + assert "has not spoken yet" not in captured["prompt"] diff --git a/tradingagents/agents/researchers/bear_researcher.py b/tradingagents/agents/researchers/bear_researcher.py index 860ae4be5..9e3e2bb23 100644 --- a/tradingagents/agents/researchers/bear_researcher.py +++ b/tradingagents/agents/researchers/bear_researcher.py @@ -1,6 +1,7 @@ from tradingagents.agents.utils.agent_utils import ( get_instrument_context_from_state, get_language_instruction, + opponent_argument_or_opening, ) @@ -10,7 +11,9 @@ def create_bear_researcher(llm): history = investment_debate_state.get("history", "") bear_history = investment_debate_state.get("bear_history", "") - current_response = investment_debate_state.get("current_response", "") + current_response = opponent_argument_or_opening( + investment_debate_state.get("current_response", ""), "bull analyst" + ) market_research_report = state["market_report"] sentiment_report = state["sentiment_report"] news_report = state["news_report"] diff --git a/tradingagents/agents/researchers/bull_researcher.py b/tradingagents/agents/researchers/bull_researcher.py index 5f939ed72..6987ac81b 100644 --- a/tradingagents/agents/researchers/bull_researcher.py +++ b/tradingagents/agents/researchers/bull_researcher.py @@ -1,6 +1,7 @@ from tradingagents.agents.utils.agent_utils import ( get_instrument_context_from_state, get_language_instruction, + opponent_argument_or_opening, ) @@ -10,7 +11,9 @@ def create_bull_researcher(llm): history = investment_debate_state.get("history", "") bull_history = investment_debate_state.get("bull_history", "") - current_response = investment_debate_state.get("current_response", "") + current_response = opponent_argument_or_opening( + investment_debate_state.get("current_response", ""), "bear analyst" + ) market_research_report = state["market_report"] sentiment_report = state["sentiment_report"] news_report = state["news_report"] diff --git a/tradingagents/agents/risk_mgmt/aggressive_debator.py b/tradingagents/agents/risk_mgmt/aggressive_debator.py index 3fd0e124a..ff1efd879 100644 --- a/tradingagents/agents/risk_mgmt/aggressive_debator.py +++ b/tradingagents/agents/risk_mgmt/aggressive_debator.py @@ -1,6 +1,7 @@ from tradingagents.agents.utils.agent_utils import ( get_instrument_context_from_state, get_language_instruction, + opponent_argument_or_opening, ) @@ -10,8 +11,12 @@ def create_aggressive_debator(llm): history = risk_debate_state.get("history", "") aggressive_history = risk_debate_state.get("aggressive_history", "") - current_conservative_response = risk_debate_state.get("current_conservative_response", "") - current_neutral_response = risk_debate_state.get("current_neutral_response", "") + current_conservative_response = opponent_argument_or_opening( + risk_debate_state.get("current_conservative_response", ""), "conservative analyst" + ) + current_neutral_response = opponent_argument_or_opening( + risk_debate_state.get("current_neutral_response", ""), "neutral analyst" + ) market_research_report = state["market_report"] sentiment_report = state["sentiment_report"] diff --git a/tradingagents/agents/risk_mgmt/conservative_debator.py b/tradingagents/agents/risk_mgmt/conservative_debator.py index b84add0c3..77d7d064b 100644 --- a/tradingagents/agents/risk_mgmt/conservative_debator.py +++ b/tradingagents/agents/risk_mgmt/conservative_debator.py @@ -1,6 +1,7 @@ from tradingagents.agents.utils.agent_utils import ( get_instrument_context_from_state, get_language_instruction, + opponent_argument_or_opening, ) @@ -10,8 +11,12 @@ def create_conservative_debator(llm): history = risk_debate_state.get("history", "") conservative_history = risk_debate_state.get("conservative_history", "") - current_aggressive_response = risk_debate_state.get("current_aggressive_response", "") - current_neutral_response = risk_debate_state.get("current_neutral_response", "") + current_aggressive_response = opponent_argument_or_opening( + risk_debate_state.get("current_aggressive_response", ""), "aggressive analyst" + ) + current_neutral_response = opponent_argument_or_opening( + risk_debate_state.get("current_neutral_response", ""), "neutral analyst" + ) market_research_report = state["market_report"] sentiment_report = state["sentiment_report"] diff --git a/tradingagents/agents/risk_mgmt/neutral_debator.py b/tradingagents/agents/risk_mgmt/neutral_debator.py index b28ade25b..746c8834e 100644 --- a/tradingagents/agents/risk_mgmt/neutral_debator.py +++ b/tradingagents/agents/risk_mgmt/neutral_debator.py @@ -1,6 +1,7 @@ from tradingagents.agents.utils.agent_utils import ( get_instrument_context_from_state, get_language_instruction, + opponent_argument_or_opening, ) @@ -10,8 +11,12 @@ def create_neutral_debator(llm): history = risk_debate_state.get("history", "") neutral_history = risk_debate_state.get("neutral_history", "") - current_aggressive_response = risk_debate_state.get("current_aggressive_response", "") - current_conservative_response = risk_debate_state.get("current_conservative_response", "") + current_aggressive_response = opponent_argument_or_opening( + risk_debate_state.get("current_aggressive_response", ""), "aggressive analyst" + ) + current_conservative_response = opponent_argument_or_opening( + risk_debate_state.get("current_conservative_response", ""), "conservative analyst" + ) market_research_report = state["market_report"] sentiment_report = state["sentiment_report"] diff --git a/tradingagents/agents/utils/agent_utils.py b/tradingagents/agents/utils/agent_utils.py index 16f45c178..cb1d3a3d2 100644 --- a/tradingagents/agents/utils/agent_utils.py +++ b/tradingagents/agents/utils/agent_utils.py @@ -65,6 +65,20 @@ def get_language_instruction() -> str: return f" Write your entire response in {lang}." +def opponent_argument_or_opening(text: str, opponent: str) -> str: + """Opponent's latest argument, or an explicit opening marker when empty. + + The first speaker in each debate round receives an empty opponent response; + interpolating it into a "refute the opponent" prompt makes the model + fabricate the other side's position. Returning a clear "has not spoken yet" + marker instead lets it open with its own case (#1176). + """ + text = (text or "").strip() + if text: + return text + return f"(The {opponent} has not spoken yet — open the debate with your own case.)" + + def _clean_identity_value(value: Any) -> str | None: """Return a trimmed string, or None for empty / placeholder-ish values.""" if not isinstance(value, str):