fix(agents): stop debate openers from rebutting a nonexistent argument

- the first speaker in each debate round received an empty opponent response,
  yet the prompt demanded a rebuttal, so models fabricated the other side
- substitute an explicit opening marker when an opponent has not spoken, across
  all five debators (bull, bear, and the three risk analysts) #1176
This commit is contained in:
Yijia-Xiao
2026-08-30 06:18:59 +00:00
parent 9b98f09613
commit 539eae8fd6
7 changed files with 154 additions and 8 deletions

View File

@@ -0,0 +1,111 @@
"""The first speaker in each debate must not rebut a nonexistent argument (#1176).
Each debate round's opening speaker receives an empty opponent response; the
prompt used to interpolate it into a "refute the opponent" instruction, so models
fabricated the other side's position. All five debators (bull, bear, and the
three risk analysts) now substitute an explicit opening marker when the opponent
has not spoken, and pass a real argument through unchanged.
"""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
from tradingagents.agents.researchers.bear_researcher import create_bear_researcher
from tradingagents.agents.researchers.bull_researcher import create_bull_researcher
from tradingagents.agents.risk_mgmt.aggressive_debator import create_aggressive_debator
from tradingagents.agents.risk_mgmt.conservative_debator import create_conservative_debator
from tradingagents.agents.risk_mgmt.neutral_debator import create_neutral_debator
from tradingagents.agents.utils.agent_utils import opponent_argument_or_opening
_REPORTS = {
"company_of_interest": "AAPL", "asset_type": "stock",
"market_report": "m", "sentiment_report": "s",
"news_report": "n", "fundamentals_report": "f",
}
def _capturing_llm(captured: dict):
llm = MagicMock()
llm.invoke.side_effect = lambda prompt: (
captured.__setitem__("prompt", prompt) or MagicMock(content="argument")
)
return llm
def _investment_state(current_response):
return {
**_REPORTS,
"count": 0,
"investment_debate_state": {
"history": "", "bull_history": "", "bear_history": "",
"current_response": current_response, "count": 0,
},
}
def _risk_state(**responses):
base = {
"current_aggressive_response": "", "current_conservative_response": "",
"current_neutral_response": "", "history": "", "aggressive_history": "",
"conservative_history": "", "neutral_history": "", "count": 0,
}
base.update(responses)
return {**_REPORTS, "trader_investment_plan": "plan", "risk_debate_state": base}
# --- shared helper ----------------------------------------------------------
@pytest.mark.unit
def test_helper_marks_empty_and_passes_through():
assert "has not spoken yet" in opponent_argument_or_opening("", "bear analyst")
assert opponent_argument_or_opening(" real point ", "bear") == "real point"
# --- researchers ------------------------------------------------------------
@pytest.mark.unit
@pytest.mark.parametrize(
"factory,opponent",
[(create_bull_researcher, "bear"), (create_bear_researcher, "bull")],
)
def test_researcher_opening_has_no_phantom_opponent(factory, opponent):
captured = {}
factory(_capturing_llm(captured))(_investment_state(""))
assert "has not spoken yet" in captured["prompt"]
@pytest.mark.unit
def test_researcher_passes_real_opponent_argument():
captured = {}
state = _investment_state("Bear Analyst: valuation is stretched")
create_bull_researcher(_capturing_llm(captured))(state)
assert "valuation is stretched" in captured["prompt"]
assert "has not spoken yet" not in captured["prompt"]
# --- risk debators ----------------------------------------------------------
@pytest.mark.unit
@pytest.mark.parametrize(
"factory", [create_aggressive_debator, create_conservative_debator, create_neutral_debator]
)
def test_risk_opening_has_no_phantom_opponent(factory):
captured = {}
factory(_capturing_llm(captured))(_risk_state())
# Both opponent slots were empty -> two opening markers, no fabricated args.
assert captured["prompt"].count("has not spoken yet") == 2
@pytest.mark.unit
def test_risk_passes_real_opponent_arguments():
captured = {}
state = _risk_state(
current_conservative_response="Conservative Analyst: trim risk",
current_neutral_response="Neutral Analyst: hold steady",
)
create_aggressive_debator(_capturing_llm(captured))(state)
assert "trim risk" in captured["prompt"]
assert "hold steady" in captured["prompt"]
assert "has not spoken yet" not in captured["prompt"]

View File

@@ -1,6 +1,7 @@
from tradingagents.agents.utils.agent_utils import (
get_instrument_context_from_state,
get_language_instruction,
opponent_argument_or_opening,
)
@@ -10,7 +11,9 @@ def create_bear_researcher(llm):
history = investment_debate_state.get("history", "")
bear_history = investment_debate_state.get("bear_history", "")
current_response = investment_debate_state.get("current_response", "")
current_response = opponent_argument_or_opening(
investment_debate_state.get("current_response", ""), "bull analyst"
)
market_research_report = state["market_report"]
sentiment_report = state["sentiment_report"]
news_report = state["news_report"]

View File

@@ -1,6 +1,7 @@
from tradingagents.agents.utils.agent_utils import (
get_instrument_context_from_state,
get_language_instruction,
opponent_argument_or_opening,
)
@@ -10,7 +11,9 @@ def create_bull_researcher(llm):
history = investment_debate_state.get("history", "")
bull_history = investment_debate_state.get("bull_history", "")
current_response = investment_debate_state.get("current_response", "")
current_response = opponent_argument_or_opening(
investment_debate_state.get("current_response", ""), "bear analyst"
)
market_research_report = state["market_report"]
sentiment_report = state["sentiment_report"]
news_report = state["news_report"]

View File

@@ -1,6 +1,7 @@
from tradingagents.agents.utils.agent_utils import (
get_instrument_context_from_state,
get_language_instruction,
opponent_argument_or_opening,
)
@@ -10,8 +11,12 @@ def create_aggressive_debator(llm):
history = risk_debate_state.get("history", "")
aggressive_history = risk_debate_state.get("aggressive_history", "")
current_conservative_response = risk_debate_state.get("current_conservative_response", "")
current_neutral_response = risk_debate_state.get("current_neutral_response", "")
current_conservative_response = opponent_argument_or_opening(
risk_debate_state.get("current_conservative_response", ""), "conservative analyst"
)
current_neutral_response = opponent_argument_or_opening(
risk_debate_state.get("current_neutral_response", ""), "neutral analyst"
)
market_research_report = state["market_report"]
sentiment_report = state["sentiment_report"]

View File

@@ -1,6 +1,7 @@
from tradingagents.agents.utils.agent_utils import (
get_instrument_context_from_state,
get_language_instruction,
opponent_argument_or_opening,
)
@@ -10,8 +11,12 @@ def create_conservative_debator(llm):
history = risk_debate_state.get("history", "")
conservative_history = risk_debate_state.get("conservative_history", "")
current_aggressive_response = risk_debate_state.get("current_aggressive_response", "")
current_neutral_response = risk_debate_state.get("current_neutral_response", "")
current_aggressive_response = opponent_argument_or_opening(
risk_debate_state.get("current_aggressive_response", ""), "aggressive analyst"
)
current_neutral_response = opponent_argument_or_opening(
risk_debate_state.get("current_neutral_response", ""), "neutral analyst"
)
market_research_report = state["market_report"]
sentiment_report = state["sentiment_report"]

View File

@@ -1,6 +1,7 @@
from tradingagents.agents.utils.agent_utils import (
get_instrument_context_from_state,
get_language_instruction,
opponent_argument_or_opening,
)
@@ -10,8 +11,12 @@ def create_neutral_debator(llm):
history = risk_debate_state.get("history", "")
neutral_history = risk_debate_state.get("neutral_history", "")
current_aggressive_response = risk_debate_state.get("current_aggressive_response", "")
current_conservative_response = risk_debate_state.get("current_conservative_response", "")
current_aggressive_response = opponent_argument_or_opening(
risk_debate_state.get("current_aggressive_response", ""), "aggressive analyst"
)
current_conservative_response = opponent_argument_or_opening(
risk_debate_state.get("current_conservative_response", ""), "conservative analyst"
)
market_research_report = state["market_report"]
sentiment_report = state["sentiment_report"]

View File

@@ -65,6 +65,20 @@ def get_language_instruction() -> str:
return f" Write your entire response in {lang}."
def opponent_argument_or_opening(text: str, opponent: str) -> str:
"""Opponent's latest argument, or an explicit opening marker when empty.
The first speaker in each debate round receives an empty opponent response;
interpolating it into a "refute the opponent" prompt makes the model
fabricate the other side's position. Returning a clear "has not spoken yet"
marker instead lets it open with its own case (#1176).
"""
text = (text or "").strip()
if text:
return text
return f"(The {opponent} has not spoken yet — open the debate with your own case.)"
def _clean_identity_value(value: Any) -> str | None:
"""Return a trimmed string, or None for empty / placeholder-ish values."""
if not isinstance(value, str):