mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-19 11:15:24 +03:00
fix(agents): stop debate openers from rebutting a nonexistent argument
- the first speaker in each debate round received an empty opponent response, yet the prompt demanded a rebuttal, so models fabricated the other side - substitute an explicit opening marker when an opponent has not spoken, across all five debators (bull, bear, and the three risk analysts) #1176
This commit is contained in:
111
tests/test_debate_opening.py
Normal file
111
tests/test_debate_opening.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""The first speaker in each debate must not rebut a nonexistent argument (#1176).
|
||||
|
||||
Each debate round's opening speaker receives an empty opponent response; the
|
||||
prompt used to interpolate it into a "refute the opponent" instruction, so models
|
||||
fabricated the other side's position. All five debators (bull, bear, and the
|
||||
three risk analysts) now substitute an explicit opening marker when the opponent
|
||||
has not spoken, and pass a real argument through unchanged.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tradingagents.agents.researchers.bear_researcher import create_bear_researcher
|
||||
from tradingagents.agents.researchers.bull_researcher import create_bull_researcher
|
||||
from tradingagents.agents.risk_mgmt.aggressive_debator import create_aggressive_debator
|
||||
from tradingagents.agents.risk_mgmt.conservative_debator import create_conservative_debator
|
||||
from tradingagents.agents.risk_mgmt.neutral_debator import create_neutral_debator
|
||||
from tradingagents.agents.utils.agent_utils import opponent_argument_or_opening
|
||||
|
||||
_REPORTS = {
|
||||
"company_of_interest": "AAPL", "asset_type": "stock",
|
||||
"market_report": "m", "sentiment_report": "s",
|
||||
"news_report": "n", "fundamentals_report": "f",
|
||||
}
|
||||
|
||||
|
||||
def _capturing_llm(captured: dict):
|
||||
llm = MagicMock()
|
||||
llm.invoke.side_effect = lambda prompt: (
|
||||
captured.__setitem__("prompt", prompt) or MagicMock(content="argument")
|
||||
)
|
||||
return llm
|
||||
|
||||
|
||||
def _investment_state(current_response):
|
||||
return {
|
||||
**_REPORTS,
|
||||
"count": 0,
|
||||
"investment_debate_state": {
|
||||
"history": "", "bull_history": "", "bear_history": "",
|
||||
"current_response": current_response, "count": 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _risk_state(**responses):
|
||||
base = {
|
||||
"current_aggressive_response": "", "current_conservative_response": "",
|
||||
"current_neutral_response": "", "history": "", "aggressive_history": "",
|
||||
"conservative_history": "", "neutral_history": "", "count": 0,
|
||||
}
|
||||
base.update(responses)
|
||||
return {**_REPORTS, "trader_investment_plan": "plan", "risk_debate_state": base}
|
||||
|
||||
|
||||
# --- shared helper ----------------------------------------------------------
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_helper_marks_empty_and_passes_through():
|
||||
assert "has not spoken yet" in opponent_argument_or_opening("", "bear analyst")
|
||||
assert opponent_argument_or_opening(" real point ", "bear") == "real point"
|
||||
|
||||
|
||||
# --- researchers ------------------------------------------------------------
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize(
|
||||
"factory,opponent",
|
||||
[(create_bull_researcher, "bear"), (create_bear_researcher, "bull")],
|
||||
)
|
||||
def test_researcher_opening_has_no_phantom_opponent(factory, opponent):
|
||||
captured = {}
|
||||
factory(_capturing_llm(captured))(_investment_state(""))
|
||||
assert "has not spoken yet" in captured["prompt"]
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_researcher_passes_real_opponent_argument():
|
||||
captured = {}
|
||||
state = _investment_state("Bear Analyst: valuation is stretched")
|
||||
create_bull_researcher(_capturing_llm(captured))(state)
|
||||
assert "valuation is stretched" in captured["prompt"]
|
||||
assert "has not spoken yet" not in captured["prompt"]
|
||||
|
||||
|
||||
# --- risk debators ----------------------------------------------------------
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize(
|
||||
"factory", [create_aggressive_debator, create_conservative_debator, create_neutral_debator]
|
||||
)
|
||||
def test_risk_opening_has_no_phantom_opponent(factory):
|
||||
captured = {}
|
||||
factory(_capturing_llm(captured))(_risk_state())
|
||||
# Both opponent slots were empty -> two opening markers, no fabricated args.
|
||||
assert captured["prompt"].count("has not spoken yet") == 2
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_risk_passes_real_opponent_arguments():
|
||||
captured = {}
|
||||
state = _risk_state(
|
||||
current_conservative_response="Conservative Analyst: trim risk",
|
||||
current_neutral_response="Neutral Analyst: hold steady",
|
||||
)
|
||||
create_aggressive_debator(_capturing_llm(captured))(state)
|
||||
assert "trim risk" in captured["prompt"]
|
||||
assert "hold steady" in captured["prompt"]
|
||||
assert "has not spoken yet" not in captured["prompt"]
|
||||
@@ -1,6 +1,7 @@
|
||||
from tradingagents.agents.utils.agent_utils import (
|
||||
get_instrument_context_from_state,
|
||||
get_language_instruction,
|
||||
opponent_argument_or_opening,
|
||||
)
|
||||
|
||||
|
||||
@@ -10,7 +11,9 @@ def create_bear_researcher(llm):
|
||||
history = investment_debate_state.get("history", "")
|
||||
bear_history = investment_debate_state.get("bear_history", "")
|
||||
|
||||
current_response = investment_debate_state.get("current_response", "")
|
||||
current_response = opponent_argument_or_opening(
|
||||
investment_debate_state.get("current_response", ""), "bull analyst"
|
||||
)
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
news_report = state["news_report"]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from tradingagents.agents.utils.agent_utils import (
|
||||
get_instrument_context_from_state,
|
||||
get_language_instruction,
|
||||
opponent_argument_or_opening,
|
||||
)
|
||||
|
||||
|
||||
@@ -10,7 +11,9 @@ def create_bull_researcher(llm):
|
||||
history = investment_debate_state.get("history", "")
|
||||
bull_history = investment_debate_state.get("bull_history", "")
|
||||
|
||||
current_response = investment_debate_state.get("current_response", "")
|
||||
current_response = opponent_argument_or_opening(
|
||||
investment_debate_state.get("current_response", ""), "bear analyst"
|
||||
)
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
news_report = state["news_report"]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from tradingagents.agents.utils.agent_utils import (
|
||||
get_instrument_context_from_state,
|
||||
get_language_instruction,
|
||||
opponent_argument_or_opening,
|
||||
)
|
||||
|
||||
|
||||
@@ -10,8 +11,12 @@ def create_aggressive_debator(llm):
|
||||
history = risk_debate_state.get("history", "")
|
||||
aggressive_history = risk_debate_state.get("aggressive_history", "")
|
||||
|
||||
current_conservative_response = risk_debate_state.get("current_conservative_response", "")
|
||||
current_neutral_response = risk_debate_state.get("current_neutral_response", "")
|
||||
current_conservative_response = opponent_argument_or_opening(
|
||||
risk_debate_state.get("current_conservative_response", ""), "conservative analyst"
|
||||
)
|
||||
current_neutral_response = opponent_argument_or_opening(
|
||||
risk_debate_state.get("current_neutral_response", ""), "neutral analyst"
|
||||
)
|
||||
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from tradingagents.agents.utils.agent_utils import (
|
||||
get_instrument_context_from_state,
|
||||
get_language_instruction,
|
||||
opponent_argument_or_opening,
|
||||
)
|
||||
|
||||
|
||||
@@ -10,8 +11,12 @@ def create_conservative_debator(llm):
|
||||
history = risk_debate_state.get("history", "")
|
||||
conservative_history = risk_debate_state.get("conservative_history", "")
|
||||
|
||||
current_aggressive_response = risk_debate_state.get("current_aggressive_response", "")
|
||||
current_neutral_response = risk_debate_state.get("current_neutral_response", "")
|
||||
current_aggressive_response = opponent_argument_or_opening(
|
||||
risk_debate_state.get("current_aggressive_response", ""), "aggressive analyst"
|
||||
)
|
||||
current_neutral_response = opponent_argument_or_opening(
|
||||
risk_debate_state.get("current_neutral_response", ""), "neutral analyst"
|
||||
)
|
||||
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from tradingagents.agents.utils.agent_utils import (
|
||||
get_instrument_context_from_state,
|
||||
get_language_instruction,
|
||||
opponent_argument_or_opening,
|
||||
)
|
||||
|
||||
|
||||
@@ -10,8 +11,12 @@ def create_neutral_debator(llm):
|
||||
history = risk_debate_state.get("history", "")
|
||||
neutral_history = risk_debate_state.get("neutral_history", "")
|
||||
|
||||
current_aggressive_response = risk_debate_state.get("current_aggressive_response", "")
|
||||
current_conservative_response = risk_debate_state.get("current_conservative_response", "")
|
||||
current_aggressive_response = opponent_argument_or_opening(
|
||||
risk_debate_state.get("current_aggressive_response", ""), "aggressive analyst"
|
||||
)
|
||||
current_conservative_response = opponent_argument_or_opening(
|
||||
risk_debate_state.get("current_conservative_response", ""), "conservative analyst"
|
||||
)
|
||||
|
||||
market_research_report = state["market_report"]
|
||||
sentiment_report = state["sentiment_report"]
|
||||
|
||||
@@ -65,6 +65,20 @@ def get_language_instruction() -> str:
|
||||
return f" Write your entire response in {lang}."
|
||||
|
||||
|
||||
def opponent_argument_or_opening(text: str, opponent: str) -> str:
|
||||
"""Opponent's latest argument, or an explicit opening marker when empty.
|
||||
|
||||
The first speaker in each debate round receives an empty opponent response;
|
||||
interpolating it into a "refute the opponent" prompt makes the model
|
||||
fabricate the other side's position. Returning a clear "has not spoken yet"
|
||||
marker instead lets it open with its own case (#1176).
|
||||
"""
|
||||
text = (text or "").strip()
|
||||
if text:
|
||||
return text
|
||||
return f"(The {opponent} has not spoken yet — open the debate with your own case.)"
|
||||
|
||||
|
||||
def _clean_identity_value(value: Any) -> str | None:
|
||||
"""Return a trimmed string, or None for empty / placeholder-ish values."""
|
||||
if not isinstance(value, str):
|
||||
|
||||
Reference in New Issue
Block a user