mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-19 11:15:24 +03:00
feat(graph): measure an outcome over a configurable window
- holding_period_days sets it; the price request covers the calendar span those trading days occupy - reflection states the window it judges, so a short one does not read as a failed thesis
This commit is contained in:
@@ -695,6 +695,7 @@ class TestDeferredReflection:
|
||||
log = make_log(tmp_path)
|
||||
log.store_decision("AAPL", "2026-01-10", DECISION_BUY)
|
||||
mock_graph = MagicMock(spec=TradingAgentsGraph)
|
||||
mock_graph.config = {}
|
||||
mock_graph.memory_log = log
|
||||
mock_graph._fetch_returns = MagicMock(return_value=(0.05, 0.02, 5, "2026-01-12"))
|
||||
TradingAgentsGraph._resolve_pending_entries(mock_graph, "NVDA")
|
||||
@@ -708,6 +709,7 @@ class TestDeferredReflection:
|
||||
mock_reflector = MagicMock()
|
||||
mock_reflector.reflect_on_final_decision.return_value = "Momentum confirmed."
|
||||
mock_graph = MagicMock(spec=TradingAgentsGraph)
|
||||
mock_graph.config = {}
|
||||
mock_graph.memory_log = log
|
||||
mock_graph.reflector = mock_reflector
|
||||
mock_graph._fetch_returns = MagicMock(return_value=(0.05, 0.02, 5, "2026-01-12"))
|
||||
@@ -727,6 +729,7 @@ class TestDeferredReflection:
|
||||
log.store_decision("NVDA", "2026-01-05", DECISION_BUY)
|
||||
mock_reflector = MagicMock()
|
||||
mock_graph = MagicMock(spec=TradingAgentsGraph)
|
||||
mock_graph.config = {}
|
||||
mock_graph.memory_log = log
|
||||
mock_graph.reflector = mock_reflector
|
||||
mock_graph._fetch_returns = MagicMock(return_value=(None, None, None, None))
|
||||
@@ -957,7 +960,7 @@ def test_a_failed_reflection_leaves_the_entry_pending_and_lets_the_run_start(tmp
|
||||
graph.memory_log.store_decision("NVDA", "2026-01-12", "Rating: Sell\n\ny")
|
||||
monkeypatch.setattr(graph, "_resolve_benchmark", lambda t: "SPY", raising=False)
|
||||
monkeypatch.setattr(graph, "_fetch_returns",
|
||||
lambda t, d, benchmark=None: (0.01, 0.005, 5, "2026-01-19"), raising=False)
|
||||
lambda t, d, holding_days=5, benchmark=None: (0.01, 0.005, holding_days, "2026-01-19"), raising=False)
|
||||
|
||||
class _Reflector:
|
||||
calls = 0
|
||||
@@ -974,3 +977,66 @@ def test_a_failed_reflection_leaves_the_entry_pending_and_lets_the_run_start(tmp
|
||||
|
||||
entries = graph.memory_log.load_entries()
|
||||
assert [e["pending"] for e in entries] == [True, False] # the failed one waits for next time
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_holding_window_is_configurable(tmp_path, monkeypatch):
|
||||
"""A decision written for months should not be graded at a week without the
|
||||
operator choosing that window."""
|
||||
from tradingagents.agents.utils.memory import TradingMemoryLog
|
||||
from tradingagents.graph.trading_graph import TradingAgentsGraph
|
||||
|
||||
graph = object.__new__(TradingAgentsGraph)
|
||||
graph.config = {"memory_log_path": str(tmp_path / "m.md"), "holding_period_days": 21}
|
||||
graph.memory_log = TradingMemoryLog(graph.config)
|
||||
graph.memory_log.store_decision("NVDA", "2026-01-05", "**Rating**: Buy\n\nx")
|
||||
monkeypatch.setattr(graph, "_resolve_benchmark", lambda t: "SPY", raising=False)
|
||||
asked = {}
|
||||
|
||||
def _returns(ticker, date, holding_days=5, benchmark=None):
|
||||
asked["holding_days"] = holding_days
|
||||
return 0.05, 0.02, holding_days, "2026-02-02"
|
||||
|
||||
monkeypatch.setattr(graph, "_fetch_returns", _returns, raising=False)
|
||||
graph.reflector = type("R", (), {"reflect_on_final_decision": lambda self, **kw: "lesson"})()
|
||||
|
||||
graph._resolve_pending_entries("NVDA")
|
||||
|
||||
assert asked["holding_days"] == 21
|
||||
assert graph.memory_log.load_entries()[0]["holding"] == "21d"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_reflection_states_the_window_it_judges():
|
||||
"""Judging a months-long thesis on a week's alpha, without saying so, turns
|
||||
a scope mismatch into a lesson that the call was wrong."""
|
||||
from tradingagents.graph.reflection import Reflector
|
||||
|
||||
prompt = Reflector(None)._system_prompt(holding_days=5)
|
||||
assert "5" in prompt and "trading day" in prompt
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_longer_window_asks_for_enough_price_history(monkeypatch):
|
||||
"""Trading days are not calendar days: a 21-day window needs about a month
|
||||
of bars, and asking for 28 days left every outcome unsettled."""
|
||||
from tradingagents.graph.trading_graph import TradingAgentsGraph
|
||||
|
||||
graph = object.__new__(TradingAgentsGraph)
|
||||
asked = {}
|
||||
|
||||
class _Ticker:
|
||||
def __init__(self, symbol):
|
||||
self.symbol = symbol
|
||||
|
||||
def history(self, start, end):
|
||||
asked["start"], asked["end"] = start, end
|
||||
import pandas as pd
|
||||
days = pd.bdate_range(start, end)
|
||||
return pd.DataFrame({"Close": range(len(days))}, index=days)
|
||||
|
||||
monkeypatch.setattr("tradingagents.graph.trading_graph.yf.Ticker", _Ticker)
|
||||
|
||||
raw, alpha, days, resolved = graph._fetch_returns("NVDA", "2026-06-01", 21, benchmark="SPY")
|
||||
|
||||
assert days == 21 and resolved is not None, (raw, alpha, days, resolved)
|
||||
|
||||
@@ -154,6 +154,9 @@ DEFAULT_CONFIG = _apply_env_overrides({
|
||||
# based on the ticker's exchange suffix. SPY remains the US default
|
||||
# so the reflection label keeps reading "Alpha vs SPY" for US tickers
|
||||
# while non-US tickers get their regional index automatically.
|
||||
# Trading days after the analysis date over which a decision's outcome is
|
||||
# measured, for reflection and for the backtest figures.
|
||||
"holding_period_days": 5,
|
||||
"benchmark_ticker": None,
|
||||
"benchmark_map": {
|
||||
".NS": "^NSEI", # NSE India (Nifty 50)
|
||||
|
||||
@@ -9,20 +9,25 @@ class Reflector:
|
||||
def __init__(self, quick_thinking_llm: Any):
|
||||
"""Initialize the reflector with an LLM."""
|
||||
self.quick_thinking_llm = quick_thinking_llm
|
||||
self.log_reflection_prompt = self._get_log_reflection_prompt()
|
||||
|
||||
def _get_log_reflection_prompt(self) -> str:
|
||||
def _system_prompt(self, holding_days: int) -> str:
|
||||
"""Concise prompt for reflect_on_final_decision (Phase B log entries).
|
||||
|
||||
Produces 2-4 sentences of plain prose — compact enough to be re-injected
|
||||
into future agent prompts without bloating the context window.
|
||||
Produces 2-4 sentences of plain prose, compact enough to be re-injected
|
||||
into future agent prompts without bloating the context window. The
|
||||
window is named because it bounds what the outcome can show: a thesis
|
||||
written for months is not disproved by a week, and a lesson that ignores
|
||||
the difference is read by later runs as an established failure.
|
||||
"""
|
||||
return (
|
||||
"You are a trading analyst reviewing your own past decision now that the outcome is known.\n"
|
||||
f"The outcome covers {holding_days} trading days after the analysis date, "
|
||||
"which may be shorter than the horizon the decision was written for.\n"
|
||||
"Write exactly 2-4 sentences of plain prose (no bullets, no headers, no markdown).\n\n"
|
||||
"Cover in order:\n"
|
||||
"1. Was the directional call correct? (cite the alpha figure)\n"
|
||||
"2. Which part of the investment thesis held or failed?\n"
|
||||
f"1. What the {holding_days}-day alpha shows about the directional call (cite the figure), "
|
||||
"and say so plainly if the window is too short to judge the thesis.\n"
|
||||
"2. Which part of the investment thesis this window supports or undercuts.\n"
|
||||
"3. One concrete lesson to apply to the next similar analysis.\n\n"
|
||||
"Be specific and terse. Your output will be stored verbatim in a decision log "
|
||||
"and re-read by future analysts, so every word must earn its place."
|
||||
@@ -34,6 +39,7 @@ class Reflector:
|
||||
raw_return: float,
|
||||
alpha_return: float,
|
||||
benchmark_name: str = "SPY",
|
||||
holding_days: int = 5,
|
||||
) -> str:
|
||||
"""Single reflection call on the final trade decision with outcome context.
|
||||
|
||||
@@ -44,11 +50,11 @@ class Reflector:
|
||||
callers that haven't been updated to thread the benchmark through.
|
||||
"""
|
||||
messages = [
|
||||
("system", self.log_reflection_prompt),
|
||||
("system", self._system_prompt(holding_days)),
|
||||
(
|
||||
"human",
|
||||
(
|
||||
f"Raw return: {raw_return:+.1%}\n"
|
||||
f"Raw return over {holding_days} trading days: {raw_return:+.1%}\n"
|
||||
f"Alpha vs {benchmark_name}: {alpha_return:+.1%}\n\n"
|
||||
f"Final Decision:\n{final_decision}"
|
||||
),
|
||||
|
||||
@@ -306,7 +306,9 @@ class TradingAgentsGraph:
|
||||
|
||||
try:
|
||||
start = datetime.strptime(trade_date, "%Y-%m-%d")
|
||||
end = start + timedelta(days=holding_days + 7) # buffer for weekends/holidays
|
||||
# holding_days counts trading days, so ask for the calendar span they
|
||||
# occupy (about 7 for every 5) plus a week for holidays.
|
||||
end = start + timedelta(days=round(holding_days * 7 / 5) + 7)
|
||||
end_str = end.strftime("%Y-%m-%d")
|
||||
|
||||
# Normalize so the realized-return lookup hits the same instrument
|
||||
@@ -359,7 +361,8 @@ class TradingAgentsGraph:
|
||||
updates = []
|
||||
for entry in pending:
|
||||
raw, alpha, days, resolution_date = self._fetch_returns(
|
||||
ticker, entry["date"], benchmark=benchmark,
|
||||
ticker, entry["date"], self.config.get("holding_period_days", 5),
|
||||
benchmark=benchmark,
|
||||
)
|
||||
if raw is None:
|
||||
continue # price not available yet — try again next run
|
||||
@@ -369,6 +372,7 @@ class TradingAgentsGraph:
|
||||
raw_return=raw,
|
||||
alpha_return=alpha,
|
||||
benchmark_name=benchmark,
|
||||
holding_days=days,
|
||||
)
|
||||
except Exception as exc:
|
||||
# Reflection calls a provider, and this runs on the way into a
|
||||
|
||||
Reference in New Issue
Block a user