feat(graph): measure an outcome over a configurable window

- holding_period_days sets it; the price request covers the calendar span those trading days occupy
- reflection states the window it judges, so a short one does not read as a failed thesis
This commit is contained in:
Yijia-Xiao
2026-09-17 07:31:44 +00:00
parent 2ca59cc795
commit 85d9137437
4 changed files with 90 additions and 11 deletions

View File

@@ -695,6 +695,7 @@ class TestDeferredReflection:
log = make_log(tmp_path)
log.store_decision("AAPL", "2026-01-10", DECISION_BUY)
mock_graph = MagicMock(spec=TradingAgentsGraph)
mock_graph.config = {}
mock_graph.memory_log = log
mock_graph._fetch_returns = MagicMock(return_value=(0.05, 0.02, 5, "2026-01-12"))
TradingAgentsGraph._resolve_pending_entries(mock_graph, "NVDA")
@@ -708,6 +709,7 @@ class TestDeferredReflection:
mock_reflector = MagicMock()
mock_reflector.reflect_on_final_decision.return_value = "Momentum confirmed."
mock_graph = MagicMock(spec=TradingAgentsGraph)
mock_graph.config = {}
mock_graph.memory_log = log
mock_graph.reflector = mock_reflector
mock_graph._fetch_returns = MagicMock(return_value=(0.05, 0.02, 5, "2026-01-12"))
@@ -727,6 +729,7 @@ class TestDeferredReflection:
log.store_decision("NVDA", "2026-01-05", DECISION_BUY)
mock_reflector = MagicMock()
mock_graph = MagicMock(spec=TradingAgentsGraph)
mock_graph.config = {}
mock_graph.memory_log = log
mock_graph.reflector = mock_reflector
mock_graph._fetch_returns = MagicMock(return_value=(None, None, None, None))
@@ -957,7 +960,7 @@ def test_a_failed_reflection_leaves_the_entry_pending_and_lets_the_run_start(tmp
graph.memory_log.store_decision("NVDA", "2026-01-12", "Rating: Sell\n\ny")
monkeypatch.setattr(graph, "_resolve_benchmark", lambda t: "SPY", raising=False)
monkeypatch.setattr(graph, "_fetch_returns",
lambda t, d, benchmark=None: (0.01, 0.005, 5, "2026-01-19"), raising=False)
lambda t, d, holding_days=5, benchmark=None: (0.01, 0.005, holding_days, "2026-01-19"), raising=False)
class _Reflector:
calls = 0
@@ -974,3 +977,66 @@ def test_a_failed_reflection_leaves_the_entry_pending_and_lets_the_run_start(tmp
entries = graph.memory_log.load_entries()
assert [e["pending"] for e in entries] == [True, False] # the failed one waits for next time
@pytest.mark.unit
def test_the_holding_window_is_configurable(tmp_path, monkeypatch):
"""A decision written for months should not be graded at a week without the
operator choosing that window."""
from tradingagents.agents.utils.memory import TradingMemoryLog
from tradingagents.graph.trading_graph import TradingAgentsGraph
graph = object.__new__(TradingAgentsGraph)
graph.config = {"memory_log_path": str(tmp_path / "m.md"), "holding_period_days": 21}
graph.memory_log = TradingMemoryLog(graph.config)
graph.memory_log.store_decision("NVDA", "2026-01-05", "**Rating**: Buy\n\nx")
monkeypatch.setattr(graph, "_resolve_benchmark", lambda t: "SPY", raising=False)
asked = {}
def _returns(ticker, date, holding_days=5, benchmark=None):
asked["holding_days"] = holding_days
return 0.05, 0.02, holding_days, "2026-02-02"
monkeypatch.setattr(graph, "_fetch_returns", _returns, raising=False)
graph.reflector = type("R", (), {"reflect_on_final_decision": lambda self, **kw: "lesson"})()
graph._resolve_pending_entries("NVDA")
assert asked["holding_days"] == 21
assert graph.memory_log.load_entries()[0]["holding"] == "21d"
@pytest.mark.unit
def test_the_reflection_states_the_window_it_judges():
"""Judging a months-long thesis on a week's alpha, without saying so, turns
a scope mismatch into a lesson that the call was wrong."""
from tradingagents.graph.reflection import Reflector
prompt = Reflector(None)._system_prompt(holding_days=5)
assert "5" in prompt and "trading day" in prompt
@pytest.mark.unit
def test_a_longer_window_asks_for_enough_price_history(monkeypatch):
"""Trading days are not calendar days: a 21-day window needs about a month
of bars, and asking for 28 days left every outcome unsettled."""
from tradingagents.graph.trading_graph import TradingAgentsGraph
graph = object.__new__(TradingAgentsGraph)
asked = {}
class _Ticker:
def __init__(self, symbol):
self.symbol = symbol
def history(self, start, end):
asked["start"], asked["end"] = start, end
import pandas as pd
days = pd.bdate_range(start, end)
return pd.DataFrame({"Close": range(len(days))}, index=days)
monkeypatch.setattr("tradingagents.graph.trading_graph.yf.Ticker", _Ticker)
raw, alpha, days, resolved = graph._fetch_returns("NVDA", "2026-06-01", 21, benchmark="SPY")
assert days == 21 and resolved is not None, (raw, alpha, days, resolved)

View File

@@ -154,6 +154,9 @@ DEFAULT_CONFIG = _apply_env_overrides({
# based on the ticker's exchange suffix. SPY remains the US default
# so the reflection label keeps reading "Alpha vs SPY" for US tickers
# while non-US tickers get their regional index automatically.
# Trading days after the analysis date over which a decision's outcome is
# measured, for reflection and for the backtest figures.
"holding_period_days": 5,
"benchmark_ticker": None,
"benchmark_map": {
".NS": "^NSEI", # NSE India (Nifty 50)

View File

@@ -9,20 +9,25 @@ class Reflector:
def __init__(self, quick_thinking_llm: Any):
"""Initialize the reflector with an LLM."""
self.quick_thinking_llm = quick_thinking_llm
self.log_reflection_prompt = self._get_log_reflection_prompt()
def _get_log_reflection_prompt(self) -> str:
def _system_prompt(self, holding_days: int) -> str:
"""Concise prompt for reflect_on_final_decision (Phase B log entries).
Produces 2-4 sentences of plain prose compact enough to be re-injected
into future agent prompts without bloating the context window.
Produces 2-4 sentences of plain prose, compact enough to be re-injected
into future agent prompts without bloating the context window. The
window is named because it bounds what the outcome can show: a thesis
written for months is not disproved by a week, and a lesson that ignores
the difference is read by later runs as an established failure.
"""
return (
"You are a trading analyst reviewing your own past decision now that the outcome is known.\n"
f"The outcome covers {holding_days} trading days after the analysis date, "
"which may be shorter than the horizon the decision was written for.\n"
"Write exactly 2-4 sentences of plain prose (no bullets, no headers, no markdown).\n\n"
"Cover in order:\n"
"1. Was the directional call correct? (cite the alpha figure)\n"
"2. Which part of the investment thesis held or failed?\n"
f"1. What the {holding_days}-day alpha shows about the directional call (cite the figure), "
"and say so plainly if the window is too short to judge the thesis.\n"
"2. Which part of the investment thesis this window supports or undercuts.\n"
"3. One concrete lesson to apply to the next similar analysis.\n\n"
"Be specific and terse. Your output will be stored verbatim in a decision log "
"and re-read by future analysts, so every word must earn its place."
@@ -34,6 +39,7 @@ class Reflector:
raw_return: float,
alpha_return: float,
benchmark_name: str = "SPY",
holding_days: int = 5,
) -> str:
"""Single reflection call on the final trade decision with outcome context.
@@ -44,11 +50,11 @@ class Reflector:
callers that haven't been updated to thread the benchmark through.
"""
messages = [
("system", self.log_reflection_prompt),
("system", self._system_prompt(holding_days)),
(
"human",
(
f"Raw return: {raw_return:+.1%}\n"
f"Raw return over {holding_days} trading days: {raw_return:+.1%}\n"
f"Alpha vs {benchmark_name}: {alpha_return:+.1%}\n\n"
f"Final Decision:\n{final_decision}"
),

View File

@@ -306,7 +306,9 @@ class TradingAgentsGraph:
try:
start = datetime.strptime(trade_date, "%Y-%m-%d")
end = start + timedelta(days=holding_days + 7) # buffer for weekends/holidays
# holding_days counts trading days, so ask for the calendar span they
# occupy (about 7 for every 5) plus a week for holidays.
end = start + timedelta(days=round(holding_days * 7 / 5) + 7)
end_str = end.strftime("%Y-%m-%d")
# Normalize so the realized-return lookup hits the same instrument
@@ -359,7 +361,8 @@ class TradingAgentsGraph:
updates = []
for entry in pending:
raw, alpha, days, resolution_date = self._fetch_returns(
ticker, entry["date"], benchmark=benchmark,
ticker, entry["date"], self.config.get("holding_period_days", 5),
benchmark=benchmark,
)
if raw is None:
continue # price not available yet — try again next run
@@ -369,6 +372,7 @@ class TradingAgentsGraph:
raw_return=raw,
alpha_return=alpha,
benchmark_name=benchmark,
holding_days=days,
)
except Exception as exc:
# Reflection calls a provider, and this runs on the way into a