github-mirrors/tradingagents
1
0
Fork 0
mirror of https://github.com/TauricResearch/TradingAgents.git synced 2026-09-19 11:15:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(memory): gate past-context lessons to point-in-time in backtests

Browse Source 
- get_past_context returned every resolved lesson regardless of the run date, so
  a historical run could learn from an outcome that had not happened yet
- record each resolved entry's resolution date (the last price bar used) and
  filter get_past_context(as_of=trade_date) on it for a historical run; a
  current-date run passes None so live behavior and pre-migration entries (no
  stored resolution date, conservatively excluded from backtests) are unaffected #1251
This commit is contained in:
Yijia-Xiao
2026-08-30 07:03:06 +00:00
parent 51a245dbe1
commit 8db41f6bca
5 changed files with 192 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
tests/test_memory_log.py
View File

@@ -54,9 +54,14 @@ def _resolve_entry(log, ticker, date, decision, reflection="Good call."):
log.update_with_outcome(ticker, date, 0.05, 0.02, 5, reflection)
def _price_df(prices):
"""Minimal DataFrame matching yfinance .history() output shape."""
return pd.DataFrame({"Close": prices})
def _price_df(prices, start="2026-01-05"):
"""Minimal DataFrame matching yfinance .history() output shape.
Uses a DatetimeIndex like real yfinance output, so resolution-date
extraction (stock.index[actual_days]) works (#1251).
"""
idx = pd.date_range(start=start, periods=len(prices), freq="D")
return pd.DataFrame({"Close": prices}, index=idx)
def _make_pm_state(past_context=""):
@@ -496,30 +501,32 @@ class TestDeferredReflection:
m.history.return_value = _price_df(spy_prices if sym == "SPY" else stock_prices)
return m
mock_ticker_cls.side_effect = _make_ticker
raw, alpha, days = TradingAgentsGraph._fetch_returns(mock_graph, "NVDA", "2026-01-05")
raw, alpha, days, resolved = TradingAgentsGraph._fetch_returns(mock_graph, "NVDA", "2026-01-05")
assert raw is not None and alpha is not None and days is not None
assert isinstance(raw, float) and isinstance(alpha, float) and isinstance(days, int)
assert days == 5
# resolution date = the bar `days` sessions after the trade date (#1251)
assert resolved == "2026-01-10"
def test_fetch_returns_too_recent(self):
"""Only 1 data point available → returns (None, None, None), no crash."""
"""Only 1 data point available → returns all-None, no crash."""
mock_graph = MagicMock(spec=TradingAgentsGraph)
with patch("yfinance.Ticker") as mock_ticker_cls:
m = MagicMock()
m.history.return_value = _price_df([100.0])
mock_ticker_cls.return_value = m
raw, alpha, days = TradingAgentsGraph._fetch_returns(mock_graph, "NVDA", "2026-04-19")
assert raw is None and alpha is None and days is None
raw, alpha, days, resolved = TradingAgentsGraph._fetch_returns(mock_graph, "NVDA", "2026-04-19")
assert (raw, alpha, days, resolved) == (None, None, None, None)
def test_fetch_returns_delisted(self):
"""Empty DataFrame → returns (None, None, None), no crash."""
"""Empty DataFrame → returns all-None, no crash."""
mock_graph = MagicMock(spec=TradingAgentsGraph)
with patch("yfinance.Ticker") as mock_ticker_cls:
m = MagicMock()
m.history.return_value = pd.DataFrame({"Close": []})
mock_ticker_cls.return_value = m
raw, alpha, days = TradingAgentsGraph._fetch_returns(mock_graph, "XXXXXFAKE", "2026-01-10")
assert raw is None and alpha is None and days is None
raw, alpha, days, resolved = TradingAgentsGraph._fetch_returns(mock_graph, "XXXXXFAKE", "2026-01-10")
assert (raw, alpha, days, resolved) == (None, None, None, None)
def test_fetch_returns_spy_shorter_than_stock(self):
"""SPY having fewer rows than the stock must not raise IndexError."""
@@ -532,9 +539,10 @@ class TestDeferredReflection:
m.history.return_value = _price_df(spy_prices if sym == "SPY" else stock_prices)
return m
mock_ticker_cls.side_effect = _make_ticker
raw, alpha, days = TradingAgentsGraph._fetch_returns(mock_graph, "NVDA", "2026-01-05")
raw, alpha, days, resolved = TradingAgentsGraph._fetch_returns(mock_graph, "NVDA", "2026-01-05")
assert raw is not None and alpha is not None and days is not None
assert days == 2
assert resolved == "2026-01-07" # 2 sessions after the trade date
# TradingAgentsGraph._resolve_benchmark — picks index for alpha calc
@@ -641,7 +649,7 @@ class TestDeferredReflection:
log.store_decision("AAPL", "2026-01-10", DECISION_BUY)
mock_graph = MagicMock(spec=TradingAgentsGraph)
mock_graph.memory_log = log
mock_graph._fetch_returns = MagicMock(return_value=(0.05, 0.02, 5))
mock_graph._fetch_returns = MagicMock(return_value=(0.05, 0.02, 5, "2026-01-12"))
TradingAgentsGraph._resolve_pending_entries(mock_graph, "NVDA")
mock_graph._fetch_returns.assert_not_called()
assert len(log.get_pending_entries()) == 1
@@ -655,7 +663,7 @@ class TestDeferredReflection:
mock_graph = MagicMock(spec=TradingAgentsGraph)
mock_graph.memory_log = log
mock_graph.reflector = mock_reflector
mock_graph._fetch_returns = MagicMock(return_value=(0.05, 0.02, 5))
mock_graph._fetch_returns = MagicMock(return_value=(0.05, 0.02, 5, "2026-01-12"))
TradingAgentsGraph._resolve_pending_entries(mock_graph, "NVDA")
assert log.get_pending_entries() == []
entries = log.load_entries()

95
tests/test_memory_pointintime.py Normal file
View File

@@ -0,0 +1,95 @@
"""Memory-log lessons must be point-in-time safe in a backtest (#1251).
get_past_context previously returned every resolved lesson regardless of the run
date, so a historical run could learn from an outcome that had not happened yet.
Resolved entries now record the date their outcome became known (``resolved:``),
and get_past_context(as_of=...) filters on it. Legacy entries without a
resolution date are excluded from a point-in-time query (conservative migration).
"""
from __future__ import annotations
import pytest
from tradingagents.agents.utils.memory import TradingMemoryLog
def _log(tmp_path):
return TradingMemoryLog({"memory_log_path": str(tmp_path / "mem.md")})
def _resolve(log, ticker, date, resolution_date, reflection):
log.store_decision(ticker, date, f"Rating: Buy\n{reflection}")
log.update_with_outcome(
ticker, date, 0.05, 0.02, 5, reflection, resolution_date=resolution_date,
)
@pytest.mark.unit
def test_resolution_date_is_stored_and_parsed(tmp_path):
log = _log(tmp_path)
_resolve(log, "NVDA", "2026-01-05", "2026-01-10", "outcome known 01-10")
entry = log.load_entries()[0]
assert entry["resolved"] == "2026-01-10"
assert "resolved:2026-01-10" in (tmp_path / "mem.md").read_text()
@pytest.mark.unit
def test_as_of_excludes_lessons_resolved_after_the_run_date(tmp_path):
log = _log(tmp_path)
# Decision on 01-05, outcome only known on 01-10.
_resolve(log, "NVDA", "2026-01-05", "2026-01-10", "great trade")
# A run as-of 01-07 must NOT see it (the outcome was still in the future).
assert log.get_past_context("NVDA", as_of="2026-01-07") == ""
# A run as-of 01-10 (and later) sees it.
assert "great trade" in log.get_past_context("NVDA", as_of="2026-01-10")
assert "great trade" in log.get_past_context("NVDA", as_of="2026-02-01")
@pytest.mark.unit
def test_no_as_of_is_unfiltered_live_behavior(tmp_path):
log = _log(tmp_path)
_resolve(log, "NVDA", "2026-01-05", "2026-01-10", "great trade")
# Live run (no as_of): unchanged behavior, lesson is shown.
assert "great trade" in log.get_past_context("NVDA")
@pytest.mark.unit
def test_legacy_entry_without_resolution_date_excluded_in_backtest(tmp_path):
log = _log(tmp_path)
# Simulate a pre-migration resolved entry: no resolution_date recorded.
log.store_decision("NVDA", "2026-01-05", "Rating: Buy\nlegacy lesson")
log.update_with_outcome("NVDA", "2026-01-05", 0.05, 0.02, 5, "legacy lesson")
entry = log.load_entries()[0]
assert entry["resolved"] is None
# Conservative: excluded from a point-in-time query (can't prove it was known)...
assert log.get_past_context("NVDA", as_of="2026-06-01") == ""
# ...but still available on a live (unfiltered) run.
assert "legacy lesson" in log.get_past_context("NVDA")
@pytest.mark.unit
def test_cross_ticker_lessons_are_also_gated(tmp_path):
log = _log(tmp_path)
_resolve(log, "AAPL", "2026-01-05", "2026-01-10", "cross lesson")
# Querying a different ticker as-of before resolution: no cross lesson leaks.
assert log.get_past_context("NVDA", as_of="2026-01-07") == ""
assert "cross lesson" in log.get_past_context("NVDA", as_of="2026-01-10")
@pytest.mark.unit
def test_memory_as_of_gates_historical_but_not_live():
# The graph filters only for a past trade date; a current-date run passes
# None so live behavior and legacy entries are unaffected (#1251).
from datetime import datetime, timedelta
from tradingagents.graph.trading_graph import TradingAgentsGraph
g = object.__new__(TradingAgentsGraph)
past = "2024-01-01"
today = datetime.now().strftime("%Y-%m-%d")
future = (datetime.now() + timedelta(days=30)).strftime("%Y-%m-%d")
assert g._memory_as_of(past) == past # backtest -> filter on the trade date
assert g._memory_as_of(today) is None # live -> no filter
assert g._memory_as_of(future) is None # future-dated run -> no filter

7
tests/test_symbol_normalization_paths.py
View File

@@ -41,18 +41,21 @@ def test_fetch_returns_normalizes_symbol(monkeypatch):
queried.append(symbol)
def history(self, *args, **kwargs):
return pd.DataFrame({"Close": [100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0]})
prices = [100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0]
idx = pd.date_range(start="2025-01-02", periods=len(prices), freq="D")
return pd.DataFrame({"Close": prices}, index=idx)
monkeypatch.setattr(tg.yf, "Ticker", FakeTicker)
# _fetch_returns does not use ``self``; call unbound to avoid building the graph.
raw, alpha, days = TradingAgentsGraph._fetch_returns(
raw, alpha, days, resolved = TradingAgentsGraph._fetch_returns(
None, "XAUUSD", "2025-01-02", holding_days=5, benchmark="SPY"
)
assert queried[0] == "GC=F" # stock symbol normalized (#984)
assert queried[1] == "SPY" # benchmark left as the canonical symbol
assert raw is not None and days is not None
assert resolved == "2025-01-07" # resolution date recorded (#1251)
def test_news_lookup_normalizes_symbol(monkeypatch):