From 13cf1e81388061c0c12a8ab151e161e353e685ca Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Mon, 28 Sep 2026 20:05:11 +0000 Subject: [PATCH] fix(memory): score a decision against its benchmark over the same dates - the stock's own sessions set the holding window; the benchmark is valued at its last close by entry and by exit, so a coin held over a weekend is compared with SPY over the same span - bars meet on their calendar day across time zones, and a missing or non-positive close is dropped rather than scored - an outcome waits until the benchmark has traded through the exit --- tests/test_memory_log.py | 73 ++++++++++++++++++++++++++++++ tradingagents/memory/settlement.py | 44 ++++++++++++++---- 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/tests/test_memory_log.py b/tests/test_memory_log.py index 73876d299..b3516b3fb 100644 --- a/tests/test_memory_log.py +++ b/tests/test_memory_log.py @@ -1060,3 +1060,76 @@ def test_concurrent_writers_keep_every_decision_and_outcome(tmp_path): assert errors == [] assert len(entries) == len(tickers) * len(dates) assert all(not e["pending"] for e in entries) + + +def _closes(by_symbol): + """A get_closes stand-in serving each symbol's series between the dates asked.""" + def get_closes(symbol, start_date, end_date): + series = by_symbol[symbol] + return series[(series.index >= start_date) & (series.index < end_date)] + return get_closes + + +@pytest.mark.unit +class TestSettlementWindow: + """Stock and benchmark are measured over the same dates, whatever their calendars.""" + + def test_a_seven_day_market_is_scored_against_the_benchmark_over_the_same_dates(self, monkeypatch): + coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D")) + weekdays = pd.bdate_range("2025-12-29", "2026-01-16") + spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays) + monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy})) + + raw, alpha, days, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY") + + # The coin is held Sat 3 -> Thu 8; SPY over the same span is Fri 2 -> Thu 8. + assert resolved == "2026-01-08" + assert raw == pytest.approx(105.0 / 100.0 - 1) + assert alpha == pytest.approx(raw - (spy["2026-01-08"] / spy["2026-01-02"] - 1)) + + def test_bars_stamped_in_different_time_zones_meet_on_their_dates(self, monkeypatch): + # Yahoo stamps a coin's daily bar at midnight UTC and SPY's at midnight + # New York, five hours later: the same day, not the day after. + coin = pd.Series([100.0 + i for i in range(10)], + index=pd.date_range("2026-01-03", periods=10, freq="D", tz="UTC")) + weekdays = pd.bdate_range("2025-12-29", "2026-01-16", tz="America/New_York") + spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays) + monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy})) + + raw, alpha, _, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY") + + assert resolved == "2026-01-08" + spy_by_day = spy.tz_localize(None) + assert alpha == pytest.approx(raw - (spy_by_day["2026-01-08"] / spy_by_day["2026-01-02"] - 1)) + + def test_the_outcome_waits_for_the_benchmark_to_trade_through_the_window(self, monkeypatch): + coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D")) + spy = pd.Series([400.0, 401.0, 402.0], index=pd.to_datetime(["2026-01-02", "2026-01-05", "2026-01-06"])) + monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy})) + + assert settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY") == (None, None, None, None) + + def test_a_missing_close_is_skipped_not_scored(self, monkeypatch): + idx = pd.bdate_range("2026-01-05", periods=8) + stock = pd.Series([100.0, float("nan"), 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx) + spy = pd.Series([400.0 + i for i in range(8)], index=idx) + monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy})) + + raw, alpha, _, resolved = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY") + + assert raw == pytest.approx(105.0 / 100.0 - 1) + assert resolved == "2026-01-13" + assert alpha == pytest.approx(raw - (spy["2026-01-13"] / spy["2026-01-05"] - 1)) + + +@pytest.mark.unit +def test_a_close_that_is_not_a_price_is_skipped_not_scored(monkeypatch): + """A zero close would score as an infinite return and be written to the memory log.""" + idx = pd.bdate_range("2026-01-05", periods=8) + stock = pd.Series([0.0, 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx) + spy = pd.Series([400.0 + i for i in range(8)], index=idx) + monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy})) + + raw, alpha, _, _ = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY") + + assert raw == pytest.approx(105.0 / 100.0 - 1) # the window opens on the first real close diff --git a/tradingagents/memory/settlement.py b/tradingagents/memory/settlement.py index 6f8354de8..de14c9b11 100644 --- a/tradingagents/memory/settlement.py +++ b/tradingagents/memory/settlement.py @@ -34,6 +34,20 @@ def resolve_benchmark(ticker: str, config: dict) -> str: return benchmark_map.get("", "SPY") +def _by_day(closes): + """Closes keyed by the calendar day of their bar, missing closes dropped. + + Yahoo stamps a daily bar at midnight in its market's time zone (UTC for a + coin, New York for SPY), so two series meet on their dates, not their + instants. A missing or non-positive close is no price at all, not a bar to + score on. + """ + closes = closes[closes > 0] + if getattr(closes.index, "tz", None) is not None: + closes = closes.tz_localize(None) + return closes + + def fetch_returns( ticker: str, trade_date: str, holding_days: int = 5, benchmark: str = "SPY", @@ -56,21 +70,31 @@ def fetch_returns( end_str = end.strftime("%Y-%m-%d") # Closes for the instrument the analysis priced (XAUUSD -> GC=F, #984). - stock = get_closes(ticker, trade_date, end_str) - bench = get_closes(benchmark, trade_date, end_str) + stock = _by_day(get_closes(ticker, trade_date, end_str)) + # From a week earlier, so the benchmark has a close on or before entry. + bench_start = (start - timedelta(days=7)).strftime("%Y-%m-%d") + bench = _by_day(get_closes(benchmark, bench_start, end_str)) - # Require the full holding window in both series. A rerun before it - # has traded leaves the entry pending to retry next run, rather than - # settling on a premature partial return (#1169). - if len(stock) <= holding_days or len(bench) <= holding_days: + # Require the full holding window to have traded. A rerun before it has + # leaves the entry pending to retry next run, rather than settling on a + # premature partial return (#1169). + if len(stock) <= holding_days: + return None, None, None, None + entry, exit_ = stock.index[0], stock.index[holding_days] + # The window is the stock's own sessions; the benchmark is valued at its + # last close by each end, so both returns span the same dates even when + # the calendars differ (a coin trades at weekends, an index does not). + # It must have traded through the exit, or its close there may yet move. + if bench.empty or bench.index[0] > entry or bench.index[-1] < exit_: return None, None, None, None raw = float((stock.iloc[holding_days] - stock.iloc[0]) / stock.iloc[0]) - bench_ret = float((bench.iloc[holding_days] - bench.iloc[0]) / bench.iloc[0]) + bench_entry, bench_exit = bench.asof(entry), bench.asof(exit_) + bench_ret = float((bench_exit - bench_entry) / bench_entry) alpha = raw - bench_ret - # The date of the last price bar used is when this outcome became - # known — the point-in-time cutoff for injecting the lesson (#1251). - resolution_date = stock.index[holding_days].strftime("%Y-%m-%d") + # Every close used is known by the exit: the point-in-time cutoff for + # injecting the lesson (#1251). + resolution_date = exit_.strftime("%Y-%m-%d") return raw, alpha, holding_days, resolution_date except Exception as e: logger.warning(