fix(memory): score a decision against its benchmark over the same dates

- the stock's own sessions set the holding window; the benchmark is valued at its last close by entry and by exit, so a coin held over a weekend is compared with SPY over the same span
- bars meet on their calendar day across time zones, and a missing or non-positive close is dropped rather than scored
- an outcome waits until the benchmark has traded through the exit
This commit is contained in:
Yijia-Xiao
2026-09-29 05:57:18 +00:00
parent 42e2d569e0
commit 13cf1e8138
2 changed files with 107 additions and 10 deletions
+73
View File
@@ -1060,3 +1060,76 @@ def test_concurrent_writers_keep_every_decision_and_outcome(tmp_path):
assert errors == []
assert len(entries) == len(tickers) * len(dates)
assert all(not e["pending"] for e in entries)
def _closes(by_symbol):
"""A get_closes stand-in serving each symbol's series between the dates asked."""
def get_closes(symbol, start_date, end_date):
series = by_symbol[symbol]
return series[(series.index >= start_date) & (series.index < end_date)]
return get_closes
@pytest.mark.unit
class TestSettlementWindow:
"""Stock and benchmark are measured over the same dates, whatever their calendars."""
def test_a_seven_day_market_is_scored_against_the_benchmark_over_the_same_dates(self, monkeypatch):
coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D"))
weekdays = pd.bdate_range("2025-12-29", "2026-01-16")
spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays)
monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))
raw, alpha, days, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")
# The coin is held Sat 3 -> Thu 8; SPY over the same span is Fri 2 -> Thu 8.
assert resolved == "2026-01-08"
assert raw == pytest.approx(105.0 / 100.0 - 1)
assert alpha == pytest.approx(raw - (spy["2026-01-08"] / spy["2026-01-02"] - 1))
def test_bars_stamped_in_different_time_zones_meet_on_their_dates(self, monkeypatch):
# Yahoo stamps a coin's daily bar at midnight UTC and SPY's at midnight
# New York, five hours later: the same day, not the day after.
coin = pd.Series([100.0 + i for i in range(10)],
index=pd.date_range("2026-01-03", periods=10, freq="D", tz="UTC"))
weekdays = pd.bdate_range("2025-12-29", "2026-01-16", tz="America/New_York")
spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays)
monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))
raw, alpha, _, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")
assert resolved == "2026-01-08"
spy_by_day = spy.tz_localize(None)
assert alpha == pytest.approx(raw - (spy_by_day["2026-01-08"] / spy_by_day["2026-01-02"] - 1))
def test_the_outcome_waits_for_the_benchmark_to_trade_through_the_window(self, monkeypatch):
coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D"))
spy = pd.Series([400.0, 401.0, 402.0], index=pd.to_datetime(["2026-01-02", "2026-01-05", "2026-01-06"]))
monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))
assert settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY") == (None, None, None, None)
def test_a_missing_close_is_skipped_not_scored(self, monkeypatch):
idx = pd.bdate_range("2026-01-05", periods=8)
stock = pd.Series([100.0, float("nan"), 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx)
spy = pd.Series([400.0 + i for i in range(8)], index=idx)
monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy}))
raw, alpha, _, resolved = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")
assert raw == pytest.approx(105.0 / 100.0 - 1)
assert resolved == "2026-01-13"
assert alpha == pytest.approx(raw - (spy["2026-01-13"] / spy["2026-01-05"] - 1))
@pytest.mark.unit
def test_a_close_that_is_not_a_price_is_skipped_not_scored(monkeypatch):
"""A zero close would score as an infinite return and be written to the memory log."""
idx = pd.bdate_range("2026-01-05", periods=8)
stock = pd.Series([0.0, 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx)
spy = pd.Series([400.0 + i for i in range(8)], index=idx)
monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy}))
raw, alpha, _, _ = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")
assert raw == pytest.approx(105.0 / 100.0 - 1) # the window opens on the first real close
+34 -10
View File
@@ -34,6 +34,20 @@ def resolve_benchmark(ticker: str, config: dict) -> str:
return benchmark_map.get("", "SPY")
def _by_day(closes):
"""Closes keyed by the calendar day of their bar, missing closes dropped.
Yahoo stamps a daily bar at midnight in its market's time zone (UTC for a
coin, New York for SPY), so two series meet on their dates, not their
instants. A missing or non-positive close is no price at all, not a bar to
score on.
"""
closes = closes[closes > 0]
if getattr(closes.index, "tz", None) is not None:
closes = closes.tz_localize(None)
return closes
def fetch_returns(
ticker: str, trade_date: str, holding_days: int = 5,
benchmark: str = "SPY",
@@ -56,21 +70,31 @@ def fetch_returns(
end_str = end.strftime("%Y-%m-%d")
# Closes for the instrument the analysis priced (XAUUSD -> GC=F, #984).
stock = get_closes(ticker, trade_date, end_str)
bench = get_closes(benchmark, trade_date, end_str)
stock = _by_day(get_closes(ticker, trade_date, end_str))
# From a week earlier, so the benchmark has a close on or before entry.
bench_start = (start - timedelta(days=7)).strftime("%Y-%m-%d")
bench = _by_day(get_closes(benchmark, bench_start, end_str))
# Require the full holding window in both series. A rerun before it
# has traded leaves the entry pending to retry next run, rather than
# settling on a premature partial return (#1169).
if len(stock) <= holding_days or len(bench) <= holding_days:
# Require the full holding window to have traded. A rerun before it has
# leaves the entry pending to retry next run, rather than settling on a
# premature partial return (#1169).
if len(stock) <= holding_days:
return None, None, None, None
entry, exit_ = stock.index[0], stock.index[holding_days]
# The window is the stock's own sessions; the benchmark is valued at its
# last close by each end, so both returns span the same dates even when
# the calendars differ (a coin trades at weekends, an index does not).
# It must have traded through the exit, or its close there may yet move.
if bench.empty or bench.index[0] > entry or bench.index[-1] < exit_:
return None, None, None, None
raw = float((stock.iloc[holding_days] - stock.iloc[0]) / stock.iloc[0])
bench_ret = float((bench.iloc[holding_days] - bench.iloc[0]) / bench.iloc[0])
bench_entry, bench_exit = bench.asof(entry), bench.asof(exit_)
bench_ret = float((bench_exit - bench_entry) / bench_entry)
alpha = raw - bench_ret
# The date of the last price bar used is when this outcome became
# known — the point-in-time cutoff for injecting the lesson (#1251).
resolution_date = stock.index[holding_days].strftime("%Y-%m-%d")
# Every close used is known by the exit: the point-in-time cutoff for
# injecting the lesson (#1251).
resolution_date = exit_.strftime("%Y-%m-%d")
return raw, alpha, holding_days, resolution_date
except Exception as e:
logger.warning(