mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-11-03 16:12:44 +02:00
fix(memory): score a decision against its benchmark over the same dates
- the stock's own sessions set the holding window; the benchmark is valued at its last close by entry and by exit, so a coin held over a weekend is compared with SPY over the same span - bars meet on their calendar day across time zones, and a missing or non-positive close is dropped rather than scored - an outcome waits until the benchmark has traded through the exit
This commit is contained in:
@@ -1060,3 +1060,76 @@ def test_concurrent_writers_keep_every_decision_and_outcome(tmp_path):
|
||||
assert errors == []
|
||||
assert len(entries) == len(tickers) * len(dates)
|
||||
assert all(not e["pending"] for e in entries)
|
||||
|
||||
|
||||
def _closes(by_symbol):
|
||||
"""A get_closes stand-in serving each symbol's series between the dates asked."""
|
||||
def get_closes(symbol, start_date, end_date):
|
||||
series = by_symbol[symbol]
|
||||
return series[(series.index >= start_date) & (series.index < end_date)]
|
||||
return get_closes
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSettlementWindow:
|
||||
"""Stock and benchmark are measured over the same dates, whatever their calendars."""
|
||||
|
||||
def test_a_seven_day_market_is_scored_against_the_benchmark_over_the_same_dates(self, monkeypatch):
|
||||
coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D"))
|
||||
weekdays = pd.bdate_range("2025-12-29", "2026-01-16")
|
||||
spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays)
|
||||
monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))
|
||||
|
||||
raw, alpha, days, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")
|
||||
|
||||
# The coin is held Sat 3 -> Thu 8; SPY over the same span is Fri 2 -> Thu 8.
|
||||
assert resolved == "2026-01-08"
|
||||
assert raw == pytest.approx(105.0 / 100.0 - 1)
|
||||
assert alpha == pytest.approx(raw - (spy["2026-01-08"] / spy["2026-01-02"] - 1))
|
||||
|
||||
def test_bars_stamped_in_different_time_zones_meet_on_their_dates(self, monkeypatch):
|
||||
# Yahoo stamps a coin's daily bar at midnight UTC and SPY's at midnight
|
||||
# New York, five hours later: the same day, not the day after.
|
||||
coin = pd.Series([100.0 + i for i in range(10)],
|
||||
index=pd.date_range("2026-01-03", periods=10, freq="D", tz="UTC"))
|
||||
weekdays = pd.bdate_range("2025-12-29", "2026-01-16", tz="America/New_York")
|
||||
spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays)
|
||||
monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))
|
||||
|
||||
raw, alpha, _, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")
|
||||
|
||||
assert resolved == "2026-01-08"
|
||||
spy_by_day = spy.tz_localize(None)
|
||||
assert alpha == pytest.approx(raw - (spy_by_day["2026-01-08"] / spy_by_day["2026-01-02"] - 1))
|
||||
|
||||
def test_the_outcome_waits_for_the_benchmark_to_trade_through_the_window(self, monkeypatch):
|
||||
coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D"))
|
||||
spy = pd.Series([400.0, 401.0, 402.0], index=pd.to_datetime(["2026-01-02", "2026-01-05", "2026-01-06"]))
|
||||
monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))
|
||||
|
||||
assert settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY") == (None, None, None, None)
|
||||
|
||||
def test_a_missing_close_is_skipped_not_scored(self, monkeypatch):
|
||||
idx = pd.bdate_range("2026-01-05", periods=8)
|
||||
stock = pd.Series([100.0, float("nan"), 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx)
|
||||
spy = pd.Series([400.0 + i for i in range(8)], index=idx)
|
||||
monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy}))
|
||||
|
||||
raw, alpha, _, resolved = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")
|
||||
|
||||
assert raw == pytest.approx(105.0 / 100.0 - 1)
|
||||
assert resolved == "2026-01-13"
|
||||
assert alpha == pytest.approx(raw - (spy["2026-01-13"] / spy["2026-01-05"] - 1))
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_close_that_is_not_a_price_is_skipped_not_scored(monkeypatch):
|
||||
"""A zero close would score as an infinite return and be written to the memory log."""
|
||||
idx = pd.bdate_range("2026-01-05", periods=8)
|
||||
stock = pd.Series([0.0, 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx)
|
||||
spy = pd.Series([400.0 + i for i in range(8)], index=idx)
|
||||
monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy}))
|
||||
|
||||
raw, alpha, _, _ = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")
|
||||
|
||||
assert raw == pytest.approx(105.0 / 100.0 - 1) # the window opens on the first real close
|
||||
|
||||
@@ -34,6 +34,20 @@ def resolve_benchmark(ticker: str, config: dict) -> str:
|
||||
return benchmark_map.get("", "SPY")
|
||||
|
||||
|
||||
def _by_day(closes):
|
||||
"""Closes keyed by the calendar day of their bar, missing closes dropped.
|
||||
|
||||
Yahoo stamps a daily bar at midnight in its market's time zone (UTC for a
|
||||
coin, New York for SPY), so two series meet on their dates, not their
|
||||
instants. A missing or non-positive close is no price at all, not a bar to
|
||||
score on.
|
||||
"""
|
||||
closes = closes[closes > 0]
|
||||
if getattr(closes.index, "tz", None) is not None:
|
||||
closes = closes.tz_localize(None)
|
||||
return closes
|
||||
|
||||
|
||||
def fetch_returns(
|
||||
ticker: str, trade_date: str, holding_days: int = 5,
|
||||
benchmark: str = "SPY",
|
||||
@@ -56,21 +70,31 @@ def fetch_returns(
|
||||
end_str = end.strftime("%Y-%m-%d")
|
||||
|
||||
# Closes for the instrument the analysis priced (XAUUSD -> GC=F, #984).
|
||||
stock = get_closes(ticker, trade_date, end_str)
|
||||
bench = get_closes(benchmark, trade_date, end_str)
|
||||
stock = _by_day(get_closes(ticker, trade_date, end_str))
|
||||
# From a week earlier, so the benchmark has a close on or before entry.
|
||||
bench_start = (start - timedelta(days=7)).strftime("%Y-%m-%d")
|
||||
bench = _by_day(get_closes(benchmark, bench_start, end_str))
|
||||
|
||||
# Require the full holding window in both series. A rerun before it
|
||||
# has traded leaves the entry pending to retry next run, rather than
|
||||
# settling on a premature partial return (#1169).
|
||||
if len(stock) <= holding_days or len(bench) <= holding_days:
|
||||
# Require the full holding window to have traded. A rerun before it has
|
||||
# leaves the entry pending to retry next run, rather than settling on a
|
||||
# premature partial return (#1169).
|
||||
if len(stock) <= holding_days:
|
||||
return None, None, None, None
|
||||
entry, exit_ = stock.index[0], stock.index[holding_days]
|
||||
# The window is the stock's own sessions; the benchmark is valued at its
|
||||
# last close by each end, so both returns span the same dates even when
|
||||
# the calendars differ (a coin trades at weekends, an index does not).
|
||||
# It must have traded through the exit, or its close there may yet move.
|
||||
if bench.empty or bench.index[0] > entry or bench.index[-1] < exit_:
|
||||
return None, None, None, None
|
||||
|
||||
raw = float((stock.iloc[holding_days] - stock.iloc[0]) / stock.iloc[0])
|
||||
bench_ret = float((bench.iloc[holding_days] - bench.iloc[0]) / bench.iloc[0])
|
||||
bench_entry, bench_exit = bench.asof(entry), bench.asof(exit_)
|
||||
bench_ret = float((bench_exit - bench_entry) / bench_entry)
|
||||
alpha = raw - bench_ret
|
||||
# The date of the last price bar used is when this outcome became
|
||||
# known — the point-in-time cutoff for injecting the lesson (#1251).
|
||||
resolution_date = stock.index[holding_days].strftime("%Y-%m-%d")
|
||||
# Every close used is known by the exit: the point-in-time cutoff for
|
||||
# injecting the lesson (#1251).
|
||||
resolution_date = exit_.strftime("%Y-%m-%d")
|
||||
return raw, alpha, holding_days, resolution_date
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
|
||||
Reference in New Issue
Block a user