diff --git a/README.md b/README.md index 38cc4f3cf..03f19b039 100644 --- a/README.md +++ b/README.md @@ -374,7 +374,7 @@ From the CLI: tradingagents backtest NVDA,AAPL --start 2026-06-01 --end 2026-08-01 --every 7 ``` -Each cell is scored on realized alpha against the instrument's regional benchmark, grouped by rating. Your own memory log is never written to, and re-running the same grid with `run_id=result.run_id` skips the cells that already ran, so an interrupted sweep continues where it stopped. +Each cell is scored on realized alpha against the instrument's regional benchmark, grouped by rating. A decision is entered at the first close after its date, because a run reads news published after that day's close. Your own memory log is never written to, and re-running the same grid with `run_id=result.run_id` skips the cells that already ran, so an interrupted sweep continues where it stopped. ## Reproducibility diff --git a/tests/test_graph_end_to_end.py b/tests/test_graph_end_to_end.py index 2f5b55e86..b55562e0d 100644 --- a/tests/test_graph_end_to_end.py +++ b/tests/test_graph_end_to_end.py @@ -312,7 +312,7 @@ def test_past_decisions_settle_alongside_the_analysts(tmp_path, monkeypatch, off model = TimedModel() graph = _graph(tmp_path, monkeypatch, model) - graph.memory_log.store_decision("NVDA", "2026-01-02", "Rating: Buy\n\nBuy NVDA.") + graph.memory_log.store_decision("NVDA", "2025-12-31", "Rating: Buy\n\nBuy NVDA.") closes = pd.Series([100.0 + i for i in range(10)], index=pd.bdate_range("2026-01-02", periods=10)) monkeypatch.setattr(settlement, "get_closes", lambda *a, **k: closes) @@ -322,7 +322,7 @@ def test_past_decisions_settle_alongside_the_analysts(tmp_path, monkeypatch, off assert len(reflections) == 1 # settled once, inside the run analyst_turns = [c for c in model.seen if c[0] == "agent"] assert reflections[0][2] < max(c[3] for c in analyst_turns), "settled before the analysts, not alongside" - settled = next(e for e in graph.memory_log.load_entries() if e["date"] == "2026-01-02") + settled = next(e for e in graph.memory_log.load_entries() if e["date"] == "2025-12-31") assert not settled["pending"] lessons = graph.memory_log.get_past_context("NVDA", as_of=TRADE_DATE) assert lessons @@ -372,7 +372,7 @@ def test_the_memory_step_reads_lessons_as_of_the_trade_date_and_settles_once(tmp from tradingagents.memory import settlement graph = _graph(tmp_path, monkeypatch, ScriptedModel(structured=True)) - graph.memory_log.store_decision("NVDA", "2026-01-02", "Rating: Buy\n\nBuy NVDA.") + graph.memory_log.store_decision("NVDA", "2025-12-31", "Rating: Buy\n\nBuy NVDA.") closes = pd.Series([100.0 + i for i in range(10)], index=pd.bdate_range("2026-01-02", periods=10)) monkeypatch.setattr(settlement, "get_closes", lambda *a, **k: closes) asked = [] diff --git a/tests/test_memory_log.py b/tests/test_memory_log.py index 375be979e..b7206afae 100644 --- a/tests/test_memory_log.py +++ b/tests/test_memory_log.py @@ -512,8 +512,8 @@ class TestDeferredReflection: # settlement.fetch_returns def test_fetch_returns_valid_ticker(self): - stock_prices = [100.0, 102.0, 104.0, 103.0, 105.0, 106.0] - spy_prices = [400.0, 402.0, 404.0, 403.0, 405.0, 406.0] + stock_prices = [100.0, 102.0, 104.0, 103.0, 105.0, 106.0, 107.0] + spy_prices = [400.0, 402.0, 404.0, 403.0, 405.0, 406.0, 407.0] with patch("yfinance.Ticker") as mock_ticker_cls: def _make_ticker(sym): m = MagicMock() @@ -524,8 +524,8 @@ class TestDeferredReflection: assert raw is not None and alpha is not None and days is not None assert isinstance(raw, float) and isinstance(alpha, float) and isinstance(days, int) assert days == 5 - # resolution date = the bar `days` sessions after the trade date (#1251) - assert resolved == "2026-01-10" + # resolution date = the bar `days` sessions after entry, the first close after the decision (#1251) + assert resolved == "2026-01-11" def test_fetch_returns_too_recent(self): """Only 1 data point available → returns all-None, no crash.""" @@ -549,7 +549,7 @@ class TestDeferredReflection: """SPY having fewer rows than the stock (but still a full window) must not raise IndexError.""" stock_prices = [100.0, 102.0, 104.0, 103.0, 105.0, 106.0, 107.0, 108.0] # 8 rows - spy_prices = [400.0, 402.0, 403.0, 405.0, 406.0, 407.0] # 6 rows + spy_prices = [400.0, 402.0, 403.0, 405.0, 406.0, 407.0, 408.0] # 7 rows with patch("yfinance.Ticker") as mock_ticker_cls: def _make_ticker(sym): m = MagicMock() @@ -559,7 +559,22 @@ class TestDeferredReflection: raw, alpha, days, resolved = settlement.fetch_returns("NVDA", "2026-01-05") assert raw is not None and alpha is not None assert days == 5 # full holding window used for both series - assert resolved == "2026-01-10" + assert resolved == "2026-01-11" + + def test_a_decision_is_scored_from_the_first_close_after_its_date(self): + """#1518: news published after the close on the decision date reaches the + analysts, so the close of that date is not a price they could have traded at.""" + stock_prices = [100.0, 150.0, 150.0, 150.0, 150.0, 150.0, 165.0] # jumps on the next close + spy_prices = [400.0] * 7 + with patch("yfinance.Ticker") as mock_ticker_cls: + def _make_ticker(sym): + m = MagicMock() + m.history.return_value = _price_df(spy_prices if sym == "SPY" else stock_prices) + return m + mock_ticker_cls.side_effect = _make_ticker + raw, alpha, days, resolved = settlement.fetch_returns("NVDA", "2026-01-05") + assert raw == pytest.approx(0.10) and alpha == pytest.approx(0.10) # 150 -> 165, not 100 -> 150 + assert resolved == "2026-01-11" def test_fetch_returns_incomplete_window_stays_pending(self): """#1169: a rerun before the full holding window has traded returns @@ -1082,10 +1097,10 @@ class TestSettlementWindow: raw, alpha, days, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY") - # The coin is held Sat 3 -> Thu 8; SPY over the same span is Fri 2 -> Thu 8. - assert resolved == "2026-01-08" - assert raw == pytest.approx(105.0 / 100.0 - 1) - assert alpha == pytest.approx(raw - (spy["2026-01-08"] / spy["2026-01-02"] - 1)) + # Decided Sat 3, the coin is held Sun 4 -> Fri 9; SPY over the same span is Fri 2 -> Fri 9. + assert resolved == "2026-01-09" + assert raw == pytest.approx(106.0 / 101.0 - 1) + assert alpha == pytest.approx(raw - (spy["2026-01-09"] / spy["2026-01-02"] - 1)) def test_bars_stamped_in_different_time_zones_meet_on_their_dates(self, monkeypatch): # Yahoo stamps a coin's daily bar at midnight UTC and SPY's at midnight @@ -1098,9 +1113,9 @@ class TestSettlementWindow: raw, alpha, _, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY") - assert resolved == "2026-01-08" + assert resolved == "2026-01-09" spy_by_day = spy.tz_localize(None) - assert alpha == pytest.approx(raw - (spy_by_day["2026-01-08"] / spy_by_day["2026-01-02"] - 1)) + assert alpha == pytest.approx(raw - (spy_by_day["2026-01-09"] / spy_by_day["2026-01-02"] - 1)) def test_the_outcome_waits_for_the_benchmark_to_trade_through_the_window(self, monkeypatch): coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D")) @@ -1117,9 +1132,10 @@ class TestSettlementWindow: raw, alpha, _, resolved = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY") - assert raw == pytest.approx(105.0 / 100.0 - 1) - assert resolved == "2026-01-13" - assert alpha == pytest.approx(raw - (spy["2026-01-13"] / spy["2026-01-05"] - 1)) + # Entered at the first real close after the decision: Wed 7, past the missing Tue 6. + assert raw == pytest.approx(106.0 / 101.0 - 1) + assert resolved == "2026-01-14" + assert alpha == pytest.approx(raw - (spy["2026-01-14"] / spy["2026-01-07"] - 1)) @pytest.mark.unit @@ -1130,7 +1146,7 @@ def test_a_close_that_is_not_a_price_is_skipped_not_scored(monkeypatch): spy = pd.Series([400.0 + i for i in range(8)], index=idx) monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy})) - raw, alpha, _, _ = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY") + raw, alpha, _, _ = settlement.fetch_returns("NVDA", "2026-01-02", 5, "SPY") assert raw == pytest.approx(105.0 / 100.0 - 1) # the window opens on the first real close diff --git a/tests/test_symbol_normalization_paths.py b/tests/test_symbol_normalization_paths.py index a1971445d..9b4cf62d3 100644 --- a/tests/test_symbol_normalization_paths.py +++ b/tests/test_symbol_normalization_paths.py @@ -54,7 +54,7 @@ def test_fetch_returns_normalizes_symbol(monkeypatch): assert queried[0] == "GC=F" # stock symbol normalized (#984) assert queried[1] == "SPY" # benchmark left as the canonical symbol assert raw is not None and days is not None - assert resolved == "2025-01-07" # resolution date recorded (#1251) + assert resolved == "2025-01-08" # resolution date recorded (#1251) def test_news_lookup_normalizes_symbol(monkeypatch): diff --git a/tradingagents/backtest.py b/tradingagents/backtest.py index c4a33cb84..1b3503118 100644 --- a/tradingagents/backtest.py +++ b/tradingagents/backtest.py @@ -119,7 +119,7 @@ class BacktestSummary: if self.pending: lines.append("Pending cells are not scored above; re-run to settle them.") lines.append( - f"Alpha is measured over {self.holding} after each analysis date. " + f"Alpha is measured over {self.holding} from the first close after each analysis date. " "One model sampling per cell, and text feeds are not archived, so " "these figures are indicative rather than repeatable." ) diff --git a/tradingagents/default_config.py b/tradingagents/default_config.py index 5723a70c0..2ce4fadc6 100644 --- a/tradingagents/default_config.py +++ b/tradingagents/default_config.py @@ -175,8 +175,8 @@ def build_default_config() -> dict: # based on the ticker's exchange suffix. SPY remains the US default # so the reflection label keeps reading "Alpha vs SPY" for US tickers # while non-US tickers get their regional index automatically. - # Trading days after the analysis date over which a decision's outcome is - # measured, for reflection and for the backtest figures. + # Trading days from the first close after the analysis date over which a + # decision's outcome is measured, for reflection and for the backtest figures. "holding_period_days": 5, "benchmark_ticker": None, "benchmark_map": { diff --git a/tradingagents/memory/reflection.py b/tradingagents/memory/reflection.py index ba8bf5f31..cf23c6395 100644 --- a/tradingagents/memory/reflection.py +++ b/tradingagents/memory/reflection.py @@ -21,7 +21,7 @@ class Reflector: """ return ( "You are a trading analyst reviewing your own past decision now that the outcome is known.\n" - f"The outcome covers {holding_days} trading days after the analysis date, " + f"The outcome covers {holding_days} trading days from the first close after the analysis date, " "which may be shorter than the horizon the decision was written for.\n" "Write exactly 2-4 sentences of plain prose (no bullets, no headers, no markdown).\n\n" "Cover in order:\n" diff --git a/tradingagents/memory/settlement.py b/tradingagents/memory/settlement.py index 032b3c68b..c997b19f2 100644 --- a/tradingagents/memory/settlement.py +++ b/tradingagents/memory/settlement.py @@ -53,7 +53,10 @@ def fetch_returns( ticker: str, trade_date: str, holding_days: int = 5, benchmark: str = "SPY", ) -> tuple[float | None, float | None, int | None, str | None]: - """Fetch raw and alpha return for ticker over holding_days from trade_date. + """Fetch raw and alpha return for ticker over holding_days after trade_date. + + The position is entered at the first close after ``trade_date``: a run dated + ``trade_date`` reads news published after that day's close (#1518). ``benchmark`` is the index used as the alpha baseline (resolved by the caller via ``resolve_benchmark``). Returns ``(raw_return, alpha_return, @@ -64,13 +67,15 @@ def fetch_returns( to report. """ start = datetime.strptime(trade_date, "%Y-%m-%d") - # holding_days counts trading days, so ask for the calendar span they - # occupy (about 7 for every 5) plus a week for holidays. - end = start + timedelta(days=round(holding_days * 7 / 5) + 7) + # holding_days counts trading days after the entry session, so ask for the + # calendar span they occupy (about 7 for every 5) plus a week for holidays. + end = start + timedelta(days=round((holding_days + 1) * 7 / 5) + 7) end_str = end.strftime("%Y-%m-%d") - # Closes for the instrument the analysis priced (XAUUSD -> GC=F, #984). + # Closes for the instrument the analysis priced (XAUUSD -> GC=F, #984), + # from the first session after the decision date. stock = _by_day(get_closes(ticker, trade_date, end_str)) + stock = stock[stock.index > start] # From a week earlier, so the benchmark has a close on or before entry. bench_start = (start - timedelta(days=7)).strftime("%Y-%m-%d") bench = _by_day(get_closes(benchmark, bench_start, end_str))