diff --git a/tests/test_backtest.py b/tests/test_backtest.py index 08aff1e08..cebb63134 100644 --- a/tests/test_backtest.py +++ b/tests/test_backtest.py @@ -186,3 +186,63 @@ def test_pending_note_appears_only_when_something_is_pending(tmp_path): settled = [("NVDA", "2026-01-05", DECISION, (0.1, 0.05))] assert "Pending" not in summarize(_log_with(tmp_path, settled)).render() assert "Pending" in summarize(_log_with(tmp_path, settled + [("AAPL", "2026-01-05", DECISION, None)])).render() + + +# --- scoring reads the direction the rating claimed --------------------------- + +def _scored(tmp_path, rows): + log = _log_with(tmp_path, rows) + return summarize(log).by_rating + + +@pytest.mark.unit +def test_a_bearish_call_that_fell_counts_as_right(tmp_path): + """Alpha below the benchmark is the outcome a Sell predicted; scoring it as + a miss reported the system as wrong exactly when it was right.""" + scores = _scored(tmp_path, [ + ("NVDA", "2026-01-05", "**Rating**: Sell\n\nx", (-0.08, -0.05)), + ("AAPL", "2026-01-05", "**Rating**: Underweight\n\nx", (-0.03, -0.02)), + ]) + assert scores["Sell"].hit_rate == 1.0 + assert scores["Underweight"].hit_rate == 1.0 + + +@pytest.mark.unit +def test_a_bearish_call_that_rose_counts_as_wrong(tmp_path): + scores = _scored(tmp_path, [("NVDA", "2026-01-05", "**Rating**: Sell\n\nx", (0.08, 0.05))]) + assert scores["Sell"].hit_rate == 0.0 + + +@pytest.mark.unit +def test_a_bullish_call_is_scored_the_same_way_as_before(tmp_path): + scores = _scored(tmp_path, [ + ("NVDA", "2026-01-05", "**Rating**: Buy\n\nx", (0.10, 0.04)), + ("AAPL", "2026-01-05", "**Rating**: Buy\n\nx", (-0.02, -0.02)), + ]) + assert scores["Buy"].hit_rate == 0.5 + + +@pytest.mark.unit +def test_hold_claims_no_direction_so_it_gets_no_hit_rate(tmp_path): + scores = _scored(tmp_path, [("NVDA", "2026-01-05", "**Rating**: Hold\n\nx", (0.01, 0.005))]) + assert scores["Hold"].hit_rate is None + assert scores["Hold"].mean_alpha == 0.005 + + +@pytest.mark.unit +def test_the_report_names_the_window_the_scores_cover(tmp_path): + text = summarize(_log_with(tmp_path, [ + ("NVDA", "2026-01-05", "**Rating**: Buy\n\nx", (0.1, 0.05))])).render() + assert "5" in text and "day" in text.lower() + assert "Hold" not in text or "no direction" in text.lower() + + +@pytest.mark.unit +def test_the_window_reported_is_the_one_the_outcomes_used(tmp_path): + """The log records the window each outcome was measured over; the summary + must not claim a different one.""" + log = TradingMemoryLog({"memory_log_path": str(tmp_path / "m.md")}) + log.store_decision("NVDA", "2026-01-05", "**Rating**: Buy\n\nx") + log.update_with_outcome("NVDA", "2026-01-05", 0.1, 0.04, 21, "note", "2026-02-01") + + assert "21 trading days" in summarize(log).render() diff --git a/tradingagents/backtest.py b/tradingagents/backtest.py index 62ef81ef7..c8d28345a 100644 --- a/tradingagents/backtest.py +++ b/tradingagents/backtest.py @@ -81,10 +81,15 @@ class BacktestResult: settlement_failures: list[tuple[str, str]] = field(default_factory=list) +# What each rating claims will happen, so an outcome can be scored against it. +# Hold claims no direction, so nothing about alpha proves it right or wrong. +_DIRECTION = {"Buy": 1, "Overweight": 1, "Hold": 0, "Underweight": -1, "Sell": -1} + + @dataclass class RatingScore: count: int - hit_rate: float + hit_rate: float | None mean_alpha: float @@ -94,19 +99,23 @@ class BacktestSummary: pending: int by_rating: dict[str, RatingScore] unscored: int = 0 + holding: str = "" def render(self) -> str: lines = [f"Resolved cells: {self.resolved} · pending: {self.pending}" + (f" · unscored: {self.unscored}" if self.unscored else "")] for rating, score in self.by_rating.items(): + called = (f"called the direction {score.hit_rate:.0%}" + if score.hit_rate is not None else "no direction claimed") lines.append( - f"- {rating}: n={score.count}, beat the benchmark " - f"{score.hit_rate:.0%}, mean alpha {score.mean_alpha:+.2%}" + f"- {rating}: n={score.count}, {called}, " + f"mean alpha {score.mean_alpha:+.2%} vs the benchmark" ) lines.append("") if self.pending: lines.append("Pending cells are not scored above; re-run to settle them.") lines.append( + f"Alpha is measured over {self.holding} after each analysis date. " "One model sampling per cell, and text feeds are not archived, so " "these figures are indicative rather than repeatable." ) @@ -174,12 +183,17 @@ def summarize(memory_log: TradingMemoryLog) -> BacktestSummary: by_rating: dict[str, RatingScore] = {} for rating in dict.fromkeys(e["rating"] for e, _ in resolved): alphas = [a for e, a in resolved if e["rating"] == rating] + direction = _DIRECTION.get(rating, 0) by_rating[rating] = RatingScore( count=len(alphas), - hit_rate=sum(a > 0 for a in alphas) / len(alphas), + hit_rate=(sum(a * direction > 0 for a in alphas) / len(alphas)) if direction else None, mean_alpha=sum(alphas) / len(alphas), ) unscored = sum(1 for e in entries if e["rating"] == RATING_REVIEW) + # Report the window the outcomes were actually measured over, from the log. + windows = {f"{e['holding'][:-1]} trading days" for e, _ in resolved + if (e.get("holding") or "").endswith("d")} return BacktestSummary(resolved=len(resolved), pending=len(entries) - len(resolved) - unscored, - by_rating=by_rating, unscored=unscored) + by_rating=by_rating, unscored=unscored, + holding=", ".join(sorted(windows)) or "the configured window")