fix(backtest): score a decision against the direction it claimed

- a Sell that fell counted as a miss and a Sell that rose as a hit
- Hold claims no direction, so it reports mean alpha and no hit rate
- the report names the window the outcomes were measured over

tests/test_backtest.py

@@ -186,3 +186,63 @@ def test_pending_note_appears_only_when_something_is_pending(tmp_path):
     settled = [("NVDA", "2026-01-05", DECISION, (0.1, 0.05))]
     assert "Pending" not in summarize(_log_with(tmp_path, settled)).render()
     assert "Pending" in summarize(_log_with(tmp_path, settled + [("AAPL", "2026-01-05", DECISION, None)])).render()
+
+
+# --- scoring reads the direction the rating claimed ---------------------------
+
+def _scored(tmp_path, rows):
+    log = _log_with(tmp_path, rows)
+    return summarize(log).by_rating
+
+
+@pytest.mark.unit
+def test_a_bearish_call_that_fell_counts_as_right(tmp_path):
+    """Alpha below the benchmark is the outcome a Sell predicted; scoring it as
+    a miss reported the system as wrong exactly when it was right."""
+    scores = _scored(tmp_path, [
+        ("NVDA", "2026-01-05", "**Rating**: Sell\n\nx", (-0.08, -0.05)),
+        ("AAPL", "2026-01-05", "**Rating**: Underweight\n\nx", (-0.03, -0.02)),
+    ])
+    assert scores["Sell"].hit_rate == 1.0
+    assert scores["Underweight"].hit_rate == 1.0
+
+
+@pytest.mark.unit
+def test_a_bearish_call_that_rose_counts_as_wrong(tmp_path):
+    scores = _scored(tmp_path, [("NVDA", "2026-01-05", "**Rating**: Sell\n\nx", (0.08, 0.05))])
+    assert scores["Sell"].hit_rate == 0.0
+
+
+@pytest.mark.unit
+def test_a_bullish_call_is_scored_the_same_way_as_before(tmp_path):
+    scores = _scored(tmp_path, [
+        ("NVDA", "2026-01-05", "**Rating**: Buy\n\nx", (0.10, 0.04)),
+        ("AAPL", "2026-01-05", "**Rating**: Buy\n\nx", (-0.02, -0.02)),
+    ])
+    assert scores["Buy"].hit_rate == 0.5
+
+
+@pytest.mark.unit
+def test_hold_claims_no_direction_so_it_gets_no_hit_rate(tmp_path):
+    scores = _scored(tmp_path, [("NVDA", "2026-01-05", "**Rating**: Hold\n\nx", (0.01, 0.005))])
+    assert scores["Hold"].hit_rate is None
+    assert scores["Hold"].mean_alpha == 0.005
+
+
+@pytest.mark.unit
+def test_the_report_names_the_window_the_scores_cover(tmp_path):
+    text = summarize(_log_with(tmp_path, [
+        ("NVDA", "2026-01-05", "**Rating**: Buy\n\nx", (0.1, 0.05))])).render()
+    assert "5" in text and "day" in text.lower()
+    assert "Hold" not in text or "no direction" in text.lower()
+
+
+@pytest.mark.unit
+def test_the_window_reported_is_the_one_the_outcomes_used(tmp_path):
+    """The log records the window each outcome was measured over; the summary
+    must not claim a different one."""
+    log = TradingMemoryLog({"memory_log_path": str(tmp_path / "m.md")})
+    log.store_decision("NVDA", "2026-01-05", "**Rating**: Buy\n\nx")
+    log.update_with_outcome("NVDA", "2026-01-05", 0.1, 0.04, 21, "note", "2026-02-01")
+
+    assert "21 trading days" in summarize(log).render()

tradingagents/backtest.py

@@ -81,10 +81,15 @@ class BacktestResult:
     settlement_failures: list[tuple[str, str]] = field(default_factory=list)
 
 
+# What each rating claims will happen, so an outcome can be scored against it.
+# Hold claims no direction, so nothing about alpha proves it right or wrong.
+_DIRECTION = {"Buy": 1, "Overweight": 1, "Hold": 0, "Underweight": -1, "Sell": -1}
+
+
 @dataclass
 class RatingScore:
     count: int
-    hit_rate: float
+    hit_rate: float | None
     mean_alpha: float
 
 
@@ -94,19 +99,23 @@ class BacktestSummary:
     pending: int
     by_rating: dict[str, RatingScore]
     unscored: int = 0
+    holding: str = ""
 
     def render(self) -> str:
         lines = [f"Resolved cells: {self.resolved} · pending: {self.pending}"
                  + (f" · unscored: {self.unscored}" if self.unscored else "")]
         for rating, score in self.by_rating.items():
+            called = (f"called the direction {score.hit_rate:.0%}"
+                      if score.hit_rate is not None else "no direction claimed")
             lines.append(
-                f"- {rating}: n={score.count}, beat the benchmark "
-                f"{score.hit_rate:.0%}, mean alpha {score.mean_alpha:+.2%}"
+                f"- {rating}: n={score.count}, {called}, "
+                f"mean alpha {score.mean_alpha:+.2%} vs the benchmark"
             )
         lines.append("")
         if self.pending:
             lines.append("Pending cells are not scored above; re-run to settle them.")
         lines.append(
+            f"Alpha is measured over {self.holding} after each analysis date. "
             "One model sampling per cell, and text feeds are not archived, so "
             "these figures are indicative rather than repeatable."
         )
@@ -174,12 +183,17 @@ def summarize(memory_log: TradingMemoryLog) -> BacktestSummary:
     by_rating: dict[str, RatingScore] = {}
     for rating in dict.fromkeys(e["rating"] for e, _ in resolved):
         alphas = [a for e, a in resolved if e["rating"] == rating]
+        direction = _DIRECTION.get(rating, 0)
         by_rating[rating] = RatingScore(
             count=len(alphas),
-            hit_rate=sum(a > 0 for a in alphas) / len(alphas),
+            hit_rate=(sum(a * direction > 0 for a in alphas) / len(alphas)) if direction else None,
             mean_alpha=sum(alphas) / len(alphas),
         )
     unscored = sum(1 for e in entries if e["rating"] == RATING_REVIEW)
+    # Report the window the outcomes were actually measured over, from the log.
+    windows = {f"{e['holding'][:-1]} trading days" for e, _ in resolved
+               if (e.get("holding") or "").endswith("d")}
     return BacktestSummary(resolved=len(resolved),
                            pending=len(entries) - len(resolved) - unscored,
-                           by_rating=by_rating, unscored=unscored)
+                           by_rating=by_rating, unscored=unscored,
+                           holding=", ".join(sorted(windows)) or "the configured window")