feat: evaluate decisions over a grid of tickers and dates (#1234)

- run_backtest runs the pipeline per cell into its own decision log and resumes by skipping logged cells
- summarize scores settled cells by rating on realized alpha
- settle_pending settles a ticker whose last decision would otherwise stay open

tests/test_backtest.py

@@ -0,0 +1,181 @@
+"""Backtesting: many single-shot decisions, scored by the decision log.
+
+A run already records its rating and later settles it with realized and alpha
+return against the regional benchmark. A backtest is that machinery over a grid
+of tickers and dates, aggregated. It evaluates decision quality; it does not
+simulate a portfolio, so there is no execution, no fees and no equity curve.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from tradingagents.agents.utils.memory import TradingMemoryLog
+from tradingagents.backtest import iter_grid, run_backtest, summarize
+
+DECISION = "Rating: Buy\n\nbuy it"
+
+
+@pytest.mark.unit
+def test_grid_spacing_and_canonical_dates():
+    assert iter_grid("2026-01-05", "2026-01-20", every_n_days=7) == ["2026-01-05", "2026-01-12", "2026-01-19"]
+
+
+@pytest.mark.unit
+def test_grid_stops_at_today(monkeypatch):
+    import tradingagents.backtest as bt
+
+    monkeypatch.setattr(bt, "get_current_date", lambda: "2026-01-10")
+    assert iter_grid("2026-01-05", "2026-02-20", every_n_days=5) == ["2026-01-05", "2026-01-10"]
+
+
+@pytest.mark.unit
+def test_grid_rejects_a_non_canonical_date():
+    with pytest.raises(ValueError, match="YYYY-MM-DD"):
+        iter_grid("2026-1-5", "2026-01-20")
+
+
+class _FakeGraph:
+    """Stands in for TradingAgentsGraph, writing to the log the harness gave it."""
+
+    instances: list = []
+    fail_on: set = set()
+
+    def __init__(self, selected_analysts=None, config=None, **kw):
+        self.analysts = list(selected_analysts) if selected_analysts else None
+        self.config = config
+        self.memory_log = TradingMemoryLog(config)
+        self.calls = []
+        self.settled = []
+        _FakeGraph.instances.append(self)
+
+    def propagate(self, ticker, trade_date, asset_type="stock", portfolio=None):
+        self.calls.append((ticker, trade_date))
+        if (ticker, trade_date) in _FakeGraph.fail_on:
+            raise RuntimeError("vendor exploded")
+        self.memory_log.store_decision(ticker, trade_date, DECISION)
+        return {"final_trade_decision": DECISION}, "Buy"
+
+    def settle_pending(self, ticker):
+        self.settled.append(ticker)
+
+
+@pytest.fixture(autouse=True)
+def _fake_graph(monkeypatch, tmp_path):
+    import tradingagents.backtest as bt
+
+    _FakeGraph.instances = []
+    _FakeGraph.fail_on = set()
+    monkeypatch.setattr(bt, "TradingAgentsGraph", _FakeGraph)
+    return _FakeGraph
+
+
+def _config(tmp_path):
+    return {"results_dir": str(tmp_path / "results"),
+            "memory_log_path": str(tmp_path / "live_trading_memory.md")}
+
+
+@pytest.mark.unit
+def test_the_live_decision_log_is_never_written(tmp_path):
+    config = _config(tmp_path)
+    result = run_backtest(["NVDA"], ["2026-01-05", "2026-01-12"], config)
+
+    assert not (tmp_path / "live_trading_memory.md").exists()
+    assert result.log_path.exists() and result.cells_run == 2
+
+
+@pytest.mark.unit
+def test_a_cell_already_in_the_log_is_not_run_again(tmp_path):
+    config = _config(tmp_path)
+    first = run_backtest(["NVDA"], ["2026-01-05"], config)
+
+    again = run_backtest(["NVDA"], ["2026-01-05", "2026-01-12"], config, run_id=first.run_id)
+
+    assert again.cells_run == 1 and again.skipped == 1
+    assert _FakeGraph.instances[-1].calls == [("NVDA", "2026-01-12")]
+
+
+@pytest.mark.unit
+def test_every_ticker_is_settled_after_the_grid(tmp_path):
+    """Settlement runs at the start of the next same-ticker run, so the last
+    date of each ticker would stay pending without an explicit pass."""
+    run_backtest(["NVDA", "AAPL"], ["2026-01-05", "2026-01-12"], _config(tmp_path))
+    assert sorted(_FakeGraph.instances[-1].settled) == ["AAPL", "NVDA"]
+
+
+@pytest.mark.unit
+def test_a_failed_cell_does_not_abort_the_sweep(tmp_path):
+    _FakeGraph.fail_on = {("NVDA", "2026-01-05")}
+    result = run_backtest(["NVDA"], ["2026-01-05", "2026-01-12"], _config(tmp_path))
+
+    assert result.cells_run == 1
+    assert result.failures == [("NVDA", "2026-01-05", "vendor exploded")]
+
+
+# --- reading the result ------------------------------------------------------
+
+def _log_with(tmp_path, rows):
+    log = TradingMemoryLog({"memory_log_path": str(tmp_path / "m.md")})
+    for ticker, date, decision, outcome in rows:
+        log.store_decision(ticker, date, decision)
+        if outcome is not None:
+            log.update_with_outcome(ticker, date, outcome[0], outcome[1], 5, "note", "2026-02-01")
+    return log
+
+
+@pytest.mark.unit
+def test_summary_scores_resolved_cells_and_keeps_pending_out_of_the_average(tmp_path):
+    log = _log_with(tmp_path, [
+        ("NVDA", "2026-01-05", "Rating: Buy\n\nx", (0.10, 0.04)),
+        ("NVDA", "2026-01-12", "Rating: Buy\n\nx", (-0.02, -0.02)),
+        ("AAPL", "2026-01-05", "Rating: Sell\n\nx", None),
+    ])
+
+    summary = summarize(log)
+
+    assert summary.resolved == 2 and summary.pending == 1
+    buys = summary.by_rating["Buy"]
+    assert buys.count == 2 and buys.hit_rate == 0.5 and round(buys.mean_alpha, 4) == 0.01
+    assert "Sell" not in summary.by_rating  # unsettled: nothing to score yet
+
+
+@pytest.mark.unit
+def test_summary_states_what_it_cannot_prove(tmp_path):
+    text = summarize(_log_with(tmp_path, [("NVDA", "2026-01-05", DECISION, (0.1, 0.05))])).render()
+    assert "not archived" in text
+    assert "one" in text.lower() and "sampl" in text.lower()
+
+
+@pytest.mark.unit
+def test_the_analyst_set_under_test_is_the_one_that_runs(tmp_path):
+    """A backtest of a two-analyst setup must not silently run four."""
+    run_backtest(["NVDA"], ["2026-01-05"], _config(tmp_path), selected_analysts=["market", "news"])
+    assert _FakeGraph.instances[-1].analysts == ["market", "news"]
+
+
+@pytest.mark.unit
+def test_a_run_id_cannot_escape_the_results_directory(tmp_path):
+    """run_id becomes a path segment, so it is validated like a ticker is."""
+    with pytest.raises(ValueError):
+        run_backtest(["NVDA"], ["2026-01-05"], _config(tmp_path), run_id="../../escaped")
+    with pytest.raises(ValueError):
+        run_backtest(["NVDA"], ["2026-01-05"], _config(tmp_path), run_id="/etc/cron.d/x")
+
+
+@pytest.mark.unit
+def test_a_failed_settlement_does_not_lose_the_remaining_tickers(tmp_path, monkeypatch):
+    """Settlement reflects with an LLM, so it can fail; the sweep still returns
+    its result and every other ticker still gets settled."""
+    settled = []
+
+    def _settle(self, ticker):
+        if ticker == "NVDA":
+            raise RuntimeError("reflector timed out")
+        settled.append(ticker)
+
+    monkeypatch.setattr(_FakeGraph, "settle_pending", _settle, raising=False)
+    result = run_backtest(["NVDA", "AAPL"], ["2026-01-05"], _config(tmp_path))
+
+    assert result.cells_run == 2
+    assert settled == ["AAPL"]
+    assert result.settlement_failures == [("NVDA", "reflector timed out")]