feat(backtest): show each cell as it starts, and the run id to continue with

- run_backtest takes a progress(done, total, ticker, date) callback; tradingagents backtest prints [n/total] per cell and ends with the --run-id that continues or settles the sweep
This commit is contained in:
Yijia-Xiao
2026-09-29 05:57:17 +00:00
parent 052a98d6eb
commit f4828b84ea
4 changed files with 59 additions and 13 deletions
+5 -1
View File
@@ -125,7 +125,10 @@ def backtest(
console.print("[red]No ticker to analyze; pass them comma-separated, e.g. NVDA,AAPL[/red]")
raise typer.Exit(code=1)
kwargs = {"asset_type": asset_type, "portfolio": book, "run_id": run_id}
def show_progress(done, total, ticker, date):
console.print(f"[dim][{done}/{total}] {ticker} {date}[/dim]")
kwargs = {"asset_type": asset_type, "portfolio": book, "run_id": run_id, "progress": show_progress}
if analysts:
kwargs["selected_analysts"] = [a.strip().lower() for a in analysts.split(",") if a.strip()]
@@ -136,6 +139,7 @@ def backtest(
raise typer.Exit(code=1) from None
console.print(summarize(result).render())
console.print(f"\nRan {result.cells_run} cells, skipped {result.skipped}. Log: {result.log_path}")
console.print(f"Continue or settle this sweep: --run-id {result.run_id}")
for ticker, date, reason in result.failures:
console.print(f"[yellow]failed:[/yellow] {ticker} {date}: {reason}")
for ticker, reason in result.settlement_failures:
+22
View File
@@ -274,3 +274,25 @@ def test_a_result_whose_cells_all_failed_summarizes_as_empty(tmp_path):
result = BacktestResult(run_id="r", log_path=tmp_path / "never-written.md")
assert summarize(result).resolved == 0
@pytest.mark.unit
def test_progress_is_reported_before_each_cell(tmp_path):
seen = []
run_backtest(["NVDA", "AAPL"], ["2026-01-05", "2026-01-12"], _config(tmp_path),
progress=lambda done, total, ticker, date: seen.append((done, total, ticker, date)))
assert seen == [(1, 4, "NVDA", "2026-01-05"), (2, 4, "NVDA", "2026-01-12"),
(3, 4, "AAPL", "2026-01-05"), (4, 4, "AAPL", "2026-01-12")]
@pytest.mark.unit
def test_a_resumed_sweep_reports_only_the_cells_it_runs(tmp_path):
first = run_backtest(["NVDA"], ["2026-01-05"], _config(tmp_path))
seen = []
run_backtest(["NVDA"], ["2026-01-05", "2026-01-12"], _config(tmp_path), run_id=first.run_id,
progress=lambda done, total, ticker, date: seen.append((done, total, date)))
assert seen == [(1, 1, "2026-01-12")]
+16
View File
@@ -135,3 +135,19 @@ def test_backtest_reports_a_setup_failure_in_one_line(runner, monkeypatch):
assert result.exit_code == 1
assert "API key" in result.output
assert "Traceback" not in result.output
@pytest.mark.unit
def test_backtest_shows_each_cell_and_how_to_continue(runner, monkeypatch, tmp_path):
def sweep(tickers, dates, config, progress=None, **kw):
for i, date in enumerate(dates, 1):
progress(i, len(dates), tickers[0], date)
return _Result(tmp_path)
monkeypatch.setattr(m, "run_backtest", sweep)
monkeypatch.setattr(m, "summarize", lambda log: _Summary())
result = runner.invoke(m.app, ["backtest", "NVDA", "--start", "2026-06-01", "--end", "2026-06-08"])
assert "[1/2] NVDA 2026-06-01" in result.output and "[2/2] NVDA 2026-06-08" in result.output
assert f"--run-id {_Result(tmp_path).run_id}" in result.output
+16 -12
View File
@@ -17,6 +17,7 @@ cell rather than a position carried forward.
from __future__ import annotations
import logging
from collections.abc import Callable
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from pathlib import Path
@@ -133,12 +134,14 @@ def run_backtest(
portfolio=None,
selected_analysts=("market", "social", "news", "fundamentals"),
run_id: str | None = None,
progress: Callable[[int, int, str, str], None] | None = None,
) -> BacktestResult:
"""Analyze every ticker on every date, into a memory log of this run's own.
The live log stays untouched: a sweep would otherwise flood the context that
real runs read back. Cells already in this run's log are skipped, so an
interrupted sweep resumes by being run again.
interrupted sweep resumes by being run again. ``progress(done, total,
ticker, date)`` is called before each cell that runs.
"""
# run_id becomes a path segment, so it is validated like a ticker: an
# absolute or dotted value would otherwise place the run outside results_dir.
@@ -152,17 +155,18 @@ def run_backtest(
result = BacktestResult(run_id=run_id, log_path=Path(run_config["memory_log_path"]))
done = {(e["ticker"], e["date"]) for e in graph.memory_log.load_entries()}
for ticker in tickers:
for date in dates:
if (ticker, date) in done:
result.skipped += 1
continue
try:
graph.propagate(ticker, date, asset_type, portfolio=portfolio)
result.cells_run += 1
except Exception as exc: # one unreachable vendor must not end the sweep
logger.warning("Backtest cell %s %s failed: %s", ticker, date, exc)
result.failures.append((ticker, date, str(exc)))
cells = [(ticker, date) for ticker in tickers for date in dates]
todo = [cell for cell in cells if cell not in done]
result.skipped = len(cells) - len(todo)
for index, (ticker, date) in enumerate(todo, 1):
if progress:
progress(index, len(todo), ticker, date)
try:
graph.propagate(ticker, date, asset_type, portfolio=portfolio)
result.cells_run += 1
except Exception as exc: # one unreachable vendor must not end the sweep
logger.warning("Backtest cell %s %s failed: %s", ticker, date, exc)
result.failures.append((ticker, date, str(exc)))
# Settlement runs at the start of the next run for a ticker, so each ticker's
# last cell would stay pending without this pass.