From dcbff5ea428b39ad99e63db0a0ff099c2b5ffa73 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Wed, 23 Sep 2026 23:24:36 +0000 Subject: [PATCH] feat(backtest): summarize a backtest result, or a decision log by its path - callers no longer construct TradingMemoryLog to score a run; README and CLI pass the result --- README.md | 3 +-- cli/main.py | 3 +-- tests/test_backtest.py | 32 ++++++++++++++++++++++++++++++-- tests/test_rating_integrity.py | 2 +- tradingagents/backtest.py | 12 +++++++++--- 5 files changed, 42 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 6cd125f3e..54213bb4e 100644 --- a/README.md +++ b/README.md @@ -325,11 +325,10 @@ One run gives one decision, which cannot tell you whether the system decides wel ```python from tradingagents.backtest import iter_grid, run_backtest, summarize -from tradingagents.agents.utils.memory import TradingMemoryLog dates = iter_grid("2026-06-01", "2026-08-01", every_n_days=7) result = run_backtest(["NVDA", "AAPL"], dates, config, selected_analysts=["market", "news"]) -print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render()) +print(summarize(result).render()) ``` From the CLI: diff --git a/cli/main.py b/cli/main.py index 4c792928a..db3dea86d 100644 --- a/cli/main.py +++ b/cli/main.py @@ -1372,7 +1372,6 @@ def backtest( ), ): """Score past decisions over a grid of tickers and dates.""" - from tradingagents.agents.utils.memory import TradingMemoryLog try: dates = iter_grid(start, end, every) @@ -1395,7 +1394,7 @@ def backtest( except Exception as exc: # a missing key or an unknown analyst is a setup error console.print(f"[red]{exc}[/red]") raise typer.Exit(code=1) from None - console.print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render()) + console.print(summarize(result).render()) console.print(f"\nRan {result.cells_run} cells, skipped {result.skipped}. Log: {result.log_path}") for ticker, date, reason in result.failures: console.print(f"[yellow]failed:[/yellow] {ticker} {date}: {reason}") diff --git a/tests/test_backtest.py b/tests/test_backtest.py index cebb63134..fb32e66ec 100644 --- a/tests/test_backtest.py +++ b/tests/test_backtest.py @@ -120,7 +120,7 @@ def _log_with(tmp_path, rows): log.store_decision(ticker, date, decision) if outcome is not None: log.update_with_outcome(ticker, date, outcome[0], outcome[1], 5, "note", "2026-02-01") - return log + return tmp_path / "m.md" @pytest.mark.unit @@ -245,4 +245,32 @@ def test_the_window_reported_is_the_one_the_outcomes_used(tmp_path): log.store_decision("NVDA", "2026-01-05", "**Rating**: Buy\n\nx") log.update_with_outcome("NVDA", "2026-01-05", 0.1, 0.04, 21, "note", "2026-02-01") - assert "21 trading days" in summarize(log).render() + assert "21 trading days" in summarize(tmp_path / "m.md").render() + + +@pytest.mark.unit +def test_a_backtest_result_is_summarized_directly(tmp_path): + """The result names its own log, so a caller never builds the log to score it.""" + from tradingagents.backtest import BacktestResult + + path = _log_with(tmp_path, [("NVDA", "2026-01-05", "Rating: Buy\n\nx", (0.10, 0.04))]) + + assert summarize(BacktestResult(run_id="r", log_path=path)).resolved == 1 + + +@pytest.mark.unit +def test_a_log_path_that_does_not_exist_is_an_error_not_an_empty_summary(tmp_path): + missing = tmp_path / "no-such-dir" / "m.md" + + with pytest.raises(FileNotFoundError): + summarize(missing) + assert not missing.parent.exists() + + +@pytest.mark.unit +def test_a_result_whose_cells_all_failed_summarizes_as_empty(tmp_path): + from tradingagents.backtest import BacktestResult + + result = BacktestResult(run_id="r", log_path=tmp_path / "never-written.md") + + assert summarize(result).resolved == 0 diff --git a/tests/test_rating_integrity.py b/tests/test_rating_integrity.py index 55a5e5627..77e8f14b9 100644 --- a/tests/test_rating_integrity.py +++ b/tests/test_rating_integrity.py @@ -98,7 +98,7 @@ def test_an_unscored_decision_is_left_out_of_the_backtest_figures(tmp_path): log.store_decision("AAPL", "2026-01-05", REFUSAL) log.update_with_outcome("AAPL", "2026-01-05", 0.1, 0.04, 5, "note", "2026-02-01") - summary = summarize(log) + summary = summarize(tmp_path / "m.md") assert set(summary.by_rating) == {"Buy"} diff --git a/tradingagents/backtest.py b/tradingagents/backtest.py index c88844035..afa0b409b 100644 --- a/tradingagents/backtest.py +++ b/tradingagents/backtest.py @@ -175,9 +175,15 @@ def run_backtest( return result -def summarize(memory_log: TradingMemoryLog) -> BacktestSummary: - """Score the settled decisions in a log, by rating.""" - entries = memory_log.load_entries() +def summarize(source: BacktestResult | str | Path) -> BacktestSummary: + """Score the settled decisions of a backtest, or of a decision log at a path, by rating.""" + if isinstance(source, BacktestResult): + path = source.log_path # a run whose cells all failed wrote no log: nothing to score + elif Path(source).is_file(): + path = Path(source) + else: + raise FileNotFoundError(f"no decision log at {source}") + entries = TradingMemoryLog({"memory_log_path": str(path)}).load_entries() # A decision with no readable rating has no direction, so it can neither # count for nor against the system; it is reported as unscored instead. resolved = [(e, _alpha(e)) for e in entries