mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-26 14:32:28 +03:00
feat(backtest): summarize a backtest result, or a decision log by its path
- callers no longer construct TradingMemoryLog to score a run; README and CLI pass the result
This commit is contained in:
@@ -325,11 +325,10 @@ One run gives one decision, which cannot tell you whether the system decides wel
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
from tradingagents.backtest import iter_grid, run_backtest, summarize
|
from tradingagents.backtest import iter_grid, run_backtest, summarize
|
||||||
from tradingagents.agents.utils.memory import TradingMemoryLog
|
|
||||||
|
|
||||||
dates = iter_grid("2026-06-01", "2026-08-01", every_n_days=7)
|
dates = iter_grid("2026-06-01", "2026-08-01", every_n_days=7)
|
||||||
result = run_backtest(["NVDA", "AAPL"], dates, config, selected_analysts=["market", "news"])
|
result = run_backtest(["NVDA", "AAPL"], dates, config, selected_analysts=["market", "news"])
|
||||||
print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render())
|
print(summarize(result).render())
|
||||||
```
|
```
|
||||||
|
|
||||||
From the CLI:
|
From the CLI:
|
||||||
|
|||||||
+1
-2
@@ -1372,7 +1372,6 @@ def backtest(
|
|||||||
),
|
),
|
||||||
):
|
):
|
||||||
"""Score past decisions over a grid of tickers and dates."""
|
"""Score past decisions over a grid of tickers and dates."""
|
||||||
from tradingagents.agents.utils.memory import TradingMemoryLog
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
dates = iter_grid(start, end, every)
|
dates = iter_grid(start, end, every)
|
||||||
@@ -1395,7 +1394,7 @@ def backtest(
|
|||||||
except Exception as exc: # a missing key or an unknown analyst is a setup error
|
except Exception as exc: # a missing key or an unknown analyst is a setup error
|
||||||
console.print(f"[red]{exc}[/red]")
|
console.print(f"[red]{exc}[/red]")
|
||||||
raise typer.Exit(code=1) from None
|
raise typer.Exit(code=1) from None
|
||||||
console.print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render())
|
console.print(summarize(result).render())
|
||||||
console.print(f"\nRan {result.cells_run} cells, skipped {result.skipped}. Log: {result.log_path}")
|
console.print(f"\nRan {result.cells_run} cells, skipped {result.skipped}. Log: {result.log_path}")
|
||||||
for ticker, date, reason in result.failures:
|
for ticker, date, reason in result.failures:
|
||||||
console.print(f"[yellow]failed:[/yellow] {ticker} {date}: {reason}")
|
console.print(f"[yellow]failed:[/yellow] {ticker} {date}: {reason}")
|
||||||
|
|||||||
+30
-2
@@ -120,7 +120,7 @@ def _log_with(tmp_path, rows):
|
|||||||
log.store_decision(ticker, date, decision)
|
log.store_decision(ticker, date, decision)
|
||||||
if outcome is not None:
|
if outcome is not None:
|
||||||
log.update_with_outcome(ticker, date, outcome[0], outcome[1], 5, "note", "2026-02-01")
|
log.update_with_outcome(ticker, date, outcome[0], outcome[1], 5, "note", "2026-02-01")
|
||||||
return log
|
return tmp_path / "m.md"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.unit
|
@pytest.mark.unit
|
||||||
@@ -245,4 +245,32 @@ def test_the_window_reported_is_the_one_the_outcomes_used(tmp_path):
|
|||||||
log.store_decision("NVDA", "2026-01-05", "**Rating**: Buy\n\nx")
|
log.store_decision("NVDA", "2026-01-05", "**Rating**: Buy\n\nx")
|
||||||
log.update_with_outcome("NVDA", "2026-01-05", 0.1, 0.04, 21, "note", "2026-02-01")
|
log.update_with_outcome("NVDA", "2026-01-05", 0.1, 0.04, 21, "note", "2026-02-01")
|
||||||
|
|
||||||
assert "21 trading days" in summarize(log).render()
|
assert "21 trading days" in summarize(tmp_path / "m.md").render()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_a_backtest_result_is_summarized_directly(tmp_path):
|
||||||
|
"""The result names its own log, so a caller never builds the log to score it."""
|
||||||
|
from tradingagents.backtest import BacktestResult
|
||||||
|
|
||||||
|
path = _log_with(tmp_path, [("NVDA", "2026-01-05", "Rating: Buy\n\nx", (0.10, 0.04))])
|
||||||
|
|
||||||
|
assert summarize(BacktestResult(run_id="r", log_path=path)).resolved == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_a_log_path_that_does_not_exist_is_an_error_not_an_empty_summary(tmp_path):
|
||||||
|
missing = tmp_path / "no-such-dir" / "m.md"
|
||||||
|
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
summarize(missing)
|
||||||
|
assert not missing.parent.exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_a_result_whose_cells_all_failed_summarizes_as_empty(tmp_path):
|
||||||
|
from tradingagents.backtest import BacktestResult
|
||||||
|
|
||||||
|
result = BacktestResult(run_id="r", log_path=tmp_path / "never-written.md")
|
||||||
|
|
||||||
|
assert summarize(result).resolved == 0
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ def test_an_unscored_decision_is_left_out_of_the_backtest_figures(tmp_path):
|
|||||||
log.store_decision("AAPL", "2026-01-05", REFUSAL)
|
log.store_decision("AAPL", "2026-01-05", REFUSAL)
|
||||||
log.update_with_outcome("AAPL", "2026-01-05", 0.1, 0.04, 5, "note", "2026-02-01")
|
log.update_with_outcome("AAPL", "2026-01-05", 0.1, 0.04, 5, "note", "2026-02-01")
|
||||||
|
|
||||||
summary = summarize(log)
|
summary = summarize(tmp_path / "m.md")
|
||||||
assert set(summary.by_rating) == {"Buy"}
|
assert set(summary.by_rating) == {"Buy"}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -175,9 +175,15 @@ def run_backtest(
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def summarize(memory_log: TradingMemoryLog) -> BacktestSummary:
|
def summarize(source: BacktestResult | str | Path) -> BacktestSummary:
|
||||||
"""Score the settled decisions in a log, by rating."""
|
"""Score the settled decisions of a backtest, or of a decision log at a path, by rating."""
|
||||||
entries = memory_log.load_entries()
|
if isinstance(source, BacktestResult):
|
||||||
|
path = source.log_path # a run whose cells all failed wrote no log: nothing to score
|
||||||
|
elif Path(source).is_file():
|
||||||
|
path = Path(source)
|
||||||
|
else:
|
||||||
|
raise FileNotFoundError(f"no decision log at {source}")
|
||||||
|
entries = TradingMemoryLog({"memory_log_path": str(path)}).load_entries()
|
||||||
# A decision with no readable rating has no direction, so it can neither
|
# A decision with no readable rating has no direction, so it can neither
|
||||||
# count for nor against the system; it is reported as unscored instead.
|
# count for nor against the system; it is reported as unscored instead.
|
||||||
resolved = [(e, _alpha(e)) for e in entries
|
resolved = [(e, _alpha(e)) for e in entries
|
||||||
|
|||||||
Reference in New Issue
Block a user