From d8eceb6571d290421fc830772871a2bd33681b29 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Wed, 16 Sep 2026 21:45:08 +0000 Subject: [PATCH] feat(cli): add a backtest command (#1234) - analysis stays the bare invocation; backtest runs a ticker and date grid - selections name the models as the config does - the pending note prints only when a cell is pending --- cli/main.py | 82 ++++++++++++++++++++++---- tests/test_backtest.py | 7 +++ tests/test_cli_commands.py | 91 +++++++++++++++++++++++++++++ tests/test_cli_config_precedence.py | 4 +- tests/test_cli_env_skip.py | 4 +- tradingagents/backtest.py | 11 ++-- 6 files changed, 178 insertions(+), 21 deletions(-) create mode 100644 tests/test_cli_commands.py diff --git a/cli/main.py b/cli/main.py index 348246d8c..00a233426 100644 --- a/cli/main.py +++ b/cli/main.py @@ -20,6 +20,7 @@ from rich.table import Table from rich.text import Text from cli.announcements import display_announcements, fetch_announcements +from cli.prefs import load_last_run, sanitize, save_last_run from cli.stats_handler import StatsCallbackHandler from cli.utils import ( ask_anthropic_effort, @@ -41,6 +42,7 @@ from cli.utils import ( select_research_depth, select_shallow_thinking_agent, ) +from tradingagents.backtest import iter_grid, run_backtest, summarize from tradingagents.default_config import DEFAULT_CONFIG from tradingagents.graph.analyst_execution import ( AnalystWallTimeTracker, @@ -49,6 +51,7 @@ from tradingagents.graph.analyst_execution import ( sync_analyst_tracker_from_chunk, ) from tradingagents.graph.trading_graph import TradingAgentsGraph +from tradingagents.portfolio import load_portfolio from tradingagents.reporting import write_report_tree console = Console() @@ -493,7 +496,14 @@ def update_display(layout, spinner_text=None, stats_handler=None, start_time=Non def get_user_selections(): - """Get all user selections before starting the analysis display.""" + """Ask for the run's settings, offering the previous run's answers.""" + selections = _prompt_selections(load_last_run()) + save_last_run(selections) + return selections + + +def _prompt_selections(prefs): + """Walk the selection steps. ``prefs`` prefills, the environment skips.""" # Display ASCII art welcome message with open(Path(__file__).parent / "static" / "welcome.txt", encoding="utf-8") as f: welcome_ascii = f.read() @@ -586,7 +596,7 @@ def get_user_selections(): "Select the language for analyst reports and final decision" ) ) - output_language = ask_output_language() + output_language = ask_output_language(prefs.get("output_language")) # Step 4: Select analysts console.print( @@ -594,7 +604,8 @@ def get_user_selections(): "Step 4: Analysts Team", "Select your LLM analyst agents for the analysis" ) ) - selected_analysts = select_analysts(asset_type) + prefs = sanitize(prefs, asset_type.value) + selected_analysts = select_analysts(asset_type, prefs.get("analysts")) console.print( f"[green]Selected analysts:[/green] {', '.join(analyst.value for analyst in selected_analysts)}" ) @@ -619,7 +630,7 @@ def get_user_selections(): "Step 5: Research Depth", "Select your research depth level" ) ) - selected_research_depth = select_research_depth() + selected_research_depth = select_research_depth(prefs.get("research_depth")) # Step 6: LLM Provider (skipped when set via TRADINGAGENTS_LLM_PROVIDER). # The backend URL comes from TRADINGAGENTS_LLM_BACKEND_URL when set, @@ -641,7 +652,7 @@ def get_user_selections(): "Step 6: LLM Provider", "Select your LLM provider" ) ) - selected_llm_provider, backend_url = select_llm_provider() + selected_llm_provider, backend_url = select_llm_provider(prefs.get("llm_provider")) # Providers with regional endpoints prompt for the region as a secondary # step so the main dropdown stays clean (mainland China and international @@ -688,8 +699,13 @@ def get_user_selections(): "Step 7: Thinking Agents", "Select your thinking agents for analysis" ) ) - selected_shallow_thinker = select_shallow_thinking_agent(selected_llm_provider) - selected_deep_thinker = select_deep_thinking_agent(selected_llm_provider) + remembered = prefs if prefs.get("llm_provider") == selected_llm_provider else {} + selected_shallow_thinker = select_shallow_thinking_agent( + selected_llm_provider, remembered.get("quick_think_llm") + ) + selected_deep_thinker = select_deep_thinking_agent( + selected_llm_provider, remembered.get("deep_think_llm") + ) # Step 8: Provider-specific reasoning/thinking configuration. Each knob is # settable via its TRADINGAGENTS_* env var; when that var is set (or the @@ -732,8 +748,8 @@ def get_user_selections(): "research_depth": selected_research_depth, "llm_provider": selected_llm_provider.lower(), "backend_url": backend_url, - "shallow_thinker": selected_shallow_thinker, - "deep_thinker": selected_deep_thinker, + "quick_think_llm": selected_shallow_thinker, + "deep_think_llm": selected_deep_thinker, "google_thinking_level": thinking_level, "openai_reasoning_effort": reasoning_effort, "anthropic_effort": anthropic_effort, @@ -985,8 +1001,8 @@ def _build_run_config(selections: dict, checkpoint: bool | None) -> dict: config["max_debate_rounds"] = selections["research_depth"] if not os.environ.get("TRADINGAGENTS_MAX_RISK_ROUNDS"): config["max_risk_discuss_rounds"] = selections["research_depth"] - config["quick_think_llm"] = selections["shallow_thinker"] - config["deep_think_llm"] = selections["deep_thinker"] + config["quick_think_llm"] = selections["quick_think_llm"] + config["deep_think_llm"] = selections["deep_think_llm"] config["backend_url"] = selections["backend_url"] config["llm_provider"] = selections["llm_provider"].lower() # Provider-specific thinking configuration @@ -1295,8 +1311,9 @@ def run_analysis(checkpoint: bool | None = None, portfolio=None): display_complete_report(final_state) -@app.command() +@app.callback(invoke_without_command=True) def analyze( + ctx: typer.Context, checkpoint: bool | None = typer.Option( None, "--checkpoint/--no-checkpoint", @@ -1315,6 +1332,9 @@ def analyze( "portfolio agents size against your actual position.", ), ): + """Run an analysis. This is what a bare `tradingagents` does.""" + if ctx.invoked_subcommand is not None: + return if clear_checkpoints: from tradingagents.graph.checkpointer import clear_all_checkpoints n = clear_all_checkpoints(DEFAULT_CONFIG["data_cache_dir"]) @@ -1343,5 +1363,43 @@ def analyze( raise typer.Exit(code=1) from None +@app.command() +def backtest( + tickers: str = typer.Argument(..., help="Comma-separated tickers, e.g. NVDA,AAPL"), + start: str = typer.Option(..., "--start", help="First analysis date, YYYY-MM-DD"), + end: str = typer.Option(..., "--end", help="Last analysis date, YYYY-MM-DD"), + every: int = typer.Option(7, "--every", help="Days between analysis dates"), + analysts: str = typer.Option( + None, "--analysts", help="Comma-separated analysts to run; omit for all four" + ), + asset_type: str = typer.Option("stock", "--asset-type", help="stock or crypto"), + portfolio: str = typer.Option( + None, "--portfolio", help="JSON file with holdings and cash, held constant across the grid" + ), +): + """Score past decisions over a grid of tickers and dates.""" + from tradingagents.agents.utils.memory import TradingMemoryLog + + try: + dates = iter_grid(start, end, every) + book = load_portfolio(portfolio) if portfolio else None + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(code=1) from None + + names = [t.strip() for t in tickers.split(",") if t.strip()] + kwargs = {"asset_type": asset_type, "portfolio": book} + if analysts: + kwargs["selected_analysts"] = [a.strip().lower() for a in analysts.split(",") if a.strip()] + + result = run_backtest(names, dates, DEFAULT_CONFIG, **kwargs) + console.print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render()) + console.print(f"\nRan {result.cells_run} cells, skipped {result.skipped}. Log: {result.log_path}") + for ticker, date, reason in result.failures: + console.print(f"[yellow]failed:[/yellow] {ticker} {date}: {reason}") + for ticker, reason in result.settlement_failures: + console.print(f"[yellow]unsettled:[/yellow] {ticker}: {reason}") + + if __name__ == "__main__": app() diff --git a/tests/test_backtest.py b/tests/test_backtest.py index a81ce7778..08aff1e08 100644 --- a/tests/test_backtest.py +++ b/tests/test_backtest.py @@ -179,3 +179,10 @@ def test_a_failed_settlement_does_not_lose_the_remaining_tickers(tmp_path, monke assert result.cells_run == 2 assert settled == ["AAPL"] assert result.settlement_failures == [("NVDA", "reflector timed out")] + + +@pytest.mark.unit +def test_pending_note_appears_only_when_something_is_pending(tmp_path): + settled = [("NVDA", "2026-01-05", DECISION, (0.1, 0.05))] + assert "Pending" not in summarize(_log_with(tmp_path, settled)).render() + assert "Pending" in summarize(_log_with(tmp_path, settled + [("AAPL", "2026-01-05", DECISION, None)])).render() diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py new file mode 100644 index 000000000..5b104a9b7 --- /dev/null +++ b/tests/test_cli_commands.py @@ -0,0 +1,91 @@ +"""The CLI keeps running an analysis with no arguments, and gains `backtest`. + +Every documented invocation is bare (`tradingagents --checkpoint`), so analysis +has to stay the default action while a second command exists alongside it. +""" + +from __future__ import annotations + +import pytest +from typer.testing import CliRunner + +import cli.main as m + + +@pytest.fixture +def runner(monkeypatch): + monkeypatch.setattr(m, "run_analysis", lambda **kw: calls.append(("analysis", kw))) + calls.clear() + return CliRunner() + + +calls: list = [] + + +@pytest.mark.unit +def test_no_arguments_still_runs_an_analysis(runner): + assert runner.invoke(m.app, []).exit_code == 0 + assert calls == [("analysis", {"checkpoint": None, "portfolio": None})] + + +@pytest.mark.unit +def test_options_still_parse_without_a_subcommand(runner): + assert runner.invoke(m.app, ["--checkpoint"]).exit_code == 0 + assert calls[0][1]["checkpoint"] is True + + +@pytest.mark.unit +def test_backtest_does_not_also_run_an_analysis(runner, monkeypatch, tmp_path): + swept = [] + monkeypatch.setattr(m, "run_backtest", lambda *a, **kw: swept.append((a, kw)) or _Result(tmp_path)) + monkeypatch.setattr(m, "summarize", lambda log: _Summary()) + + result = runner.invoke(m.app, ["backtest", "NVDA,AAPL", "--start", "2026-06-01", + "--end", "2026-06-15", "--every", "7"]) + + assert result.exit_code == 0, result.output + assert calls == [] # the interactive analysis must not run + (tickers, dates, _config), kwargs = swept[0] + assert tickers == ["NVDA", "AAPL"] + assert dates == ["2026-06-01", "2026-06-08", "2026-06-15"] + assert "scored" in result.output + + +@pytest.mark.unit +def test_backtest_reports_a_bad_date_instead_of_a_traceback(runner): + result = runner.invoke(m.app, ["backtest", "NVDA", "--start", "June", "--end", "2026-06-15"]) + assert result.exit_code == 1 + assert "YYYY-MM-DD" in result.output + + +@pytest.mark.unit +def test_help_lists_the_backtest_command(runner): + assert "backtest" in runner.invoke(m.app, ["--help"]).output + + +class _Result: + def __init__(self, tmp_path): + self.run_id = "20260916_000000" + self.log_path = tmp_path / "trading_memory.md" + self.cells_run = 2 + self.skipped = 0 + self.failures = [] + self.settlement_failures = [] + + +class _Summary: + def render(self): + return "scored 2 cells" + + +@pytest.mark.unit +def test_every_command_is_registered_when_run_as_a_module(): + """README documents `python -m cli.main`, which executes the file top to + bottom, so a command defined after the __main__ block would not exist.""" + import subprocess + import sys + + out = subprocess.run([sys.executable, "-m", "cli.main", "backtest", "--help"], + capture_output=True, text=True, timeout=120) + assert out.returncode == 0, out.stderr[-400:] + assert "--start" in out.stdout diff --git a/tests/test_cli_config_precedence.py b/tests/test_cli_config_precedence.py index e2ba90d14..b2f1f0cc8 100644 --- a/tests/test_cli_config_precedence.py +++ b/tests/test_cli_config_precedence.py @@ -14,8 +14,8 @@ import cli.main as m # Minimal selections dict shaped like get_user_selections()'s return value. SELECTIONS = { "research_depth": 5, - "shallow_thinker": "gpt-5.4-mini", - "deep_thinker": "gpt-5.5", + "quick_think_llm": "gpt-5.4-mini", + "deep_think_llm": "gpt-5.5", "backend_url": None, "llm_provider": "openai", "google_thinking_level": None, diff --git a/tests/test_cli_env_skip.py b/tests/test_cli_env_skip.py index c98c24549..b3d50d956 100644 --- a/tests/test_cli_env_skip.py +++ b/tests/test_cli_env_skip.py @@ -77,8 +77,8 @@ class TestCliSkipsPromptsFromEnv(unittest.TestCase): # The env values flow into the returned selections. self.assertEqual(sel["llm_provider"], "openai") self.assertEqual(sel["backend_url"], "https://opencode.ai/zen/go/v1") - self.assertEqual(sel["shallow_thinker"], "deepseek-v4-pro") - self.assertEqual(sel["deep_thinker"], "kimi-k2.5") + self.assertEqual(sel["quick_think_llm"], "deepseek-v4-pro") + self.assertEqual(sel["deep_think_llm"], "kimi-k2.5") self.assertEqual(sel["output_language"], "Japanese") diff --git a/tradingagents/backtest.py b/tradingagents/backtest.py index 6557f3f96..f6d9a4e23 100644 --- a/tradingagents/backtest.py +++ b/tradingagents/backtest.py @@ -100,12 +100,13 @@ class BacktestSummary: f"- {rating}: n={score.count}, beat the benchmark " f"{score.hit_rate:.0%}, mean alpha {score.mean_alpha:+.2%}" ) - lines += [ - "", - "Pending cells are not scored above; re-run to settle them.", + lines.append("") + if self.pending: + lines.append("Pending cells are not scored above; re-run to settle them.") + lines.append( "One model sampling per cell, and text feeds are not archived, so " - "these figures are indicative rather than repeatable.", - ] + "these figures are indicative rather than repeatable." + ) return "\n".join(lines)