diff --git a/tests/test_date_boundaries.py b/tests/test_date_boundaries.py index 80d0f08e1..9d5300882 100644 --- a/tests/test_date_boundaries.py +++ b/tests/test_date_boundaries.py @@ -4,6 +4,8 @@ requested end_date (and the current day) is actually included. Regressions for #986 (current-day OHLCV excluded) and #987 (requested end_date row omitted). """ +from types import SimpleNamespace + import pandas as pd import pytest @@ -44,7 +46,7 @@ def test_load_ohlcv_requests_inclusive_end(monkeypatch, tmp_path): set_config({"data_cache_dir": str(tmp_path)}) captured = {} - def fake_download(symbol, start, end, **kwargs): + def fake_history(start, end, **kwargs): captured["end"] = end idx = pd.to_datetime([pd.Timestamp.today().normalize()]) return pd.DataFrame( @@ -53,7 +55,7 @@ def test_load_ohlcv_requests_inclusive_end(monkeypatch, tmp_path): index=idx, ) - monkeypatch.setattr(ohlcv.yf, "download", fake_download) + monkeypatch.setattr(ohlcv.yf, "Ticker", lambda symbol: SimpleNamespace(history=fake_history)) today = pd.Timestamp.today().strftime("%Y-%m-%d") ohlcv.load_ohlcv("AAPL", today) diff --git a/tests/test_no_data_handling.py b/tests/test_no_data_handling.py index b70ff3354..f1eadd439 100644 --- a/tests/test_no_data_handling.py +++ b/tests/test_no_data_handling.py @@ -33,19 +33,19 @@ class TestLoadOhlcvNoPoison(unittest.TestCase): os.rmdir(self._tmp) def test_empty_download_raises_and_does_not_cache(self): - empty = pd.DataFrame() + empty = mock.Mock(history=mock.Mock(return_value=pd.DataFrame())) # Yahoo answers, so an empty download means the symbol has no data. reachable = mock.patch.object(ohlcv, "vendor_reachable", return_value=True) reachable.start() self.addCleanup(reachable.stop) - with mock.patch.object(ohlcv.yf, "download", return_value=empty), \ + with mock.patch.object(ohlcv.yf, "Ticker", return_value=empty), \ self.assertRaises(NoMarketDataError): ohlcv.load_ohlcv("FAKE", "2026-01-01") # Nothing should have been written to the cache. self.assertEqual(os.listdir(self._tmp), []) # A second call must re-attempt the fetch (no poisoned cache served). - with mock.patch.object(ohlcv.yf, "download", return_value=empty) as dl2: + with mock.patch.object(ohlcv.yf, "Ticker", return_value=empty) as dl2: with self.assertRaises(NoMarketDataError): ohlcv.load_ohlcv("FAKE", "2026-01-01") self.assertTrue(dl2.called) diff --git a/tests/test_ohlcv_cache_freshness.py b/tests/test_ohlcv_cache_freshness.py index c7718b41c..da905e383 100644 --- a/tests/test_ohlcv_cache_freshness.py +++ b/tests/test_ohlcv_cache_freshness.py @@ -8,6 +8,7 @@ day (#1330). from __future__ import annotations import os +from types import SimpleNamespace import pandas as pd import pytest @@ -35,7 +36,7 @@ def _write(tmp_path, name="AAPL-YFin-data.csv", age_seconds=0.0, last_date="2026 def _load(tmp_path, monkeypatch, curr_date, download): monkeypatch.setattr(ohlcv, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) monkeypatch.setattr(ohlcv.pd.Timestamp, "today", staticmethod(lambda: NOW)) - monkeypatch.setattr(ohlcv.yf, "download", download) + monkeypatch.setattr(ohlcv.yf, "Ticker", lambda symbol: SimpleNamespace(history=download)) return ohlcv.load_ohlcv("AAPL", curr_date) @@ -98,7 +99,8 @@ def test_one_cache_file_per_symbol_across_days(tmp_path, monkeypatch): monkeypatch.setattr(ohlcv, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) frame = pd.DataFrame({"Date": pd.to_datetime(["2026-07-16", "2026-07-17"]), "Close": [1.0, 2.0]}) downloads = [] - monkeypatch.setattr(ohlcv.yf, "download", lambda *a, **k: downloads.append(1) or frame.set_index("Date")) + monkeypatch.setattr(ohlcv.yf, "Ticker", lambda symbol: SimpleNamespace( + history=lambda *a, **k: downloads.append(1) or frame.set_index("Date"))) for day in ("2026-07-18 10:00", "2026-07-19 10:00", "2026-07-20 10:00"): now = pd.Timestamp(day) diff --git a/tests/test_ohlcv_latest_bar.py b/tests/test_ohlcv_latest_bar.py index 1fb82d866..7b48b5894 100644 --- a/tests/test_ohlcv_latest_bar.py +++ b/tests/test_ohlcv_latest_bar.py @@ -14,6 +14,7 @@ anywhere counts as no data and the staleness check judges the rest. from __future__ import annotations import os +from types import SimpleNamespace import pandas as pd import pytest @@ -105,7 +106,7 @@ def _run_load(monkeypatch, tmp_path, frame, curr_date): def _fail_download(*a, **k): raise AssertionError("should use the seeded cache, not download") - monkeypatch.setattr(ohlcv.yf, "download", _fail_download) + monkeypatch.setattr(ohlcv.yf, "Ticker", lambda symbol: SimpleNamespace(history=_fail_download)) return ohlcv.load_ohlcv("AAPL", curr_date) @@ -198,8 +199,8 @@ def test_the_snapshot_does_not_present_a_filled_price_as_reported(monkeypatch, t cache = tmp_path / "AAPL-YFin-data.csv" cache.write_text(frame.to_csv(index=False)) _stamp(cache, today) - monkeypatch.setattr(ohlcv.yf, "download", lambda *a, **k: (_ for _ in ()).throw( - AssertionError("should read the seeded cache"))) + monkeypatch.setattr(ohlcv.yf, "Ticker", lambda symbol: SimpleNamespace( + history=lambda *a, **k: (_ for _ in ()).throw(AssertionError("should read the seeded cache")))) out = snapshot.build_verified_market_snapshot("AAPL", "2026-05-08", 3) diff --git a/tests/test_yahoo_rate_limit.py b/tests/test_yahoo_rate_limit.py new file mode 100644 index 000000000..97a6882ad --- /dev/null +++ b/tests/test_yahoo_rate_limit.py @@ -0,0 +1,130 @@ +"""A Yahoo rate limit is retried, then reported as a rate limit. + +When the limit outlasts the retries, the agent must hear that the vendor is +throttled. "The symbol may be invalid or delisted" is a claim about the company +that nobody checked, and an exception out of a tool ends the run. +""" + +import pandas as pd +import pytest +import yfinance as yf +from yfinance.data import YfData +from yfinance.exceptions import YFRateLimitError + +from tradingagents.agents.tools import ( + get_global_news, + get_indicators, + get_insider_transactions, + get_news, + get_stock_data, + get_verified_market_snapshot, +) +from tradingagents.dataflows import router +from tradingagents.dataflows.config import set_config +from tradingagents.dataflows.errors import NoMarketDataError, VendorRateLimitError +from tradingagents.dataflows.vendors.yahoo import fundamentals, ohlcv + +DAY = "2026-09-18" + + +def _rate_limited(*args, **kwargs): + raise YFRateLimitError() + + +@pytest.fixture +def yahoo(monkeypatch, tmp_path): + set_config({"data_cache_dir": str(tmp_path)}) + monkeypatch.setattr(ohlcv.time, "sleep", lambda seconds: None) + for module in (ohlcv, fundamentals): + monkeypatch.setattr(module, "vendor_reachable", lambda url: True) + return monkeypatch + + +@pytest.mark.unit +@pytest.mark.parametrize("tool, args", [ + pytest.param(get_stock_data, ("AAPL", "2026-09-10", DAY), id="stock_data"), + pytest.param(get_indicators, ("AAPL", "rsi", DAY, 5), id="indicators"), + pytest.param(get_verified_market_snapshot, ("AAPL", DAY), id="snapshot"), + pytest.param(get_news, ("AAPL", "2026-09-10", DAY), id="news"), + pytest.param(get_global_news, (DAY, 7, 5), id="global_news"), + pytest.param(get_insider_transactions, ("AAPL",), id="insider"), +]) +def test_a_rate_limit_that_outlasts_the_retries_is_reported_as_one(yahoo, tool, args): + for name in ("history", "get_news"): + yahoo.setattr(yf.Ticker, name, _rate_limited) + yahoo.setattr(yf.Ticker, "insider_transactions", property(_rate_limited)) + yahoo.setattr(yf, "Search", _rate_limited) + + out = tool.func(*args, trade_date=DAY) + + assert out.startswith("DATA_UNAVAILABLE"), out + assert "delisted" not in out + + +@pytest.mark.unit +def test_the_indicator_path_retries_a_rate_limit(yahoo): + """The prices behind every indicator were fetched with ``yf.download``, which + returns an empty frame for a 429, so they were never retried.""" + bar = pd.DataFrame({"Open": [1.0], "High": [1.0], "Low": [1.0], "Close": [1.0], + "Volume": [100]}, index=pd.DatetimeIndex([DAY], name="Date")) + answers = [YFRateLimitError(), bar] + + def history(self, **kwargs): + answer = answers.pop(0) + if isinstance(answer, Exception): + raise answer + return answer + + yahoo.setattr(yf.Ticker, "history", history) + + assert ohlcv.load_ohlcv("AAPL", DAY)["Close"].tolist() == [1.0] + + +@pytest.mark.unit +def test_yfinance_raises_the_rate_limit_from_history(yahoo): + yahoo.setattr(YfData, "_make_request", _rate_limited) + + with pytest.raises(VendorRateLimitError, match="rate limited"): + ohlcv.load_ohlcv("AAPL", DAY) + + +@pytest.mark.unit +def test_fundamentals_ask_a_new_ticker_after_a_rate_limit(yahoo): + """A Ticker keeps a failed ``info`` fetch as done, so asking the same one + again reads an empty profile, which looks like a symbol with no data.""" + + class Ticker: + def __init__(self, symbol): + self.fetched = False + + @property + def info(self): + if self.fetched: + return {} + self.fetched = True + raise YFRateLimitError() + + yahoo.setattr(yf, "Ticker", Ticker) + + out = router.route_to_vendor("get_fundamentals", "AAPL", None) + + assert out.startswith("DATA_UNAVAILABLE"), out + + +@pytest.mark.unit +def test_another_price_fetch_error_still_tells_an_outage_from_an_unknown_symbol(yahoo): + """``Ticker.history`` lets some errors through that ``yf.download`` turned + into an empty frame. Whether Yahoo answers still decides which one it is.""" + + def refused(self, **kwargs): + raise ConnectionError("curl: (7) Failed to connect to query2.finance.yahoo.com") + + yahoo.setattr(yf.Ticker, "history", refused) + + yahoo.setattr(ohlcv, "vendor_reachable", lambda url: False) + with pytest.raises(VendorRateLimitError, match="unreachable"): + ohlcv.load_ohlcv("AAPL", DAY) + + yahoo.setattr(ohlcv, "vendor_reachable", lambda url: True) + with pytest.raises(NoMarketDataError): + ohlcv.load_ohlcv("AAPL", DAY) diff --git a/tradingagents/agents/tools.py b/tradingagents/agents/tools.py index 0ea4667e2..712dd90f3 100644 --- a/tradingagents/agents/tools.py +++ b/tradingagents/agents/tools.py @@ -10,7 +10,8 @@ from langchain_core.tools import tool from langgraph.prebuilt import InjectedState from tradingagents.dataflows.date_window import as_of, as_of_window -from tradingagents.dataflows.router import route_to_vendor +from tradingagents.dataflows.errors import VendorRateLimitError +from tradingagents.dataflows.router import route_to_vendor, vendor_unavailable from tradingagents.dataflows.vendors.yahoo.snapshot import build_verified_market_snapshot @@ -83,7 +84,11 @@ def get_verified_market_snapshot( price levels, Bollinger bands, RSI, MACD, moving averages, support / resistance, or historical comparisons, and treat it as the source of truth. """ - return build_verified_market_snapshot(symbol, as_of(curr_date, trade_date), look_back_days) + # An exception out of a tool would end the run. + try: + return build_verified_market_snapshot(symbol, as_of(curr_date, trade_date), look_back_days) + except VendorRateLimitError as exc: + return vendor_unavailable("get_verified_market_snapshot", exc) @tool diff --git a/tradingagents/dataflows/router.py b/tradingagents/dataflows/router.py index 33d7df866..7bfb1eefb 100644 --- a/tradingagents/dataflows/router.py +++ b/tradingagents/dataflows/router.py @@ -181,6 +181,15 @@ def get_vendor(category: str, method: str = None) -> str: return config.get("data_vendors", {}).get(category, "default") +def vendor_unavailable(method: str, error: VendorRateLimitError) -> str: + """What a call returns when every vendor was throttled or unreachable.""" + return ( + f"DATA_UNAVAILABLE: no configured vendor could serve {method} right now " + f"({error}). This says nothing about the instrument; report the " + f"data as unavailable and do not estimate or fabricate values." + ) + + def route_to_vendor(method: str, *args, **kwargs): """Route method calls to appropriate vendor implementation with fallback support.""" category = get_category_for_method(method) @@ -273,11 +282,7 @@ def route_to_vendor(method: str, *args, **kwargs): # Every vendor was throttled or unreachable: that is a fact about the # vendors, not about the instrument, and it must not end the run. if last_unavailable is not None: - return ( - f"DATA_UNAVAILABLE: no configured vendor could serve {method} right now " - f"({last_unavailable}). This says nothing about the instrument; report the " - f"data as unavailable and do not estimate or fabricate values." - ) + return vendor_unavailable(method, last_unavailable) if first_error is not None: if category in OPTIONAL_CATEGORIES: diff --git a/tradingagents/dataflows/vendors/yahoo/fundamentals.py b/tradingagents/dataflows/vendors/yahoo/fundamentals.py index 72974f633..8c8e108b9 100644 --- a/tradingagents/dataflows/vendors/yahoo/fundamentals.py +++ b/tradingagents/dataflows/vendors/yahoo/fundamentals.py @@ -33,8 +33,7 @@ def get_fundamentals( return withheld try: - ticker_obj = yf.Ticker(canonical) - info = yf_retry(lambda: ticker_obj.info) + info = yf_retry(lambda: yf.Ticker(canonical).info) if not info: raise_for_empty(ticker, canonical, "fundamentals") diff --git a/tradingagents/dataflows/vendors/yahoo/news.py b/tradingagents/dataflows/vendors/yahoo/news.py index 9f0498b9a..9561e9fad 100644 --- a/tradingagents/dataflows/vendors/yahoo/news.py +++ b/tradingagents/dataflows/vendors/yahoo/news.py @@ -8,7 +8,7 @@ from dateutil.relativedelta import relativedelta from tradingagents.dataflows.config import get_config from tradingagents.dataflows.date_window import coverage_gap, in_window -from tradingagents.dataflows.errors import NoMarketDataError +from tradingagents.dataflows.errors import NoMarketDataError, VendorError from tradingagents.dataflows.symbols import normalize_symbol from tradingagents.dataflows.vendors.yahoo.ohlcv import yf_retry @@ -114,6 +114,8 @@ def get_news_yfinance( return f"## {ticker}{resolved} News, from {start_date} to {end_date}:\n\n{news_str}" + except VendorError: + raise except Exception as e: raise NoMarketDataError(ticker, ticker, f"news unavailable: {e}") from e @@ -192,5 +194,7 @@ def get_global_news_yfinance( return f"## Global Market News, from {start_date} to {curr_date}:\n\n{news_str}" + except VendorError: + raise except Exception as e: raise NoMarketDataError("global news", "global news", f"unavailable: {e}") from e diff --git a/tradingagents/dataflows/vendors/yahoo/ohlcv.py b/tradingagents/dataflows/vendors/yahoo/ohlcv.py index 6325d6172..e5001aeb1 100644 --- a/tradingagents/dataflows/vendors/yahoo/ohlcv.py +++ b/tradingagents/dataflows/vendors/yahoo/ohlcv.py @@ -43,18 +43,24 @@ def yf_retry(func, max_retries=3, base_delay=2.0): yfinance raises YFRateLimitError on HTTP 429 responses but does not retry them internally. This wrapper adds retry logic specifically - for rate limits. Other exceptions propagate immediately. + for rate limits. Other exceptions propagate immediately. A rate limit + that outlasts the retries is raised as VendorRateLimitError, so the + router reports a throttled vendor rather than a symbol with no data. + ``func`` should build its own Ticker: a Ticker keeps a failed ``info`` + fetch as done, so asking the same one again reads an empty profile. """ for attempt in range(max_retries + 1): try: return func() - except YFRateLimitError: + except YFRateLimitError as exc: if attempt < max_retries: delay = base_delay * (2 ** attempt) logger.warning(f"Yahoo Finance rate limited, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries})") time.sleep(delay) else: - raise + raise VendorRateLimitError( + f"Yahoo Finance rate limited after {max_retries} retries: {exc}" + ) from exc def _ensure_date_column(data: pd.DataFrame) -> pd.DataFrame: @@ -240,14 +246,22 @@ def load_ohlcv(symbol: str, curr_date: str, fill_gaps: bool = True) -> pd.DataFr data = cached if data is None: - downloaded = yf_retry(lambda: yf.download( - canonical, - start=start_str, - end=end_str, - multi_level_index=False, - progress=False, - auto_adjust=True, - )) + # yf.download catches every error, a rate limit included, and returns + # an empty frame. Ticker.history raises the rate limit, so it is retried. + try: + downloaded = yf_retry(lambda: yf.Ticker(canonical).history( + start=start_str, + end=end_str, + auto_adjust=True, + actions=False, + )) + except VendorRateLimitError: + raise + except Exception as exc: + # Any other failure is an outage or an unknown symbol, which + # raise_for_empty tells apart by whether Yahoo answers at all. + logger.warning("Yahoo Finance price request for %s failed: %s", canonical, exc) + raise_for_empty(symbol, canonical, "price rows") downloaded = _ensure_date_column(downloaded.reset_index()) # Only cache real data — never persist an empty frame. if downloaded.empty or "Close" not in downloaded.columns: