From ad3913356989227b0efcdb90b9e65f7c6cace653 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Mon, 28 Sep 2026 21:12:56 +0000 Subject: [PATCH] refactor(yahoo): keep what every Yahoo request shares in yahoo.common - yf_retry, raise_for_empty and YAHOO_HOST move from yahoo.ohlcv, which the news, fundamentals and market modules imported them from --- tests/test_fundamentals_lookahead.py | 4 +- tests/test_no_data_handling.py | 4 +- tests/test_undated_tools_as_of.py | 12 ++-- tests/test_yahoo_rate_limit.py | 10 +-- .../dataflows/vendors/yahoo/common.py | 65 +++++++++++++++++++ .../dataflows/vendors/yahoo/fundamentals.py | 2 +- .../dataflows/vendors/yahoo/market.py | 8 +-- tradingagents/dataflows/vendors/yahoo/news.py | 2 +- .../dataflows/vendors/yahoo/ohlcv.py | 59 +---------------- 9 files changed, 86 insertions(+), 80 deletions(-) create mode 100644 tradingagents/dataflows/vendors/yahoo/common.py diff --git a/tests/test_fundamentals_lookahead.py b/tests/test_fundamentals_lookahead.py index f39f88124..6852dc3d1 100644 --- a/tests/test_fundamentals_lookahead.py +++ b/tests/test_fundamentals_lookahead.py @@ -129,9 +129,9 @@ class TestNoUsableFieldsStillRaises: # yfinance returns {"trailingPegRatio": None} for unknown symbols; on a # live run that must stay a hard "no data", not a bare header. from tradingagents.dataflows.errors import NoMarketDataError - from tradingagents.dataflows.vendors.yahoo import ohlcv + from tradingagents.dataflows.vendors.yahoo import common - monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: True) + monkeypatch.setattr(common, "vendor_reachable", lambda url: True) with pytest.raises(NoMarketDataError): _yf(_TODAY, info={"trailingPegRatio": None}) diff --git a/tests/test_no_data_handling.py b/tests/test_no_data_handling.py index 20e1adb7b..5caab2ce6 100644 --- a/tests/test_no_data_handling.py +++ b/tests/test_no_data_handling.py @@ -17,7 +17,7 @@ import pytest from tradingagents.dataflows import router from tradingagents.dataflows.config import set_config from tradingagents.dataflows.errors import NoMarketDataError -from tradingagents.dataflows.vendors.yahoo import ohlcv +from tradingagents.dataflows.vendors.yahoo import common, ohlcv @pytest.mark.unit @@ -35,7 +35,7 @@ class TestLoadOhlcvNoPoison(unittest.TestCase): def test_empty_download_raises_and_does_not_cache(self): empty = mock.Mock(history=mock.Mock(return_value=pd.DataFrame())) # Yahoo answers, so an empty download means the symbol has no data. - reachable = mock.patch.object(ohlcv, "vendor_reachable", return_value=True) + reachable = mock.patch.object(common, "vendor_reachable", return_value=True) reachable.start() self.addCleanup(reachable.stop) with mock.patch.object(ohlcv.yf, "Ticker", return_value=empty), \ diff --git a/tests/test_undated_tools_as_of.py b/tests/test_undated_tools_as_of.py index bd4aa01c3..521507509 100644 --- a/tests/test_undated_tools_as_of.py +++ b/tests/test_undated_tools_as_of.py @@ -200,16 +200,16 @@ def test_an_unreachable_vendor_is_not_reported_as_a_missing_symbol(monkeypatch): import pandas as pd from tradingagents.dataflows.errors import NoMarketDataError, VendorUnavailableError - from tradingagents.dataflows.vendors.yahoo import ohlcv + from tradingagents.dataflows.vendors.yahoo import common empty = mock.Mock(quarterly_balance_sheet=pd.DataFrame(), balance_sheet=pd.DataFrame()) monkeypatch.setattr(yahoo_market.yf, "Ticker", lambda s: empty) - monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: False) + monkeypatch.setattr(common, "vendor_reachable", lambda url: False) with pytest.raises(VendorUnavailableError, match="unreachable"): yahoo_fundamentals.get_balance_sheet("AAPL", "annual", get_current_date()) - monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: True) + monkeypatch.setattr(common, "vendor_reachable", lambda url: True) with pytest.raises(NoMarketDataError): yahoo_fundamentals.get_balance_sheet("AAPL", "annual", get_current_date()) @@ -241,15 +241,15 @@ def test_the_price_path_also_tells_an_outage_from_an_unknown_symbol(monkeypatch) import pandas as pd from tradingagents.dataflows.errors import NoMarketDataError, VendorUnavailableError - from tradingagents.dataflows.vendors.yahoo import ohlcv + from tradingagents.dataflows.vendors.yahoo import common monkeypatch.setattr(yahoo_market.yf, "Ticker", lambda s: mock.Mock(history=lambda **k: pd.DataFrame())) - monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: False) + monkeypatch.setattr(common, "vendor_reachable", lambda url: False) with pytest.raises(VendorUnavailableError, match="unreachable"): yahoo_market.get_YFin_data_online("AAPL", "2026-09-01", "2026-09-10") - monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: True) + monkeypatch.setattr(common, "vendor_reachable", lambda url: True) with pytest.raises(NoMarketDataError): yahoo_market.get_YFin_data_online("AAPL", "2026-09-01", "2026-09-10") diff --git a/tests/test_yahoo_rate_limit.py b/tests/test_yahoo_rate_limit.py index 3669a4fa2..81a585fc8 100644 --- a/tests/test_yahoo_rate_limit.py +++ b/tests/test_yahoo_rate_limit.py @@ -22,7 +22,7 @@ from tradingagents.agents.tools import ( from tradingagents.dataflows import router from tradingagents.dataflows.config import set_config from tradingagents.dataflows.errors import VendorUnavailableError -from tradingagents.dataflows.vendors.yahoo import fundamentals, ohlcv +from tradingagents.dataflows.vendors.yahoo import common, fundamentals, ohlcv DAY = "2026-09-18" @@ -34,8 +34,8 @@ def _rate_limited(*args, **kwargs): @pytest.fixture def yahoo(monkeypatch, tmp_path): set_config({"data_cache_dir": str(tmp_path)}) - monkeypatch.setattr(ohlcv.time, "sleep", lambda seconds: None) - for module in (ohlcv, fundamentals): + monkeypatch.setattr(common.time, "sleep", lambda seconds: None) + for module in (common, fundamentals): monkeypatch.setattr(module, "vendor_reachable", lambda url: True) return monkeypatch @@ -121,7 +121,7 @@ def test_a_price_request_that_raises_is_unavailable_whether_or_not_yahoo_answers yahoo.setattr(yf.Ticker, "history", refused) for reachable in (False, True): - yahoo.setattr(ohlcv, "vendor_reachable", lambda url, _r=reachable: _r) + yahoo.setattr(common, "vendor_reachable", lambda url, _r=reachable: _r) with pytest.raises(VendorUnavailableError, match="request failed"): ohlcv.load_ohlcv("AAPL", DAY) @@ -134,7 +134,7 @@ def test_a_price_request_that_fails_is_unavailable_even_when_yahoo_answers(monke monkeypatch.setattr(ohlcv, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) monkeypatch.setattr(ohlcv.yf, "Ticker", lambda s: type("T", (), {"history": lambda self, **k: timed_out(**k)})()) - monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: True) + monkeypatch.setattr(common, "vendor_reachable", lambda url: True) with pytest.raises(VendorUnavailableError): ohlcv.load_ohlcv("NVDA", "2026-09-23") diff --git a/tradingagents/dataflows/vendors/yahoo/common.py b/tradingagents/dataflows/vendors/yahoo/common.py new file mode 100644 index 000000000..40b0a3bf0 --- /dev/null +++ b/tradingagents/dataflows/vendors/yahoo/common.py @@ -0,0 +1,65 @@ +"""What every Yahoo request shares: its retry, and how a failure or an empty answer is reported.""" + +import logging +import time + +import yfinance as yf +from yfinance.exceptions import YFRateLimitError + +from tradingagents.dataflows.errors import NoMarketDataError, VendorUnavailableError +from tradingagents.dataflows.net import vendor_reachable + +logger = logging.getLogger(__name__) + +YAHOO_HOST = "https://query2.finance.yahoo.com" + + +def raise_for_empty(symbol: str, canonical: str, what: str) -> None: + """Report an empty Yahoo answer as an absence, or as an outage if it is one. + + An empty answer from a Yahoo that cannot be reached is not an answer about + the symbol, so the host is probed before "this symbol has no {what}" is said. + """ + if not vendor_reachable(YAHOO_HOST): + raise VendorUnavailableError(f"Yahoo Finance is unreachable; no {what} was retrieved") + raise NoMarketDataError(symbol, canonical, f"no {what}") + + +# yfinance answers some failed requests with an empty result, which would read as +# "no data" for a symbol nobody checked; raised, the failure is reported as one. +yf.config.debug.hide_exceptions = False + + +def _not_found(exc: Exception) -> bool: + """Whether Yahoo answered that it has no such symbol (HTTP 404).""" + return getattr(getattr(exc, "response", None), "status_code", None) == 404 + + +def yf_retry(func, max_retries=3, base_delay=2.0): + """Execute a yfinance call with exponential backoff on rate limits. + + yfinance raises YFRateLimitError on HTTP 429 responses but does not + retry them internally, so this wrapper retries them. A rate limit that + outlasts the retries, or any other exception, is raised as + VendorUnavailableError: it failed in transit and says nothing about the + symbol. Yahoo answering that it has no such symbol (HTTP 404) returns None, + an empty answer. ``func`` should build its own Ticker and make the request + itself: a Ticker keeps a failed ``info`` fetch as done, so asking the same + one again reads an empty profile. + """ + for attempt in range(max_retries + 1): + try: + return func() + except YFRateLimitError as exc: + if attempt < max_retries: + delay = base_delay * (2 ** attempt) + logger.warning(f"Yahoo Finance rate limited, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries})") + time.sleep(delay) + else: + raise VendorUnavailableError( + f"Yahoo Finance rate limited after {max_retries} retries: {exc}" + ) from exc + except Exception as exc: + if _not_found(exc): + return None + raise VendorUnavailableError(f"Yahoo Finance request failed: {type(exc).__name__}") from exc diff --git a/tradingagents/dataflows/vendors/yahoo/fundamentals.py b/tradingagents/dataflows/vendors/yahoo/fundamentals.py index 8fa5a0a5c..22055805c 100644 --- a/tradingagents/dataflows/vendors/yahoo/fundamentals.py +++ b/tradingagents/dataflows/vendors/yahoo/fundamentals.py @@ -7,7 +7,7 @@ from tradingagents.dataflows.date_window import withhold_live_profile, withhold_ from tradingagents.dataflows.errors import VendorUnavailableError from tradingagents.dataflows.net import vendor_reachable from tradingagents.dataflows.symbols import normalize_symbol -from tradingagents.dataflows.vendors.yahoo.ohlcv import ( +from tradingagents.dataflows.vendors.yahoo.common import ( YAHOO_HOST, raise_for_empty, yf_retry, diff --git a/tradingagents/dataflows/vendors/yahoo/market.py b/tradingagents/dataflows/vendors/yahoo/market.py index 7ce6e0735..b099caabf 100644 --- a/tradingagents/dataflows/vendors/yahoo/market.py +++ b/tradingagents/dataflows/vendors/yahoo/market.py @@ -9,12 +9,8 @@ from stockstats import wrap from tradingagents.dataflows.errors import NoMarketDataError, VendorError from tradingagents.dataflows.symbols import normalize_symbol -from tradingagents.dataflows.vendors.yahoo.ohlcv import ( - _assert_ohlcv_not_stale, - load_ohlcv, - raise_for_empty, - yf_retry, -) +from tradingagents.dataflows.vendors.yahoo.common import raise_for_empty, yf_retry +from tradingagents.dataflows.vendors.yahoo.ohlcv import _assert_ohlcv_not_stale, load_ohlcv logger = logging.getLogger(__name__) diff --git a/tradingagents/dataflows/vendors/yahoo/news.py b/tradingagents/dataflows/vendors/yahoo/news.py index 1a17ddf67..49e5c08cf 100644 --- a/tradingagents/dataflows/vendors/yahoo/news.py +++ b/tradingagents/dataflows/vendors/yahoo/news.py @@ -9,7 +9,7 @@ from dateutil.relativedelta import relativedelta from tradingagents.dataflows.config import get_config from tradingagents.dataflows.date_window import coverage_gap, in_window from tradingagents.dataflows.symbols import normalize_symbol -from tradingagents.dataflows.vendors.yahoo.ohlcv import yf_retry +from tradingagents.dataflows.vendors.yahoo.common import yf_retry def _extract_article_data(article: dict) -> dict: diff --git a/tradingagents/dataflows/vendors/yahoo/ohlcv.py b/tradingagents/dataflows/vendors/yahoo/ohlcv.py index f7c1cd8d1..beb612f22 100644 --- a/tradingagents/dataflows/vendors/yahoo/ohlcv.py +++ b/tradingagents/dataflows/vendors/yahoo/ohlcv.py @@ -1,21 +1,17 @@ import logging import os -import time import pandas as pd import yfinance as yf -from yfinance.exceptions import YFRateLimitError from tradingagents.dataflows.config import get_config -from tradingagents.dataflows.errors import NoMarketDataError, VendorUnavailableError +from tradingagents.dataflows.errors import NoMarketDataError from tradingagents.dataflows.files import replace_file -from tradingagents.dataflows.net import vendor_reachable from tradingagents.dataflows.symbols import normalize_symbol, safe_ticker_component +from tradingagents.dataflows.vendors.yahoo.common import raise_for_empty, yf_retry logger = logging.getLogger(__name__) -YAHOO_HOST = "https://query2.finance.yahoo.com" - # A vendor's latest OHLCV row this many calendar days before the requested date # is treated as stale. Generous enough to span long holiday weekends, tight # enough to catch the year-old frames yfinance occasionally returns (#1021). @@ -28,57 +24,6 @@ MAX_OHLCV_STALE_DAYS = 10 OHLCV_CACHE_TTL_SECONDS = 900 -def raise_for_empty(symbol: str, canonical: str, what: str) -> None: - """Report an empty Yahoo result as an absence, or as an outage if it is one. - - yfinance returns an empty frame for a failed request rather than raising, so - without this a Yahoo outage reads as "this symbol has no {what}". - """ - if not vendor_reachable(YAHOO_HOST): - raise VendorUnavailableError(f"Yahoo Finance is unreachable; no {what} was retrieved") - raise NoMarketDataError(symbol, canonical, f"no {what}") - - -# yfinance answers some failed requests with an empty result, which would read as -# "no data" for a symbol nobody checked; raised, the failure is reported as one. -yf.config.debug.hide_exceptions = False - - -def _not_found(exc: Exception) -> bool: - """Whether Yahoo answered that it has no such symbol (HTTP 404).""" - return getattr(getattr(exc, "response", None), "status_code", None) == 404 - - -def yf_retry(func, max_retries=3, base_delay=2.0): - """Execute a yfinance call with exponential backoff on rate limits. - - yfinance raises YFRateLimitError on HTTP 429 responses but does not - retry them internally, so this wrapper retries them. A rate limit that - outlasts the retries, or any other exception, is raised as - VendorUnavailableError: it failed in transit and says nothing about the - symbol. Yahoo answering that it has no such symbol (HTTP 404) returns None, - an empty answer. ``func`` should build its own Ticker and make the request - itself: a Ticker keeps a failed ``info`` fetch as done, so asking the same - one again reads an empty profile. - """ - for attempt in range(max_retries + 1): - try: - return func() - except YFRateLimitError as exc: - if attempt < max_retries: - delay = base_delay * (2 ** attempt) - logger.warning(f"Yahoo Finance rate limited, retrying in {delay:.0f}s (attempt {attempt + 1}/{max_retries})") - time.sleep(delay) - else: - raise VendorUnavailableError( - f"Yahoo Finance rate limited after {max_retries} retries: {exc}" - ) from exc - except Exception as exc: - if _not_found(exc): - return None - raise VendorUnavailableError(f"Yahoo Finance request failed: {type(exc).__name__}") from exc - - def _ensure_date_column(data: pd.DataFrame) -> pd.DataFrame: """Normalize the date column to ``Date``.