From 6097b582d9424753a57abbd578b7d73786c28d59 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Wed, 23 Sep 2026 23:20:52 +0000 Subject: [PATCH] refactor(dataflows): group the vendors under dataflows/vendors - vendors/yahoo: ohlcv (loader and cache), market (prices, indicators), fundamentals (profile, statements, insider), news, snapshot - vendors/alpha_vantage is a package; sec_edgar, fred, polymarket, reddit and stocktwits sit beside it - the one-method StockstatsUtils class is a function; the duplicate Yahoo host constant is gone - tests are named after the modules they cover: test_ohlcv_date_column, test_yahoo_snapshot, and the ohlcv and snapshot aliases --- tests/test_alpha_vantage_hardening.py | 14 +- tests/test_date_boundaries.py | 8 +- tests/test_fred.py | 3 +- tests/test_fundamentals_lookahead.py | 17 +- tests/test_graph_end_to_end.py | 7 +- tests/test_instrument_identity.py | 12 +- tests/test_memory_log.py | 2 +- tests/test_news_lookahead.py | 2 +- tests/test_no_data_handling.py | 11 +- tests/test_ohlcv_cache_freshness.py | 30 +-- ...te_column.py => test_ohlcv_date_column.py} | 20 +- tests/test_ohlcv_latest_bar.py | 32 +-- tests/test_polymarket.py | 3 +- tests/test_reddit_fallback.py | 2 +- tests/test_sec_edgar.py | 2 +- tests/test_social_lookahead.py | 2 +- tests/test_stocktwits_resilience.py | 2 +- tests/test_symbol_normalization_paths.py | 8 +- tests/test_undated_tools_as_of.py | 73 +++--- tests/test_vendor_errors.py | 10 +- ...ta_validator.py => test_yahoo_snapshot.py} | 2 +- tests/test_yfinance_stale_ohlcv_guard.py | 8 +- .../agents/analysts/sentiment_analyst.py | 4 +- tradingagents/agents/utils/agent_utils.py | 2 +- .../utils/market_data_validation_tools.py | 2 +- tradingagents/dataflows/router.py | 26 ++- tradingagents/dataflows/vendors/__init__.py | 1 + .../alpha_vantage/__init__.py} | 8 +- .../alpha_vantage/common.py} | 0 .../alpha_vantage/fundamentals.py} | 2 +- .../alpha_vantage/indicator.py} | 2 +- .../alpha_vantage/news.py} | 5 +- .../alpha_vantage/stock.py} | 2 +- tradingagents/dataflows/{ => vendors}/fred.py | 0 .../dataflows/{ => vendors}/polymarket.py | 0 .../dataflows/{ => vendors}/reddit.py | 0 .../dataflows/{ => vendors}/sec_edgar.py | 0 .../dataflows/{ => vendors}/stocktwits.py | 0 .../dataflows/vendors/yahoo/__init__.py | 1 + .../dataflows/vendors/yahoo/fundamentals.py | 211 +++++++++++++++++ .../{y_finance.py => vendors/yahoo/market.py} | 220 +++--------------- .../yahoo/news.py} | 2 +- .../yahoo/ohlcv.py} | 44 +--- .../yahoo/snapshot.py} | 2 +- tradingagents/graph/trading_graph.py | 2 +- 45 files changed, 417 insertions(+), 389 deletions(-) rename tests/{test_stockstats_date_column.py => test_ohlcv_date_column.py} (76%) rename tests/{test_market_data_validator.py => test_yahoo_snapshot.py} (98%) create mode 100644 tradingagents/dataflows/vendors/__init__.py rename tradingagents/dataflows/{alpha_vantage.py => vendors/alpha_vantage/__init__.py} (64%) rename tradingagents/dataflows/{alpha_vantage_common.py => vendors/alpha_vantage/common.py} (100%) rename tradingagents/dataflows/{alpha_vantage_fundamentals.py => vendors/alpha_vantage/fundamentals.py} (96%) rename tradingagents/dataflows/{alpha_vantage_indicator.py => vendors/alpha_vantage/indicator.py} (99%) rename tradingagents/dataflows/{alpha_vantage_news.py => vendors/alpha_vantage/news.py} (96%) rename tradingagents/dataflows/{alpha_vantage_stock.py => vendors/alpha_vantage/stock.py} (95%) rename tradingagents/dataflows/{ => vendors}/fred.py (100%) rename tradingagents/dataflows/{ => vendors}/polymarket.py (100%) rename tradingagents/dataflows/{ => vendors}/reddit.py (100%) rename tradingagents/dataflows/{ => vendors}/sec_edgar.py (100%) rename tradingagents/dataflows/{ => vendors}/stocktwits.py (100%) create mode 100644 tradingagents/dataflows/vendors/yahoo/__init__.py create mode 100644 tradingagents/dataflows/vendors/yahoo/fundamentals.py rename tradingagents/dataflows/{y_finance.py => vendors/yahoo/market.py} (58%) rename tradingagents/dataflows/{yfinance_news.py => vendors/yahoo/news.py} (99%) rename tradingagents/dataflows/{stockstats_utils.py => vendors/yahoo/ohlcv.py} (88%) rename tradingagents/dataflows/{market_data_validator.py => vendors/yahoo/snapshot.py} (98%) diff --git a/tests/test_alpha_vantage_hardening.py b/tests/test_alpha_vantage_hardening.py index cc9aaf210..0d15bf12b 100644 --- a/tests/test_alpha_vantage_hardening.py +++ b/tests/test_alpha_vantage_hardening.py @@ -10,10 +10,10 @@ import json import pytest -import tradingagents.dataflows.alpha_vantage_common as av -import tradingagents.dataflows.alpha_vantage_fundamentals as avf -import tradingagents.dataflows.alpha_vantage_stock as avs import tradingagents.dataflows.net as net +import tradingagents.dataflows.vendors.alpha_vantage.common as av +import tradingagents.dataflows.vendors.alpha_vantage.fundamentals as avf +import tradingagents.dataflows.vendors.alpha_vantage.stock as avs class _FakeResponse: @@ -156,7 +156,7 @@ def test_request_error_message_carries_no_key(monkeypatch): @pytest.mark.unit def test_global_news_omitted_optionals_use_the_configured_defaults(monkeypatch): """The tool passes None for an omitted look_back_days or limit (#1326).""" - from tradingagents.dataflows import alpha_vantage_news + from tradingagents.dataflows.vendors.alpha_vantage import news as alpha_vantage_news monkeypatch.setattr(alpha_vantage_news, "get_config", lambda: {"global_news_lookback_days": 3, "global_news_article_limit": 9}) @@ -173,7 +173,7 @@ def test_the_news_window_includes_the_analysis_day(monkeypatch): """time_to was midnight at the start of the end date, so everything published during the analysis day, the most decision-relevant day, was excluded. The yfinance path includes it.""" - from tradingagents.dataflows import alpha_vantage_news + from tradingagents.dataflows.vendors.alpha_vantage import news as alpha_vantage_news seen = {} monkeypatch.setattr(alpha_vantage_news, "_make_api_request", @@ -190,8 +190,8 @@ def test_the_news_window_includes_the_analysis_day(monkeypatch): def test_an_indicator_this_vendor_lacks_lets_the_next_one_serve_it(indicator): """Returning prose counts as success to the router, so the chain stops at a vendor that cannot compute the indicator while the next one can.""" - from tradingagents.dataflows import alpha_vantage_indicator from tradingagents.dataflows.errors import VendorError + from tradingagents.dataflows.vendors.alpha_vantage import indicator as alpha_vantage_indicator with pytest.raises(VendorError): alpha_vantage_indicator.get_indicator("AAPL", indicator, "2026-05-08", 30) @@ -201,7 +201,7 @@ def test_an_indicator_this_vendor_lacks_lets_the_next_one_serve_it(indicator): def test_ticker_news_asks_for_only_as_many_articles_as_configured(monkeypatch): """The endpoint returns 50 articles with per-article sentiment arrays by default, and the whole payload went into the prompt.""" - from tradingagents.dataflows import alpha_vantage_news + from tradingagents.dataflows.vendors.alpha_vantage import news as alpha_vantage_news monkeypatch.setattr(alpha_vantage_news, "get_config", lambda: {"news_article_limit": 8}) seen = {} diff --git a/tests/test_date_boundaries.py b/tests/test_date_boundaries.py index 263a8cbb6..80d0f08e1 100644 --- a/tests/test_date_boundaries.py +++ b/tests/test_date_boundaries.py @@ -7,9 +7,9 @@ row omitted). import pandas as pd import pytest -import tradingagents.dataflows.stockstats_utils as su -import tradingagents.dataflows.y_finance as yfin +import tradingagents.dataflows.vendors.yahoo.market as yfin from tradingagents.dataflows.config import set_config +from tradingagents.dataflows.vendors.yahoo import ohlcv @pytest.mark.unit @@ -53,9 +53,9 @@ def test_load_ohlcv_requests_inclusive_end(monkeypatch, tmp_path): index=idx, ) - monkeypatch.setattr(su.yf, "download", fake_download) + monkeypatch.setattr(ohlcv.yf, "download", fake_download) today = pd.Timestamp.today().strftime("%Y-%m-%d") - su.load_ohlcv("AAPL", today) + ohlcv.load_ohlcv("AAPL", today) expected_end = (pd.Timestamp.today() + pd.Timedelta(days=1)).strftime("%Y-%m-%d") assert captured["end"] == expected_end # tomorrow -> today's row included (#986) diff --git a/tests/test_fred.py b/tests/test_fred.py index e393ac327..6bcb6bbfc 100644 --- a/tests/test_fred.py +++ b/tests/test_fred.py @@ -12,8 +12,9 @@ import requests import tradingagents.dataflows.config as config_module import tradingagents.default_config as default_config -from tradingagents.dataflows import fred, router +from tradingagents.dataflows import router from tradingagents.dataflows.config import set_config +from tradingagents.dataflows.vendors import fred # A small, stable set of observations to format against. _META = { diff --git a/tests/test_fundamentals_lookahead.py b/tests/test_fundamentals_lookahead.py index bbd91f2ca..d604c330e 100644 --- a/tests/test_fundamentals_lookahead.py +++ b/tests/test_fundamentals_lookahead.py @@ -19,7 +19,12 @@ from unittest import mock import pytest -from tradingagents.dataflows import alpha_vantage_fundamentals as av, date_window, y_finance +from tradingagents.dataflows import date_window +from tradingagents.dataflows.vendors.alpha_vantage import fundamentals as av +from tradingagents.dataflows.vendors.yahoo import ( + fundamentals as yahoo_fundamentals, + market as yahoo_market, +) _TODAY = "2026-09-07" _PAST = "2024-05-10" @@ -42,9 +47,9 @@ _LEAKY = ("3500000000000", "34.2", "260.1", "391000000000", def _yf(curr_date, info=_INFO, today=_TODAY): with mock.patch.object(date_window, "get_current_date", return_value=today), \ - mock.patch.object(y_finance, "yf_retry", lambda fn: info), \ - mock.patch.object(y_finance.yf, "Ticker"): - return y_finance.get_fundamentals("AAPL", curr_date) + mock.patch.object(yahoo_fundamentals, "yf_retry", lambda fn: info), \ + mock.patch.object(yahoo_market.yf, "Ticker"): + return yahoo_fundamentals.get_fundamentals("AAPL", curr_date) def _av(curr_date, today=_TODAY): @@ -78,8 +83,8 @@ class TestYFinanceHistoricalRun: # The response would only be discarded; skipping it also avoids burning # vendor quota on a call whose result cannot be used. with mock.patch.object(date_window, "get_current_date", return_value=_TODAY), \ - mock.patch.object(y_finance.yf, "Ticker") as tk: - y_finance.get_fundamentals("AAPL", _PAST) + mock.patch.object(yahoo_market.yf, "Ticker") as tk: + yahoo_fundamentals.get_fundamentals("AAPL", _PAST) tk.assert_not_called() diff --git a/tests/test_graph_end_to_end.py b/tests/test_graph_end_to_end.py index 003a692c4..65e0bdf63 100644 --- a/tests/test_graph_end_to_end.py +++ b/tests/test_graph_end_to_end.py @@ -20,7 +20,8 @@ from pydantic import Field from tradingagents.agents import schemas from tradingagents.agents.analysts import sentiment_analyst from tradingagents.agents.utils import agent_utils -from tradingagents.dataflows import market_data_validator, router, y_finance +from tradingagents.dataflows import router +from tradingagents.dataflows.vendors.yahoo import market as yahoo_market, snapshot from tradingagents.default_config import DEFAULT_CONFIG from tradingagents.graph import trading_graph @@ -102,11 +103,11 @@ def offline(monkeypatch, tmp_path): "Date": pd.bdate_range(end=TRADE_DATE, periods=60), "Open": 100.0, "High": 101.0, "Low": 99.0, "Close": 100.5, "Volume": 1_000_000, }) - monkeypatch.setattr(market_data_validator, "load_ohlcv", + monkeypatch.setattr(snapshot, "load_ohlcv", lambda *a, **k: called.add("ohlcv") or prices.copy()) monkeypatch.setattr(sentiment_analyst, "fetch_stocktwits_messages", lambda *a, **k: "no posts") monkeypatch.setattr(sentiment_analyst, "fetch_reddit_posts", lambda *a, **k: "no posts") - monkeypatch.setattr(y_finance.yf, "Ticker", lambda s: type("T", (), {"info": {"longName": "NVIDIA"}})()) + monkeypatch.setattr(yahoo_market.yf, "Ticker", lambda s: type("T", (), {"info": {"longName": "NVIDIA"}})()) agent_utils.resolve_instrument_identity.cache_clear() return called diff --git a/tests/test_instrument_identity.py b/tests/test_instrument_identity.py index 5930f13ed..c226b48b5 100644 --- a/tests/test_instrument_identity.py +++ b/tests/test_instrument_identity.py @@ -21,7 +21,7 @@ class ResolveInstrumentIdentityTests(unittest.TestCase): resolve_instrument_identity.cache_clear() def test_resolves_company_metadata_from_yfinance(self): - with patch("tradingagents.dataflows.y_finance.yf.Ticker") as mock: + with patch("tradingagents.dataflows.vendors.yahoo.market.yf.Ticker") as mock: mock.return_value.info = { "longName": "TOTO LTD.", "shortName": "TOTO", @@ -38,26 +38,26 @@ class ResolveInstrumentIdentityTests(unittest.TestCase): self.assertEqual(identity["exchange"], "PNK") def test_falls_back_to_short_name(self): - with patch("tradingagents.dataflows.y_finance.yf.Ticker") as mock: + with patch("tradingagents.dataflows.vendors.yahoo.market.yf.Ticker") as mock: mock.return_value.info = {"shortName": "TOTO", "sector": "Industrials"} identity = resolve_instrument_identity("TOTDY") self.assertEqual(identity["company_name"], "TOTO") def test_skips_placeholder_values(self): - with patch("tradingagents.dataflows.y_finance.yf.Ticker") as mock: + with patch("tradingagents.dataflows.vendors.yahoo.market.yf.Ticker") as mock: mock.return_value.info = {"longName": " ", "sector": "None", "industry": "n/a"} identity = resolve_instrument_identity("TOTDY") self.assertEqual(identity, {}) def test_fails_open_on_exception(self): with patch( - "tradingagents.dataflows.y_finance.yf.Ticker", + "tradingagents.dataflows.vendors.yahoo.market.yf.Ticker", side_effect=RuntimeError("rate limited"), ): self.assertEqual(resolve_instrument_identity("TOTDY"), {}) def test_result_is_cached(self): - with patch("tradingagents.dataflows.y_finance.yf.Ticker") as mock: + with patch("tradingagents.dataflows.vendors.yahoo.market.yf.Ticker") as mock: mock.return_value.info = {"longName": "TOTO LTD."} first = resolve_instrument_identity("TOTDY") second = resolve_instrument_identity("TOTDY") @@ -104,7 +104,7 @@ class GetInstrumentContextFromStateTests(unittest.TestCase): def test_fallback_is_network_free_ticker_only(self): # No instrument_context and no yfinance call — must not hit the network. - with patch("tradingagents.dataflows.y_finance.yf.Ticker") as mock: + with patch("tradingagents.dataflows.vendors.yahoo.market.yf.Ticker") as mock: context = get_instrument_context_from_state( {"company_of_interest": "NVDA", "asset_type": "stock"} ) diff --git a/tests/test_memory_log.py b/tests/test_memory_log.py index b8fea6544..1cb894443 100644 --- a/tests/test_memory_log.py +++ b/tests/test_memory_log.py @@ -1042,7 +1042,7 @@ def test_a_longer_window_asks_for_enough_price_history(monkeypatch): days = pd.bdate_range(start, end) return pd.DataFrame({"Close": range(len(days))}, index=days) - monkeypatch.setattr("tradingagents.dataflows.y_finance.yf.Ticker", _Ticker) + monkeypatch.setattr("tradingagents.dataflows.vendors.yahoo.market.yf.Ticker", _Ticker) raw, alpha, days, resolved = graph._fetch_returns("NVDA", "2026-06-01", 21, benchmark="SPY") diff --git a/tests/test_news_lookahead.py b/tests/test_news_lookahead.py index 01ce1a08f..09717d5e6 100644 --- a/tests/test_news_lookahead.py +++ b/tests/test_news_lookahead.py @@ -10,7 +10,7 @@ from datetime import datetime, timezone import pytest -import tradingagents.dataflows.yfinance_news as ynews +import tradingagents.dataflows.vendors.yahoo.news as ynews from tradingagents.dataflows.date_window import in_window diff --git a/tests/test_no_data_handling.py b/tests/test_no_data_handling.py index fe85a3373..7b63d91ff 100644 --- a/tests/test_no_data_handling.py +++ b/tests/test_no_data_handling.py @@ -14,9 +14,10 @@ from unittest import mock import pandas as pd import pytest -from tradingagents.dataflows import router, stockstats_utils +from tradingagents.dataflows import router from tradingagents.dataflows.config import set_config from tradingagents.dataflows.errors import NoMarketDataError +from tradingagents.dataflows.vendors.yahoo import ohlcv @pytest.mark.unit @@ -33,16 +34,16 @@ class TestLoadOhlcvNoPoison(unittest.TestCase): def test_empty_download_raises_and_does_not_cache(self): empty = pd.DataFrame() - with mock.patch.object(stockstats_utils.yf, "download", return_value=empty), \ + with mock.patch.object(ohlcv.yf, "download", return_value=empty), \ self.assertRaises(NoMarketDataError): - stockstats_utils.load_ohlcv("FAKE", "2026-01-01") + ohlcv.load_ohlcv("FAKE", "2026-01-01") # Nothing should have been written to the cache. self.assertEqual(os.listdir(self._tmp), []) # A second call must re-attempt the fetch (no poisoned cache served). - with mock.patch.object(stockstats_utils.yf, "download", return_value=empty) as dl2: + with mock.patch.object(ohlcv.yf, "download", return_value=empty) as dl2: with self.assertRaises(NoMarketDataError): - stockstats_utils.load_ohlcv("FAKE", "2026-01-01") + ohlcv.load_ohlcv("FAKE", "2026-01-01") self.assertTrue(dl2.called) diff --git a/tests/test_ohlcv_cache_freshness.py b/tests/test_ohlcv_cache_freshness.py index f11313c58..c7718b41c 100644 --- a/tests/test_ohlcv_cache_freshness.py +++ b/tests/test_ohlcv_cache_freshness.py @@ -12,10 +12,10 @@ import os import pandas as pd import pytest -import tradingagents.dataflows.stockstats_utils as su +from tradingagents.dataflows.vendors.yahoo import ohlcv NOW = pd.Timestamp("2026-07-18 12:00") -STALE = su.OHLCV_CACHE_TTL_SECONDS + 60 +STALE = ohlcv.OHLCV_CACHE_TTL_SECONDS + 60 def _stamp(path, ts): @@ -33,10 +33,10 @@ def _write(tmp_path, name="AAPL-YFin-data.csv", age_seconds=0.0, last_date="2026 def _load(tmp_path, monkeypatch, curr_date, download): - monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) - monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: NOW)) - monkeypatch.setattr(su.yf, "download", download) - return su.load_ohlcv("AAPL", curr_date) + monkeypatch.setattr(ohlcv, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) + monkeypatch.setattr(ohlcv.pd.Timestamp, "today", staticmethod(lambda: NOW)) + monkeypatch.setattr(ohlcv.yf, "download", download) + return ohlcv.load_ohlcv("AAPL", curr_date) def _fail_download(*a, **k): @@ -46,27 +46,27 @@ def _fail_download(*a, **k): @pytest.mark.unit def test_current_day_cache_past_ttl_is_not_fresh(tmp_path): # Today's bar missing or still in progress: row inspection can't tell, so the TTL governs. - assert su._cache_is_fresh(_write(tmp_path, age_seconds=STALE), NOW.normalize(), NOW) is False + assert ohlcv._cache_is_fresh(_write(tmp_path, age_seconds=STALE), NOW.normalize(), NOW) is False f = _write(tmp_path, age_seconds=STALE, last_date="2026-07-18") - assert su._cache_is_fresh(f, NOW.normalize(), NOW) is False + assert ohlcv._cache_is_fresh(f, NOW.normalize(), NOW) is False @pytest.mark.unit def test_recent_cache_is_fresh(tmp_path): # Written moments ago: don't hammer the vendor (weekend/holiday guard). - assert su._cache_is_fresh(_write(tmp_path), NOW.normalize(), NOW) is True + assert ohlcv._cache_is_fresh(_write(tmp_path), NOW.normalize(), NOW) is True @pytest.mark.unit def test_historical_request_uses_todays_cache_past_the_ttl(tmp_path): f = _write(tmp_path, age_seconds=STALE, last_date="2026-04-30") - assert su._cache_is_fresh(f, pd.Timestamp("2026-05-01"), NOW) is True + assert ohlcv._cache_is_fresh(f, pd.Timestamp("2026-05-01"), NOW) is True @pytest.mark.unit def test_a_download_from_an_earlier_day_is_not_fresh(tmp_path): f = _write(tmp_path, age_seconds=13 * 3600) # yesterday 23:00 - assert su._cache_is_fresh(f, pd.Timestamp("2026-05-01"), NOW) is False + assert ohlcv._cache_is_fresh(f, pd.Timestamp("2026-05-01"), NOW) is False @pytest.mark.unit @@ -95,15 +95,15 @@ def test_load_ohlcv_reuses_fresh_same_day_cache(tmp_path, monkeypatch): @pytest.mark.unit def test_one_cache_file_per_symbol_across_days(tmp_path, monkeypatch): """A later day's download replaces the symbol's file instead of adding one (#1330).""" - monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) + monkeypatch.setattr(ohlcv, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) frame = pd.DataFrame({"Date": pd.to_datetime(["2026-07-16", "2026-07-17"]), "Close": [1.0, 2.0]}) downloads = [] - monkeypatch.setattr(su.yf, "download", lambda *a, **k: downloads.append(1) or frame.set_index("Date")) + monkeypatch.setattr(ohlcv.yf, "download", lambda *a, **k: downloads.append(1) or frame.set_index("Date")) for day in ("2026-07-18 10:00", "2026-07-19 10:00", "2026-07-20 10:00"): now = pd.Timestamp(day) - monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda now=now: now)) - su.load_ohlcv("AAPL", "2026-07-17") + monkeypatch.setattr(ohlcv.pd.Timestamp, "today", staticmethod(lambda now=now: now)) + ohlcv.load_ohlcv("AAPL", "2026-07-17") written = list(tmp_path.glob("AAPL-*.csv")) _stamp(written[0], now) diff --git a/tests/test_stockstats_date_column.py b/tests/test_ohlcv_date_column.py similarity index 76% rename from tests/test_stockstats_date_column.py rename to tests/test_ohlcv_date_column.py index e55a5c70c..8c1cd76df 100644 --- a/tests/test_stockstats_date_column.py +++ b/tests/test_ohlcv_date_column.py @@ -1,4 +1,4 @@ -"""Tests for tolerating a non-`Date` index column in stockstats_utils (#890). +"""Tests for tolerating a non-`Date` index column in the Yahoo OHLCV loader (#890). Guards against a download frame whose date column is `index` or `Datetime` instead of `Date`, which would otherwise silently drop every indicator. @@ -9,7 +9,7 @@ from __future__ import annotations import pandas as pd import pytest -from tradingagents.dataflows import stockstats_utils as su +from tradingagents.dataflows.vendors.yahoo import ohlcv def _ohlcv(date_col: str) -> pd.DataFrame: @@ -28,20 +28,20 @@ def _ohlcv(date_col: str) -> pd.DataFrame: @pytest.mark.unit class TestEnsureDateColumn: def test_renames_index_column(self): - out = su._ensure_date_column(_ohlcv("index")) + out = ohlcv._ensure_date_column(_ohlcv("index")) assert "Date" in out.columns and "index" not in out.columns def test_renames_datetime_and_date_variants(self): - assert "Date" in su._ensure_date_column(_ohlcv("Datetime")).columns - assert "Date" in su._ensure_date_column(_ohlcv("date")).columns + assert "Date" in ohlcv._ensure_date_column(_ohlcv("Datetime")).columns + assert "Date" in ohlcv._ensure_date_column(_ohlcv("date")).columns def test_leaves_existing_date_untouched(self): df = _ohlcv("Date") - assert su._ensure_date_column(df) is df # no-op short-circuit + assert ohlcv._ensure_date_column(df) is df # no-op short-circuit def test_no_datelike_column_is_left_alone(self): df = pd.DataFrame({"Close": [1, 2, 3]}) - out = su._ensure_date_column(df) + out = ohlcv._ensure_date_column(df) assert "Date" not in out.columns # nothing to rename; caller handles @@ -50,20 +50,20 @@ class TestCleanDataframeAcrossVersions: def test_clean_handles_index_column(self): """A frame with `index` instead of `Date` must still clean to a usable, date-parsed frame (was KeyError: 'Date').""" - cleaned = su._clean_dataframe(_ohlcv("index")) + cleaned = ohlcv._clean_dataframe(_ohlcv("index")) assert "Date" in cleaned.columns assert pd.api.types.is_datetime64_any_dtype(cleaned["Date"]) assert len(cleaned) == 10 def test_clean_handles_legacy_date_column(self): - cleaned = su._clean_dataframe(_ohlcv("Date")) + cleaned = ohlcv._clean_dataframe(_ohlcv("Date")) assert len(cleaned) == 10 def test_indicators_compute_after_index_rename(self): """stockstats must compute indicators on a frame whose date column arrived as `index`, instead of erroring per indicator.""" from stockstats import wrap - cleaned = su._clean_dataframe(_ohlcv("index")) + cleaned = ohlcv._clean_dataframe(_ohlcv("index")) df = wrap(cleaned) df["close_5_sma"] # triggers calculation assert "close_5_sma" in df.columns diff --git a/tests/test_ohlcv_latest_bar.py b/tests/test_ohlcv_latest_bar.py index 3975e06ba..b326d3ef2 100644 --- a/tests/test_ohlcv_latest_bar.py +++ b/tests/test_ohlcv_latest_bar.py @@ -18,8 +18,8 @@ import os import pandas as pd import pytest -from tradingagents.dataflows import stockstats_utils as su from tradingagents.dataflows.errors import NoMarketDataError +from tradingagents.dataflows.vendors.yahoo import ohlcv def _stamp(path, ts): @@ -35,7 +35,7 @@ def test_normalize_dates_strips_tz_and_normalizes_to_midnight(): aware = pd.Series(pd.to_datetime( ["2026-05-08 09:30:00-04:00", "2026-05-09 16:00:00-04:00"] )) - out = su._normalize_dates(aware) + out = ohlcv._normalize_dates(aware) assert out.dt.tz is None assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")] @@ -43,7 +43,7 @@ def test_normalize_dates_strips_tz_and_normalizes_to_midnight(): @pytest.mark.unit def test_normalize_dates_leaves_naive_dates_at_midnight(): naive = pd.Series(pd.to_datetime(["2026-05-08 14:30:00", "2026-05-09 00:00:00"])) - out = su._normalize_dates(naive) + out = ohlcv._normalize_dates(naive) assert out.dt.tz is None assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")] @@ -57,7 +57,7 @@ def test_normalize_dates_handles_mixed_dst_offsets(): "2026-06-08 00:00:00-04:00", # EDT "not-a-date", # -> NaT ]) - out = su._normalize_dates(mixed) + out = ohlcv._normalize_dates(mixed) assert out.iloc[0] == pd.Timestamp("2026-01-08") assert out.iloc[1] == pd.Timestamp("2026-06-08") assert pd.isna(out.iloc[2]) @@ -68,7 +68,7 @@ def test_normalize_dates_keeps_positive_offset_local_date(): # A Tokyo bar at local midnight (+09:00) must stay on its own calendar day, # not shift to the previous UTC day (which utc=True parsing would cause). jst = pd.Series(["2026-05-08 00:00:00+09:00"]) - assert su._normalize_dates(jst).iloc[0] == pd.Timestamp("2026-05-08") + assert ohlcv._normalize_dates(jst).iloc[0] == pd.Timestamp("2026-05-08") # --- fill vs guard responsibilities ---------------------------------------- @@ -77,7 +77,7 @@ def test_normalize_dates_keeps_positive_offset_local_date(): def test_clean_dataframe_keeps_nan_close_for_the_caller_to_inspect(): # _clean_dataframe normalizes but no longer drops the NaN close itself. df = pd.DataFrame({"Date": ["2026-05-08", "2026-05-09"], "Close": [100.0, float("nan")]}) - cleaned = su._clean_dataframe(df) + cleaned = ohlcv._clean_dataframe(df) assert len(cleaned) == 2 assert pd.isna(cleaned["Close"].iloc[-1]) @@ -86,7 +86,7 @@ def test_clean_dataframe_keeps_nan_close_for_the_caller_to_inspect(): def test_fill_price_gaps_drops_nan_close_rows(): df = pd.DataFrame({"Date": pd.to_datetime(["2026-05-07", "2026-05-08"]), "Close": [float("nan"), 100.0]}) - filled = su._fill_price_gaps(df) + filled = ohlcv._fill_price_gaps(df) assert len(filled) == 1 assert filled["Close"].iloc[0] == 100.0 @@ -95,17 +95,17 @@ def test_fill_price_gaps_drops_nan_close_rows(): def _run_load(monkeypatch, tmp_path, frame, curr_date): """Drive load_ohlcv against a pre-seeded cache frame (no network).""" - monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) + monkeypatch.setattr(ohlcv, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) today = pd.Timestamp(curr_date) - monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today)) + monkeypatch.setattr(ohlcv.pd.Timestamp, "today", staticmethod(lambda: today)) cache_file = tmp_path / "AAPL-YFin-data.csv" cache_file.write_text(frame.to_csv(index=False)) _stamp(cache_file, today) def _fail_download(*a, **k): raise AssertionError("should use the seeded cache, not download") - monkeypatch.setattr(su.yf, "download", _fail_download) - return su.load_ohlcv("AAPL", curr_date) + monkeypatch.setattr(ohlcv.yf, "download", _fail_download) + return ohlcv.load_ohlcv("AAPL", curr_date) @pytest.mark.unit @@ -181,7 +181,7 @@ def test_the_snapshot_does_not_present_a_filled_price_as_reported(monkeypatch, t """Gap filling exists so indicators compute on a continuous series. The verification snapshot is the one place a number must be what the vendor reported, or the module built to stop invented prices supplies them.""" - from tradingagents.dataflows import market_data_validator as mdv, stockstats_utils as su + from tradingagents.dataflows.vendors.yahoo import ohlcv, snapshot frame = pd.DataFrame({ "Date": ["2026-05-06", "2026-05-07", "2026-05-08"], @@ -192,15 +192,15 @@ def test_the_snapshot_does_not_present_a_filled_price_as_reported(monkeypatch, t "Volume": [1000000, 1000000, ""], }) today = pd.Timestamp("2026-05-08 12:00") - monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) - monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today)) + monkeypatch.setattr(ohlcv, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) + monkeypatch.setattr(ohlcv.pd.Timestamp, "today", staticmethod(lambda: today)) cache = tmp_path / "AAPL-YFin-data.csv" cache.write_text(frame.to_csv(index=False)) _stamp(cache, today) - monkeypatch.setattr(su.yf, "download", lambda *a, **k: (_ for _ in ()).throw( + monkeypatch.setattr(ohlcv.yf, "download", lambda *a, **k: (_ for _ in ()).throw( AssertionError("should read the seeded cache"))) - out = mdv.build_verified_market_snapshot("AAPL", "2026-05-08", 3) + out = snapshot.build_verified_market_snapshot("AAPL", "2026-05-08", 3) row = out.split("Latest verified OHLCV row")[1].split("###")[0] assert "104.50" not in row and "105.50" not in row # the previous session's numbers diff --git a/tests/test_polymarket.py b/tests/test_polymarket.py index 1db11dc9e..5d06a7563 100644 --- a/tests/test_polymarket.py +++ b/tests/test_polymarket.py @@ -12,8 +12,9 @@ import requests import tradingagents.dataflows.config as config_module import tradingagents.default_config as default_config -from tradingagents.dataflows import polymarket, router +from tradingagents.dataflows import router from tradingagents.dataflows.config import set_config +from tradingagents.dataflows.vendors import polymarket def _market(question, prob, *, volume, end_date, closed=False, wk=None): diff --git a/tests/test_reddit_fallback.py b/tests/test_reddit_fallback.py index fedd08160..79096b996 100644 --- a/tests/test_reddit_fallback.py +++ b/tests/test_reddit_fallback.py @@ -9,7 +9,7 @@ from urllib.error import HTTPError import pytest -from tradingagents.dataflows import reddit +from tradingagents.dataflows.vendors import reddit _SAMPLE_ATOM = """ diff --git a/tests/test_sec_edgar.py b/tests/test_sec_edgar.py index eaebcc83e..40b2def9f 100644 --- a/tests/test_sec_edgar.py +++ b/tests/test_sec_edgar.py @@ -13,8 +13,8 @@ from unittest import mock import pytest -from tradingagents.dataflows import sec_edgar from tradingagents.dataflows.errors import NoMarketDataError +from tradingagents.dataflows.vendors import sec_edgar _REAL_FETCH = sec_edgar._fetch_json diff --git a/tests/test_social_lookahead.py b/tests/test_social_lookahead.py index 09a4d0648..92427f680 100644 --- a/tests/test_social_lookahead.py +++ b/tests/test_social_lookahead.py @@ -12,8 +12,8 @@ from datetime import datetime, timezone import pytest -from tradingagents.dataflows import reddit, stocktwits from tradingagents.dataflows.date_window import in_window +from tradingagents.dataflows.vendors import reddit, stocktwits class _JsonResp: diff --git a/tests/test_stocktwits_resilience.py b/tests/test_stocktwits_resilience.py index 8e963f5c5..429c0a138 100644 --- a/tests/test_stocktwits_resilience.py +++ b/tests/test_stocktwits_resilience.py @@ -13,7 +13,7 @@ from urllib.error import HTTPError import pytest -from tradingagents.dataflows import stocktwits +from tradingagents.dataflows.vendors import stocktwits def _raise(exc): diff --git a/tests/test_symbol_normalization_paths.py b/tests/test_symbol_normalization_paths.py index 4f4fde383..fbddbcb41 100644 --- a/tests/test_symbol_normalization_paths.py +++ b/tests/test_symbol_normalization_paths.py @@ -8,8 +8,8 @@ hit the right instrument instead of failing/mismatching. import pandas as pd import tradingagents.agents.utils.agent_utils as au -import tradingagents.dataflows.y_finance as y_finance -import tradingagents.dataflows.yfinance_news as ynews +import tradingagents.dataflows.vendors.yahoo.market as yahoo_market +import tradingagents.dataflows.vendors.yahoo.news as ynews from tradingagents.graph.trading_graph import TradingAgentsGraph @@ -24,7 +24,7 @@ def test_identity_lookup_normalizes_symbol(monkeypatch): def info(self): return {"longName": "Gold Futures", "quoteType": "FUTURE"} - monkeypatch.setattr(y_finance.yf, "Ticker", FakeTicker) + monkeypatch.setattr(yahoo_market.yf, "Ticker", FakeTicker) au.resolve_instrument_identity.cache_clear() identity = au.resolve_instrument_identity("XAUUSD") @@ -45,7 +45,7 @@ def test_fetch_returns_normalizes_symbol(monkeypatch): idx = pd.date_range(start="2025-01-02", periods=len(prices), freq="D") return pd.DataFrame({"Close": prices}, index=idx) - monkeypatch.setattr(y_finance.yf, "Ticker", FakeTicker) + monkeypatch.setattr(yahoo_market.yf, "Ticker", FakeTicker) # _fetch_returns does not use ``self``; call unbound to avoid building the graph. raw, alpha, days, resolved = TradingAgentsGraph._fetch_returns( diff --git a/tests/test_undated_tools_as_of.py b/tests/test_undated_tools_as_of.py index 304c55682..254489a68 100644 --- a/tests/test_undated_tools_as_of.py +++ b/tests/test_undated_tools_as_of.py @@ -14,7 +14,12 @@ import pandas as pd import pytest from tradingagents.agents.utils import news_data_tools, prediction_markets_tools -from tradingagents.dataflows import alpha_vantage_news, polymarket, y_finance +from tradingagents.dataflows.vendors import polymarket +from tradingagents.dataflows.vendors.alpha_vantage import news as alpha_vantage_news +from tradingagents.dataflows.vendors.yahoo import ( + fundamentals as yahoo_fundamentals, + market as yahoo_market, +) def _insider_frame(*dates): @@ -27,8 +32,8 @@ def _insider_frame(*dates): def _yf_insider(frame, curr_date): ticker = mock.Mock(insider_transactions=frame) - with mock.patch.object(y_finance.yf, "Ticker", return_value=ticker): - return y_finance.get_insider_transactions("AAPL", curr_date) + with mock.patch.object(yahoo_market.yf, "Ticker", return_value=ticker): + return yahoo_fundamentals.get_insider_transactions("AAPL", curr_date) @pytest.mark.unit @@ -121,16 +126,14 @@ def test_insider_rows_are_dated_by_the_trade_not_the_filing(): run must not be told these rows were public on their transaction date.""" import pandas as pd - from tradingagents.dataflows import y_finance - frame = pd.DataFrame({ "Shares": [100, 200], "Text": ["Sale at price 10.00 per share.", "Sale at price 11.00 per share."], "Start Date": pd.to_datetime(["2026-05-01", "2026-05-20"]), }) ticker = mock.Mock(insider_transactions=frame) - with mock.patch.object(y_finance.yf, "Ticker", return_value=ticker): - out = y_finance.get_insider_transactions("AAPL", "2026-05-10") + with mock.patch.object(yahoo_market.yf, "Ticker", return_value=ticker): + out = yahoo_fundamentals.get_insider_transactions("AAPL", "2026-05-10") assert "2026-05-01" in out and "2026-05-20" not in out # still bounded by the date assert "transaction date" in out.lower() # and says what the date means @@ -142,13 +145,12 @@ def test_an_indicator_that_could_not_be_read_is_not_shown_as_a_blank_value(): """The per-day fallback returned an empty string for a failed read, so the table rendered a row per day with nothing after the colon: an analyst reads that as "no value on that day" rather than "could not be obtained".""" - from tradingagents.dataflows import y_finance from tradingagents.dataflows.errors import VendorError - with mock.patch.object(y_finance.StockstatsUtils, "get_stock_stats", + with mock.patch.object(yahoo_market, "get_stock_stats", side_effect=RuntimeError("cache parse failed")), \ pytest.raises(VendorError): - y_finance.get_stockstats_indicator("AAPL", "rsi", "2026-05-08") + yahoo_market.get_stockstats_indicator("AAPL", "rsi", "2026-05-08") @pytest.mark.unit @@ -165,12 +167,11 @@ def test_a_yfinance_failure_is_a_vendor_error_not_a_report(func, args): """Returning the failure as text makes the router count it as an answer, so the chain stops and the analyst reads the error message as if it were data. yfinance serves the default path, so this is the one that matters most.""" - from tradingagents.dataflows import y_finance from tradingagents.dataflows.errors import VendorError - with mock.patch.object(y_finance.yf, "Ticker", side_effect=RuntimeError("yahoo hiccup")), \ + with mock.patch.object(yahoo_market.yf, "Ticker", side_effect=RuntimeError("yahoo hiccup")), \ pytest.raises(VendorError): - getattr(y_finance, func)(*args) + getattr(yahoo_fundamentals, func)(*args) @pytest.mark.unit @@ -179,13 +180,13 @@ def test_a_yfinance_failure_is_a_vendor_error_not_a_report(func, args): ("get_global_news_yfinance", ("2026-09-01", 7, 5)), ]) def test_a_yfinance_news_failure_is_a_vendor_error_not_a_report(func, args): - from tradingagents.dataflows import yfinance_news from tradingagents.dataflows.errors import VendorError + from tradingagents.dataflows.vendors.yahoo import news as yahoo_news target = "Ticker" if "global" not in func else "Search" - with mock.patch.object(yfinance_news.yf, target, side_effect=RuntimeError("yahoo hiccup")), \ + with mock.patch.object(yahoo_news.yf, target, side_effect=RuntimeError("yahoo hiccup")), \ pytest.raises(VendorError): - getattr(yfinance_news, func)(*args) + getattr(yahoo_news, func)(*args) @pytest.mark.unit @@ -195,19 +196,19 @@ def test_an_unreachable_vendor_is_not_reported_as_a_missing_symbol(monkeypatch): company has no balance sheet, when the truth is we could not ask.""" import pandas as pd - from tradingagents.dataflows import stockstats_utils, y_finance from tradingagents.dataflows.errors import NoMarketDataError, VendorRateLimitError + from tradingagents.dataflows.vendors.yahoo import ohlcv empty = mock.Mock(quarterly_balance_sheet=pd.DataFrame(), balance_sheet=pd.DataFrame()) - monkeypatch.setattr(y_finance.yf, "Ticker", lambda s: empty) + monkeypatch.setattr(yahoo_market.yf, "Ticker", lambda s: empty) - monkeypatch.setattr(stockstats_utils, "vendor_reachable", lambda url: False) + monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: False) with pytest.raises(VendorRateLimitError, match="unreachable"): - y_finance.get_balance_sheet("AAPL", "annual", "2026-09-01") + yahoo_fundamentals.get_balance_sheet("AAPL", "annual", "2026-09-01") - monkeypatch.setattr(stockstats_utils, "vendor_reachable", lambda url: True) + monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: True) with pytest.raises(NoMarketDataError): - y_finance.get_balance_sheet("AAPL", "annual", "2026-09-01") + yahoo_fundamentals.get_balance_sheet("AAPL", "annual", "2026-09-01") @pytest.mark.unit @@ -234,27 +235,29 @@ def test_the_price_path_also_tells_an_outage_from_an_unknown_symbol(monkeypatch) delisted symbol either.""" import pandas as pd - from tradingagents.dataflows import stockstats_utils, y_finance from tradingagents.dataflows.errors import NoMarketDataError, VendorRateLimitError + from tradingagents.dataflows.vendors.yahoo import ohlcv - monkeypatch.setattr(y_finance.yf, "Ticker", lambda s: mock.Mock(history=lambda **k: pd.DataFrame())) + monkeypatch.setattr(yahoo_market.yf, "Ticker", lambda s: mock.Mock(history=lambda **k: pd.DataFrame())) - monkeypatch.setattr(stockstats_utils, "vendor_reachable", lambda url: False) + monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: False) with pytest.raises(VendorRateLimitError, match="unreachable"): - y_finance.get_YFin_data_online("AAPL", "2026-09-01", "2026-09-10") + yahoo_market.get_YFin_data_online("AAPL", "2026-09-01", "2026-09-10") - monkeypatch.setattr(stockstats_utils, "vendor_reachable", lambda url: True) + monkeypatch.setattr(ohlcv, "vendor_reachable", lambda url: True) with pytest.raises(NoMarketDataError): - y_finance.get_YFin_data_online("AAPL", "2026-09-01", "2026-09-10") + yahoo_market.get_YFin_data_online("AAPL", "2026-09-01", "2026-09-10") @pytest.mark.unit @pytest.mark.parametrize("func, args", [ - ("get_YFin_data_online", ("AAPL", "2025-06-02", "2025-06-06")), - ("get_balance_sheet", ("AAPL", "quarterly", "2025-06-06")), - ("get_cashflow", ("AAPL", "quarterly", "2025-06-06")), - ("get_income_statement", ("AAPL", "quarterly", "2025-06-06")), - ("get_insider_transactions", ("AAPL", "2025-06-06")), + pytest.param(f, a, id=f.__name__) for f, a in ( + (yahoo_market.get_YFin_data_online, ("AAPL", "2025-06-02", "2025-06-06")), + (yahoo_fundamentals.get_balance_sheet, ("AAPL", "quarterly", "2025-06-06")), + (yahoo_fundamentals.get_cashflow, ("AAPL", "quarterly", "2025-06-06")), + (yahoo_fundamentals.get_income_statement, ("AAPL", "quarterly", "2025-06-06")), + (yahoo_fundamentals.get_insider_transactions, ("AAPL", "2025-06-06")), + ) ]) def test_a_historical_run_is_not_told_todays_date(func, args): """A header stamped with the wall clock tells a backtest when it is really running.""" @@ -267,8 +270,8 @@ def test_a_historical_run_is_not_told_todays_date(func, args): quarterly_income_stmt=statement, insider_transactions=_insider_frame("2025-05-30"), history=lambda **k: prices) - with mock.patch.object(y_finance.yf, "Ticker", return_value=ticker): - out = getattr(y_finance, func)(*args) + with mock.patch.object(yahoo_market.yf, "Ticker", return_value=ticker): + out = func(*args) assert date.today().isoformat() not in out diff --git a/tests/test_vendor_errors.py b/tests/test_vendor_errors.py index 7b31334ea..51e2f15fd 100644 --- a/tests/test_vendor_errors.py +++ b/tests/test_vendor_errors.py @@ -11,10 +11,6 @@ import pytest import tradingagents.dataflows.config as config_module import tradingagents.default_config as default_config from tradingagents.dataflows import router -from tradingagents.dataflows.alpha_vantage_common import ( - AlphaVantageNotConfiguredError, - AlphaVantageRateLimitError, -) from tradingagents.dataflows.config import set_config from tradingagents.dataflows.errors import ( NoMarketDataError, @@ -22,7 +18,11 @@ from tradingagents.dataflows.errors import ( VendorNotConfiguredError, VendorRateLimitError, ) -from tradingagents.dataflows.fred import FredNotConfiguredError +from tradingagents.dataflows.vendors.alpha_vantage.common import ( + AlphaVantageNotConfiguredError, + AlphaVantageRateLimitError, +) +from tradingagents.dataflows.vendors.fred import FredNotConfiguredError @pytest.mark.unit diff --git a/tests/test_market_data_validator.py b/tests/test_yahoo_snapshot.py similarity index 98% rename from tests/test_market_data_validator.py rename to tests/test_yahoo_snapshot.py index c26d6f537..066cf517e 100644 --- a/tests/test_market_data_validator.py +++ b/tests/test_yahoo_snapshot.py @@ -5,7 +5,7 @@ from __future__ import annotations import pandas as pd import pytest -import tradingagents.dataflows.market_data_validator as validator +import tradingagents.dataflows.vendors.yahoo.snapshot as validator def _sample_ohlcv() -> pd.DataFrame: diff --git a/tests/test_yfinance_stale_ohlcv_guard.py b/tests/test_yfinance_stale_ohlcv_guard.py index f1faaa3d4..5acff5cfb 100644 --- a/tests/test_yfinance_stale_ohlcv_guard.py +++ b/tests/test_yfinance_stale_ohlcv_guard.py @@ -13,12 +13,12 @@ import pandas as pd import pytest import tradingagents.dataflows.config as config_module -import tradingagents.dataflows.y_finance as y_finance +import tradingagents.dataflows.vendors.yahoo.market as yahoo_market import tradingagents.default_config as default_config from tradingagents.dataflows import router from tradingagents.dataflows.config import set_config from tradingagents.dataflows.errors import NoMarketDataError -from tradingagents.dataflows.stockstats_utils import _assert_ohlcv_not_stale +from tradingagents.dataflows.vendors.yahoo.ohlcv import _assert_ohlcv_not_stale def _frame(date): @@ -76,9 +76,9 @@ class StaleGuardPropagationTests(unittest.TestCase): def history(self, start, end): return stale - with mock.patch.object(y_finance.yf, "Ticker", DummyTicker), \ + with mock.patch.object(yahoo_market.yf, "Ticker", DummyTicker), \ self.assertRaises(NoMarketDataError): - y_finance.get_YFin_data_online("CB", "2026-06-01", "2026-06-11") + yahoo_market.get_YFin_data_online("CB", "2026-06-01", "2026-06-11") @pytest.mark.unit diff --git a/tradingagents/agents/analysts/sentiment_analyst.py b/tradingagents/agents/analysts/sentiment_analyst.py index acabea433..9dca1bd6d 100644 --- a/tradingagents/agents/analysts/sentiment_analyst.py +++ b/tradingagents/agents/analysts/sentiment_analyst.py @@ -32,8 +32,8 @@ from tradingagents.agents.utils.structured import ( bind_structured, invoke_structured_or_freetext, ) -from tradingagents.dataflows.reddit import fetch_reddit_posts -from tradingagents.dataflows.stocktwits import fetch_stocktwits_messages +from tradingagents.dataflows.vendors.reddit import fetch_reddit_posts +from tradingagents.dataflows.vendors.stocktwits import fetch_stocktwits_messages def _seven_days_back(trade_date: str) -> str: diff --git a/tradingagents/agents/utils/agent_utils.py b/tradingagents/agents/utils/agent_utils.py index 04841a1eb..2454ee0b2 100644 --- a/tradingagents/agents/utils/agent_utils.py +++ b/tradingagents/agents/utils/agent_utils.py @@ -23,7 +23,7 @@ from tradingagents.agents.utils.news_data_tools import ( from tradingagents.agents.utils.prediction_markets_tools import get_prediction_markets from tradingagents.agents.utils.technical_indicators_tools import get_indicators from tradingagents.dataflows.date_window import get_current_date -from tradingagents.dataflows.y_finance import get_company_profile +from tradingagents.dataflows.vendors.yahoo.fundamentals import get_company_profile # Public surface: the data tools are imported here so agents and the graph # import them from one place, plus the instrument/language helpers defined below. diff --git a/tradingagents/agents/utils/market_data_validation_tools.py b/tradingagents/agents/utils/market_data_validation_tools.py index 853bda007..5ace13ef1 100644 --- a/tradingagents/agents/utils/market_data_validation_tools.py +++ b/tradingagents/agents/utils/market_data_validation_tools.py @@ -4,7 +4,7 @@ from langchain_core.tools import tool from langgraph.prebuilt import InjectedState from tradingagents.dataflows.date_window import as_of -from tradingagents.dataflows.market_data_validator import build_verified_market_snapshot +from tradingagents.dataflows.vendors.yahoo.snapshot import build_verified_market_snapshot @tool diff --git a/tradingagents/dataflows/router.py b/tradingagents/dataflows/router.py index 6b3deecd8..8f6d277af 100644 --- a/tradingagents/dataflows/router.py +++ b/tradingagents/dataflows/router.py @@ -1,6 +1,12 @@ import logging -from tradingagents.dataflows.alpha_vantage import ( +from tradingagents.dataflows.config import get_config +from tradingagents.dataflows.errors import ( + NoMarketDataError, + VendorNotConfiguredError, + VendorRateLimitError, +) +from tradingagents.dataflows.vendors.alpha_vantage import ( get_balance_sheet as get_alpha_vantage_balance_sheet, get_cashflow as get_alpha_vantage_cashflow, get_fundamentals as get_alpha_vantage_fundamentals, @@ -11,31 +17,27 @@ from tradingagents.dataflows.alpha_vantage import ( get_news as get_alpha_vantage_news, get_stock as get_alpha_vantage_stock, ) -from tradingagents.dataflows.config import get_config -from tradingagents.dataflows.errors import ( - NoMarketDataError, - VendorNotConfiguredError, - VendorRateLimitError, -) -from tradingagents.dataflows.fred import get_macro_data as get_fred_macro_data -from tradingagents.dataflows.polymarket import ( +from tradingagents.dataflows.vendors.fred import get_macro_data as get_fred_macro_data +from tradingagents.dataflows.vendors.polymarket import ( get_prediction_markets as get_polymarket_prediction_markets, ) -from tradingagents.dataflows.sec_edgar import ( +from tradingagents.dataflows.vendors.sec_edgar import ( get_balance_sheet as get_sec_edgar_balance_sheet, get_cashflow as get_sec_edgar_cashflow, get_income_statement as get_sec_edgar_income_statement, ) -from tradingagents.dataflows.y_finance import ( +from tradingagents.dataflows.vendors.yahoo.fundamentals import ( get_balance_sheet as get_yfinance_balance_sheet, get_cashflow as get_yfinance_cashflow, get_fundamentals as get_yfinance_fundamentals, get_income_statement as get_yfinance_income_statement, get_insider_transactions as get_yfinance_insider_transactions, +) +from tradingagents.dataflows.vendors.yahoo.market import ( get_stock_stats_indicators_window, get_YFin_data_online, ) -from tradingagents.dataflows.yfinance_news import get_global_news_yfinance, get_news_yfinance +from tradingagents.dataflows.vendors.yahoo.news import get_global_news_yfinance, get_news_yfinance logger = logging.getLogger(__name__) diff --git a/tradingagents/dataflows/vendors/__init__.py b/tradingagents/dataflows/vendors/__init__.py new file mode 100644 index 000000000..3d695e663 --- /dev/null +++ b/tradingagents/dataflows/vendors/__init__.py @@ -0,0 +1 @@ +"""Data vendors: one module or package per source, serving the router's methods.""" diff --git a/tradingagents/dataflows/alpha_vantage.py b/tradingagents/dataflows/vendors/alpha_vantage/__init__.py similarity index 64% rename from tradingagents/dataflows/alpha_vantage.py rename to tradingagents/dataflows/vendors/alpha_vantage/__init__.py index 68ead2f8a..d2acecda0 100644 --- a/tradingagents/dataflows/alpha_vantage.py +++ b/tradingagents/dataflows/vendors/alpha_vantage/__init__.py @@ -1,18 +1,18 @@ # Aggregates the per-category Alpha Vantage implementations into one module the # vendor router imports from; the imports below are the public surface. -from tradingagents.dataflows.alpha_vantage_fundamentals import ( +from tradingagents.dataflows.vendors.alpha_vantage.fundamentals import ( get_balance_sheet, get_cashflow, get_fundamentals, get_income_statement, ) -from tradingagents.dataflows.alpha_vantage_indicator import get_indicator -from tradingagents.dataflows.alpha_vantage_news import ( +from tradingagents.dataflows.vendors.alpha_vantage.indicator import get_indicator +from tradingagents.dataflows.vendors.alpha_vantage.news import ( get_global_news, get_insider_transactions, get_news, ) -from tradingagents.dataflows.alpha_vantage_stock import get_stock +from tradingagents.dataflows.vendors.alpha_vantage.stock import get_stock __all__ = [ "get_balance_sheet", diff --git a/tradingagents/dataflows/alpha_vantage_common.py b/tradingagents/dataflows/vendors/alpha_vantage/common.py similarity index 100% rename from tradingagents/dataflows/alpha_vantage_common.py rename to tradingagents/dataflows/vendors/alpha_vantage/common.py diff --git a/tradingagents/dataflows/alpha_vantage_fundamentals.py b/tradingagents/dataflows/vendors/alpha_vantage/fundamentals.py similarity index 96% rename from tradingagents/dataflows/alpha_vantage_fundamentals.py rename to tradingagents/dataflows/vendors/alpha_vantage/fundamentals.py index e92d2c5fa..a53698190 100644 --- a/tradingagents/dataflows/alpha_vantage_fundamentals.py +++ b/tradingagents/dataflows/vendors/alpha_vantage/fundamentals.py @@ -1,7 +1,7 @@ import json -from tradingagents.dataflows.alpha_vantage_common import _make_api_request from tradingagents.dataflows.date_window import withhold_live_profile +from tradingagents.dataflows.vendors.alpha_vantage.common import _make_api_request def _filter_reports_by_date(result, curr_date: str): diff --git a/tradingagents/dataflows/alpha_vantage_indicator.py b/tradingagents/dataflows/vendors/alpha_vantage/indicator.py similarity index 99% rename from tradingagents/dataflows/alpha_vantage_indicator.py rename to tradingagents/dataflows/vendors/alpha_vantage/indicator.py index 46e401576..a40a1594b 100644 --- a/tradingagents/dataflows/alpha_vantage_indicator.py +++ b/tradingagents/dataflows/vendors/alpha_vantage/indicator.py @@ -1,7 +1,7 @@ import logging -from tradingagents.dataflows.alpha_vantage_common import _make_api_request from tradingagents.dataflows.errors import NoMarketDataError, VendorError +from tradingagents.dataflows.vendors.alpha_vantage.common import _make_api_request logger = logging.getLogger(__name__) diff --git a/tradingagents/dataflows/alpha_vantage_news.py b/tradingagents/dataflows/vendors/alpha_vantage/news.py similarity index 96% rename from tradingagents/dataflows/alpha_vantage_news.py rename to tradingagents/dataflows/vendors/alpha_vantage/news.py index 4b446a15c..76c16b89f 100644 --- a/tradingagents/dataflows/alpha_vantage_news.py +++ b/tradingagents/dataflows/vendors/alpha_vantage/news.py @@ -1,7 +1,10 @@ import json -from tradingagents.dataflows.alpha_vantage_common import _make_api_request, format_datetime_for_api from tradingagents.dataflows.config import get_config +from tradingagents.dataflows.vendors.alpha_vantage.common import ( + _make_api_request, + format_datetime_for_api, +) def get_news(ticker, start_date, end_date) -> dict[str, str] | str: diff --git a/tradingagents/dataflows/alpha_vantage_stock.py b/tradingagents/dataflows/vendors/alpha_vantage/stock.py similarity index 95% rename from tradingagents/dataflows/alpha_vantage_stock.py rename to tradingagents/dataflows/vendors/alpha_vantage/stock.py index 50b050911..b42923e58 100644 --- a/tradingagents/dataflows/alpha_vantage_stock.py +++ b/tradingagents/dataflows/vendors/alpha_vantage/stock.py @@ -1,6 +1,6 @@ from datetime import datetime -from tradingagents.dataflows.alpha_vantage_common import ( +from tradingagents.dataflows.vendors.alpha_vantage.common import ( _filter_csv_by_date_range, _make_api_request, ) diff --git a/tradingagents/dataflows/fred.py b/tradingagents/dataflows/vendors/fred.py similarity index 100% rename from tradingagents/dataflows/fred.py rename to tradingagents/dataflows/vendors/fred.py diff --git a/tradingagents/dataflows/polymarket.py b/tradingagents/dataflows/vendors/polymarket.py similarity index 100% rename from tradingagents/dataflows/polymarket.py rename to tradingagents/dataflows/vendors/polymarket.py diff --git a/tradingagents/dataflows/reddit.py b/tradingagents/dataflows/vendors/reddit.py similarity index 100% rename from tradingagents/dataflows/reddit.py rename to tradingagents/dataflows/vendors/reddit.py diff --git a/tradingagents/dataflows/sec_edgar.py b/tradingagents/dataflows/vendors/sec_edgar.py similarity index 100% rename from tradingagents/dataflows/sec_edgar.py rename to tradingagents/dataflows/vendors/sec_edgar.py diff --git a/tradingagents/dataflows/stocktwits.py b/tradingagents/dataflows/vendors/stocktwits.py similarity index 100% rename from tradingagents/dataflows/stocktwits.py rename to tradingagents/dataflows/vendors/stocktwits.py diff --git a/tradingagents/dataflows/vendors/yahoo/__init__.py b/tradingagents/dataflows/vendors/yahoo/__init__.py new file mode 100644 index 000000000..a5798011a --- /dev/null +++ b/tradingagents/dataflows/vendors/yahoo/__init__.py @@ -0,0 +1 @@ +"""Yahoo Finance: prices, indicators, statements, insider filings and news.""" diff --git a/tradingagents/dataflows/vendors/yahoo/fundamentals.py b/tradingagents/dataflows/vendors/yahoo/fundamentals.py new file mode 100644 index 000000000..7684a794a --- /dev/null +++ b/tradingagents/dataflows/vendors/yahoo/fundamentals.py @@ -0,0 +1,211 @@ +from typing import Annotated + +import pandas as pd +import yfinance as yf + +from tradingagents.dataflows.date_window import withhold_live_profile +from tradingagents.dataflows.errors import NoMarketDataError, VendorError, VendorRateLimitError +from tradingagents.dataflows.net import vendor_reachable +from tradingagents.dataflows.symbols import normalize_symbol +from tradingagents.dataflows.vendors.yahoo.ohlcv import ( + YAHOO_HOST, + raise_for_empty, + yf_retry, +) + + +def get_fundamentals( + ticker: Annotated[str, "ticker symbol of the company"], + curr_date: Annotated[str, "analysis date in YYYY-MM-DD format"] = None +): + """Get company fundamentals overview from yfinance. + + ``Ticker.info`` is a present-day snapshot with no historical vintage, so a + past ``curr_date`` withholds it through the shared point-in-time guard + (``date_window.withhold_live_profile``, #1300). + """ + canonical = normalize_symbol(ticker) + + # Guard before the request: the response would only be discarded, and the + # answer does not depend on it. + withheld = withhold_live_profile(curr_date, canonical) + if withheld: + return withheld + + try: + ticker_obj = yf.Ticker(canonical) + info = yf_retry(lambda: ticker_obj.info) + + if not info: + raise_for_empty(ticker, canonical, "fundamentals") + + fields = [ + ("Name", info.get("longName")), + ("Sector", info.get("sector")), + ("Industry", info.get("industry")), + ("Market Cap", info.get("marketCap")), + ("PE Ratio (TTM)", info.get("trailingPE")), + ("Forward PE", info.get("forwardPE")), + ("PEG Ratio", info.get("pegRatio")), + ("Price to Book", info.get("priceToBook")), + ("EPS (TTM)", info.get("trailingEps")), + ("Forward EPS", info.get("forwardEps")), + ("Dividend Yield", info.get("dividendYield")), + ("Beta", info.get("beta")), + ("52 Week High", info.get("fiftyTwoWeekHigh")), + ("52 Week Low", info.get("fiftyTwoWeekLow")), + ("50 Day Average", info.get("fiftyDayAverage")), + ("200 Day Average", info.get("twoHundredDayAverage")), + ("Revenue (TTM)", info.get("totalRevenue")), + ("Gross Profit", info.get("grossProfits")), + ("EBITDA", info.get("ebitda")), + ("Net Income", info.get("netIncomeToCommon")), + ("Profit Margin", info.get("profitMargins")), + ("Operating Margin", info.get("operatingMargins")), + ("Return on Equity", info.get("returnOnEquity")), + ("Return on Assets", info.get("returnOnAssets")), + ("Debt to Equity", info.get("debtToEquity")), + ("Current Ratio", info.get("currentRatio")), + ("Book Value", info.get("bookValue")), + ("Free Cash Flow", info.get("freeCashflow")), + ] + + lines = [f"{label}: {v}" for label, v in fields if v is not None] + + # yfinance returns a stub dict (e.g. {"trailingPegRatio": None}) for + # unknown symbols, so `info` is truthy but every field is empty. Treat + # "no usable fields" as no data rather than emitting a bare header the + # agent might fabricate around. + if not lines: + raise NoMarketDataError(ticker, canonical, "no fundamental fields returned") + + header = f"# Company Fundamentals for {canonical}\n\n" + + return header + "\n".join(lines) + + except VendorError: + raise + except Exception as e: + raise NoMarketDataError(ticker, canonical, f"fundamentals unavailable: {e}") from e + + +# This vendor dates a statement by the period it covers, not by the day it was +# filed, and carries no filing date to do better. A company files weeks after its +# period ends, so a run dated in that gap can be served figures that were not yet +# public. Say so rather than implying the stricter guarantee (SEC EDGAR, which +# does carry filing dates, serves US filers as filed). +_PERIOD_END_VINTAGE = ( + "# Periods are cut at the fiscal period end; this vendor does not report " + "filing dates, so the most recent period may not have been published yet.\n\n" +) + + +def _statement(ticker, freq, curr_date, title, quarterly_attr, annual_attr) -> str: + """One financial statement as CSV, cut at ``curr_date`` by period end.""" + canonical = normalize_symbol(ticker) + what = title.lower() + try: + ticker_obj = yf.Ticker(canonical) + attr = quarterly_attr if freq.lower() == "quarterly" else annual_attr + data = filter_financials_by_date(yf_retry(lambda: getattr(ticker_obj, attr)), curr_date) + if data.empty: + raise_for_empty(ticker, canonical, f"{what} data") + return f"# {title} data for {canonical} ({freq})\n" + _PERIOD_END_VINTAGE + data.to_csv() + except VendorError: + raise + except Exception as e: + raise NoMarketDataError(ticker, canonical, f"{what} unavailable: {e}") from e + + +def get_balance_sheet( + ticker: Annotated[str, "ticker symbol of the company"], + freq: Annotated[str, "frequency of data: 'annual' or 'quarterly'"] = "quarterly", + curr_date: Annotated[str, "current date in YYYY-MM-DD format"] = None +): + """Get balance sheet data from yfinance.""" + return _statement(ticker, freq, curr_date, "Balance Sheet", "quarterly_balance_sheet", "balance_sheet") + + +def get_cashflow( + ticker: Annotated[str, "ticker symbol of the company"], + freq: Annotated[str, "frequency of data: 'annual' or 'quarterly'"] = "quarterly", + curr_date: Annotated[str, "current date in YYYY-MM-DD format"] = None +): + """Get cash flow data from yfinance.""" + return _statement(ticker, freq, curr_date, "Cash Flow", "quarterly_cashflow", "cashflow") + + +def get_income_statement( + ticker: Annotated[str, "ticker symbol of the company"], + freq: Annotated[str, "frequency of data: 'annual' or 'quarterly'"] = "quarterly", + curr_date: Annotated[str, "current date in YYYY-MM-DD format"] = None +): + """Get income statement data from yfinance.""" + return _statement(ticker, freq, curr_date, "Income Statement", "quarterly_income_stmt", "income_stmt") + + +# Rows are dated by the transaction, which is when the insider traded, not when +# the market learned of it: a Form 4 is filed up to two business days later and +# this vendor reports no filing date, so the most recent rows may not have been +# public on the analysis date. +_TRANSACTION_DATE_VINTAGE = ( + "# Rows are dated by transaction date. A trade becomes public when its Form 4 " + "is filed, up to two business days later, so the newest rows may not have been " + "known on this date.\n\n" +) + + +def get_insider_transactions( + ticker: Annotated[str, "ticker symbol of the company"], + curr_date: Annotated[str | None, "only transactions on or before this date, yyyy-mm-dd"] = None, +): + """Get insider transactions data from yfinance.""" + canonical = normalize_symbol(ticker) + try: + ticker_obj = yf.Ticker(canonical) + data = yf_retry(lambda: ticker_obj.insider_transactions) + + # Empty is normal here (many valid symbols have no insider filings), + # so report it plainly rather than treating the symbol as invalid. + if data is None or data.empty: + if not vendor_reachable(YAHOO_HOST): + raise VendorRateLimitError("Yahoo Finance is unreachable; insider filings were not retrieved") + return f"No insider transactions reported for symbol '{canonical}'" + + if curr_date: + traded = data["Start Date"] + kept = data[traded <= pd.Timestamp(curr_date)] + if kept.empty: + return ( + f"" + ) + data = kept + + return f"# Insider Transactions data for {canonical}\n" + _TRANSACTION_DATE_VINTAGE + data.to_csv() + + except Exception as e: + raise NoMarketDataError(ticker, canonical, f"insider transactions unavailable: {e}") from e + + +def get_company_profile(ticker: str) -> dict: + """Yahoo's current profile for ``ticker``: name, sector, industry and the like.""" + canonical = normalize_symbol(ticker) + try: + return yf_retry(lambda: yf.Ticker(canonical).info) or {} + except Exception as e: + raise NoMarketDataError(ticker, canonical, f"profile unavailable: {e}") from e + + +def filter_financials_by_date(data: pd.DataFrame, curr_date: str) -> pd.DataFrame: + """Drop financial statement columns (fiscal period timestamps) after curr_date. + + yfinance financial statements use fiscal period end dates as columns. + Columns after curr_date represent future data and are removed to + prevent look-ahead bias. + """ + if not curr_date or data.empty: + return data + cutoff = pd.Timestamp(curr_date) + mask = pd.to_datetime(data.columns, errors="coerce") <= cutoff + return data.loc[:, mask] diff --git a/tradingagents/dataflows/y_finance.py b/tradingagents/dataflows/vendors/yahoo/market.py similarity index 58% rename from tradingagents/dataflows/y_finance.py rename to tradingagents/dataflows/vendors/yahoo/market.py index 251eb9d7b..31a0784da 100644 --- a/tradingagents/dataflows/y_finance.py +++ b/tradingagents/dataflows/vendors/yahoo/market.py @@ -5,21 +5,16 @@ from typing import Annotated import pandas as pd import yfinance as yf from dateutil.relativedelta import relativedelta +from stockstats import wrap -from tradingagents.dataflows.date_window import withhold_live_profile -from tradingagents.dataflows.errors import NoMarketDataError, VendorError, VendorRateLimitError -from tradingagents.dataflows.net import vendor_reachable -from tradingagents.dataflows.stockstats_utils import ( - StockstatsUtils, +from tradingagents.dataflows.errors import NoMarketDataError, VendorError +from tradingagents.dataflows.symbols import normalize_symbol +from tradingagents.dataflows.vendors.yahoo.ohlcv import ( _assert_ohlcv_not_stale, - filter_financials_by_date, load_ohlcv, raise_for_empty, yf_retry, ) -from tradingagents.dataflows.symbols import normalize_symbol - -_YAHOO_HOST = "https://query2.finance.yahoo.com" logger = logging.getLogger(__name__) @@ -260,7 +255,7 @@ def get_stockstats_indicator( curr_date = curr_date_dt.strftime("%Y-%m-%d") try: - indicator_value = StockstatsUtils.get_stock_stats( + indicator_value = get_stock_stats( symbol, indicator, curr_date, @@ -278,187 +273,6 @@ def get_stockstats_indicator( return str(indicator_value) -def get_fundamentals( - ticker: Annotated[str, "ticker symbol of the company"], - curr_date: Annotated[str, "analysis date in YYYY-MM-DD format"] = None -): - """Get company fundamentals overview from yfinance. - - ``Ticker.info`` is a present-day snapshot with no historical vintage, so a - past ``curr_date`` withholds it through the shared point-in-time guard - (``date_window.withhold_live_profile``, #1300). - """ - canonical = normalize_symbol(ticker) - - # Guard before the request: the response would only be discarded, and the - # answer does not depend on it. - withheld = withhold_live_profile(curr_date, canonical) - if withheld: - return withheld - - try: - ticker_obj = yf.Ticker(canonical) - info = yf_retry(lambda: ticker_obj.info) - - if not info: - raise_for_empty(ticker, canonical, "fundamentals") - - fields = [ - ("Name", info.get("longName")), - ("Sector", info.get("sector")), - ("Industry", info.get("industry")), - ("Market Cap", info.get("marketCap")), - ("PE Ratio (TTM)", info.get("trailingPE")), - ("Forward PE", info.get("forwardPE")), - ("PEG Ratio", info.get("pegRatio")), - ("Price to Book", info.get("priceToBook")), - ("EPS (TTM)", info.get("trailingEps")), - ("Forward EPS", info.get("forwardEps")), - ("Dividend Yield", info.get("dividendYield")), - ("Beta", info.get("beta")), - ("52 Week High", info.get("fiftyTwoWeekHigh")), - ("52 Week Low", info.get("fiftyTwoWeekLow")), - ("50 Day Average", info.get("fiftyDayAverage")), - ("200 Day Average", info.get("twoHundredDayAverage")), - ("Revenue (TTM)", info.get("totalRevenue")), - ("Gross Profit", info.get("grossProfits")), - ("EBITDA", info.get("ebitda")), - ("Net Income", info.get("netIncomeToCommon")), - ("Profit Margin", info.get("profitMargins")), - ("Operating Margin", info.get("operatingMargins")), - ("Return on Equity", info.get("returnOnEquity")), - ("Return on Assets", info.get("returnOnAssets")), - ("Debt to Equity", info.get("debtToEquity")), - ("Current Ratio", info.get("currentRatio")), - ("Book Value", info.get("bookValue")), - ("Free Cash Flow", info.get("freeCashflow")), - ] - - lines = [f"{label}: {v}" for label, v in fields if v is not None] - - # yfinance returns a stub dict (e.g. {"trailingPegRatio": None}) for - # unknown symbols, so `info` is truthy but every field is empty. Treat - # "no usable fields" as no data rather than emitting a bare header the - # agent might fabricate around. - if not lines: - raise NoMarketDataError(ticker, canonical, "no fundamental fields returned") - - header = f"# Company Fundamentals for {canonical}\n\n" - - return header + "\n".join(lines) - - except VendorError: - raise - except Exception as e: - raise NoMarketDataError(ticker, canonical, f"fundamentals unavailable: {e}") from e - - -# This vendor dates a statement by the period it covers, not by the day it was -# filed, and carries no filing date to do better. A company files weeks after its -# period ends, so a run dated in that gap can be served figures that were not yet -# public. Say so rather than implying the stricter guarantee (SEC EDGAR, which -# does carry filing dates, serves US filers as filed). -_PERIOD_END_VINTAGE = ( - "# Periods are cut at the fiscal period end; this vendor does not report " - "filing dates, so the most recent period may not have been published yet.\n\n" -) - - -def _statement(ticker, freq, curr_date, title, quarterly_attr, annual_attr) -> str: - """One financial statement as CSV, cut at ``curr_date`` by period end.""" - canonical = normalize_symbol(ticker) - what = title.lower() - try: - ticker_obj = yf.Ticker(canonical) - attr = quarterly_attr if freq.lower() == "quarterly" else annual_attr - data = filter_financials_by_date(yf_retry(lambda: getattr(ticker_obj, attr)), curr_date) - if data.empty: - raise_for_empty(ticker, canonical, f"{what} data") - return f"# {title} data for {canonical} ({freq})\n" + _PERIOD_END_VINTAGE + data.to_csv() - except VendorError: - raise - except Exception as e: - raise NoMarketDataError(ticker, canonical, f"{what} unavailable: {e}") from e - - -def get_balance_sheet( - ticker: Annotated[str, "ticker symbol of the company"], - freq: Annotated[str, "frequency of data: 'annual' or 'quarterly'"] = "quarterly", - curr_date: Annotated[str, "current date in YYYY-MM-DD format"] = None -): - """Get balance sheet data from yfinance.""" - return _statement(ticker, freq, curr_date, "Balance Sheet", "quarterly_balance_sheet", "balance_sheet") - - -def get_cashflow( - ticker: Annotated[str, "ticker symbol of the company"], - freq: Annotated[str, "frequency of data: 'annual' or 'quarterly'"] = "quarterly", - curr_date: Annotated[str, "current date in YYYY-MM-DD format"] = None -): - """Get cash flow data from yfinance.""" - return _statement(ticker, freq, curr_date, "Cash Flow", "quarterly_cashflow", "cashflow") - - -def get_income_statement( - ticker: Annotated[str, "ticker symbol of the company"], - freq: Annotated[str, "frequency of data: 'annual' or 'quarterly'"] = "quarterly", - curr_date: Annotated[str, "current date in YYYY-MM-DD format"] = None -): - """Get income statement data from yfinance.""" - return _statement(ticker, freq, curr_date, "Income Statement", "quarterly_income_stmt", "income_stmt") - - -# Rows are dated by the transaction, which is when the insider traded, not when -# the market learned of it: a Form 4 is filed up to two business days later and -# this vendor reports no filing date, so the most recent rows may not have been -# public on the analysis date. -_TRANSACTION_DATE_VINTAGE = ( - "# Rows are dated by transaction date. A trade becomes public when its Form 4 " - "is filed, up to two business days later, so the newest rows may not have been " - "known on this date.\n\n" -) - - -def get_insider_transactions( - ticker: Annotated[str, "ticker symbol of the company"], - curr_date: Annotated[str | None, "only transactions on or before this date, yyyy-mm-dd"] = None, -): - """Get insider transactions data from yfinance.""" - canonical = normalize_symbol(ticker) - try: - ticker_obj = yf.Ticker(canonical) - data = yf_retry(lambda: ticker_obj.insider_transactions) - - # Empty is normal here (many valid symbols have no insider filings), - # so report it plainly rather than treating the symbol as invalid. - if data is None or data.empty: - if not vendor_reachable(_YAHOO_HOST): - raise VendorRateLimitError("Yahoo Finance is unreachable; insider filings were not retrieved") - return f"No insider transactions reported for symbol '{canonical}'" - - if curr_date: - traded = data["Start Date"] - kept = data[traded <= pd.Timestamp(curr_date)] - if kept.empty: - return ( - f"" - ) - data = kept - - return f"# Insider Transactions data for {canonical}\n" + _TRANSACTION_DATE_VINTAGE + data.to_csv() - - except Exception as e: - raise NoMarketDataError(ticker, canonical, f"insider transactions unavailable: {e}") from e - - -def get_company_profile(ticker: str) -> dict: - """Yahoo's current profile for ``ticker``: name, sector, industry and the like.""" - canonical = normalize_symbol(ticker) - try: - return yf_retry(lambda: yf.Ticker(canonical).info) or {} - except Exception as e: - raise NoMarketDataError(ticker, canonical, f"profile unavailable: {e}") from e def get_closes(symbol: str, start_date: str, end_date: str) -> pd.Series: @@ -469,3 +283,27 @@ def get_closes(symbol: str, start_date: str, end_date: str) -> pd.Series: except Exception as e: raise NoMarketDataError(symbol, canonical, f"prices unavailable: {e}") from e return history["Close"] if "Close" in history else pd.Series(dtype=float) + + +def get_stock_stats( + symbol: Annotated[str, "ticker symbol for the company"], + indicator: Annotated[ + str, "quantitative indicators based off of the stock data for the company" + ], + curr_date: Annotated[ + str, "curr date for retrieving stock price data, YYYY-mm-dd" + ], +): + data = load_ohlcv(symbol, curr_date) + df = wrap(data) + df["Date"] = df["Date"].dt.strftime("%Y-%m-%d") + curr_date_str = pd.to_datetime(curr_date).strftime("%Y-%m-%d") + + df[indicator] # trigger stockstats to calculate the indicator + matching_rows = df[df["Date"].str.startswith(curr_date_str)] + + if not matching_rows.empty: + indicator_value = matching_rows[indicator].values[0] + return indicator_value + else: + return "N/A: Not a trading day (weekend or holiday)" diff --git a/tradingagents/dataflows/yfinance_news.py b/tradingagents/dataflows/vendors/yahoo/news.py similarity index 99% rename from tradingagents/dataflows/yfinance_news.py rename to tradingagents/dataflows/vendors/yahoo/news.py index b47f29a07..b97a77ab0 100644 --- a/tradingagents/dataflows/yfinance_news.py +++ b/tradingagents/dataflows/vendors/yahoo/news.py @@ -9,8 +9,8 @@ from dateutil.relativedelta import relativedelta from tradingagents.dataflows.config import get_config from tradingagents.dataflows.date_window import coverage_gap, in_window from tradingagents.dataflows.errors import NoMarketDataError -from tradingagents.dataflows.stockstats_utils import yf_retry from tradingagents.dataflows.symbols import normalize_symbol +from tradingagents.dataflows.vendors.yahoo.ohlcv import yf_retry def _extract_article_data(article: dict) -> dict: diff --git a/tradingagents/dataflows/stockstats_utils.py b/tradingagents/dataflows/vendors/yahoo/ohlcv.py similarity index 88% rename from tradingagents/dataflows/stockstats_utils.py rename to tradingagents/dataflows/vendors/yahoo/ohlcv.py index 843b12575..4c4cacbde 100644 --- a/tradingagents/dataflows/stockstats_utils.py +++ b/tradingagents/dataflows/vendors/yahoo/ohlcv.py @@ -1,11 +1,9 @@ import logging import os import time -from typing import Annotated import pandas as pd import yfinance as yf -from stockstats import wrap from yfinance.exceptions import YFRateLimitError from tradingagents.dataflows.config import get_config @@ -15,7 +13,7 @@ from tradingagents.dataflows.symbols import normalize_symbol, safe_ticker_compon logger = logging.getLogger(__name__) -_YAHOO_HOST = "https://query2.finance.yahoo.com" +YAHOO_HOST = "https://query2.finance.yahoo.com" # A vendor's latest OHLCV row this many calendar days before the requested date # is treated as stale. Generous enough to span long holiday weekends, tight @@ -35,7 +33,7 @@ def raise_for_empty(symbol: str, canonical: str, what: str) -> None: yfinance returns an empty frame for a failed request rather than raising, so without this a Yahoo outage reads as "this symbol has no {what}". """ - if not vendor_reachable(_YAHOO_HOST): + if not vendor_reachable(YAHOO_HOST): raise VendorRateLimitError(f"Yahoo Finance is unreachable; no {what} was retrieved") raise NoMarketDataError(symbol, canonical, f"no {what}") @@ -290,41 +288,3 @@ def load_ohlcv(symbol: str, curr_date: str, fill_gaps: bool = True) -> pd.DataFr return data -def filter_financials_by_date(data: pd.DataFrame, curr_date: str) -> pd.DataFrame: - """Drop financial statement columns (fiscal period timestamps) after curr_date. - - yfinance financial statements use fiscal period end dates as columns. - Columns after curr_date represent future data and are removed to - prevent look-ahead bias. - """ - if not curr_date or data.empty: - return data - cutoff = pd.Timestamp(curr_date) - mask = pd.to_datetime(data.columns, errors="coerce") <= cutoff - return data.loc[:, mask] - - -class StockstatsUtils: - @staticmethod - def get_stock_stats( - symbol: Annotated[str, "ticker symbol for the company"], - indicator: Annotated[ - str, "quantitative indicators based off of the stock data for the company" - ], - curr_date: Annotated[ - str, "curr date for retrieving stock price data, YYYY-mm-dd" - ], - ): - data = load_ohlcv(symbol, curr_date) - df = wrap(data) - df["Date"] = df["Date"].dt.strftime("%Y-%m-%d") - curr_date_str = pd.to_datetime(curr_date).strftime("%Y-%m-%d") - - df[indicator] # trigger stockstats to calculate the indicator - matching_rows = df[df["Date"].str.startswith(curr_date_str)] - - if not matching_rows.empty: - indicator_value = matching_rows[indicator].values[0] - return indicator_value - else: - return "N/A: Not a trading day (weekend or holiday)" diff --git a/tradingagents/dataflows/market_data_validator.py b/tradingagents/dataflows/vendors/yahoo/snapshot.py similarity index 98% rename from tradingagents/dataflows/market_data_validator.py rename to tradingagents/dataflows/vendors/yahoo/snapshot.py index 05c062457..6c490ff71 100644 --- a/tradingagents/dataflows/market_data_validator.py +++ b/tradingagents/dataflows/vendors/yahoo/snapshot.py @@ -15,7 +15,7 @@ from collections.abc import Iterable import pandas as pd from stockstats import wrap -from tradingagents.dataflows.stockstats_utils import load_ohlcv +from tradingagents.dataflows.vendors.yahoo.ohlcv import load_ohlcv # A fixed, common indicator set so the snapshot is the same shape every run. DEFAULT_SNAPSHOT_INDICATORS: tuple[str, ...] = ( diff --git a/tradingagents/graph/trading_graph.py b/tradingagents/graph/trading_graph.py index ba185be3d..caab877c1 100644 --- a/tradingagents/graph/trading_graph.py +++ b/tradingagents/graph/trading_graph.py @@ -17,7 +17,7 @@ from tradingagents.agents.utils.rating import parse_rating from tradingagents.dataflows.config import run_config, set_config from tradingagents.dataflows.date_window import get_current_date from tradingagents.dataflows.symbols import safe_ticker_component -from tradingagents.dataflows.y_finance import get_closes +from tradingagents.dataflows.vendors.yahoo.market import get_closes from tradingagents.default_config import DEFAULT_CONFIG from tradingagents.llm_clients import create_llm_client from tradingagents.reporting import write_report_tree