diff --git a/tests/test_news_lookahead.py b/tests/test_news_lookahead.py index 366605075..98db2bdbe 100644 --- a/tests/test_news_lookahead.py +++ b/tests/test_news_lookahead.py @@ -106,7 +106,11 @@ def test_global_news_empty_after_filter_is_informative(monkeypatch): assert "unavailable" in out and "not an absence" in out -def _ticker_with(articles, monkeypatch): +def _ticker_with(articles, monkeypatch, search_news=(), search_quotes=("AAPL",)): + """A quote feed answering ``articles``, and a search answering the rest. + + By default the search knows the symbol but has no articles, i.e. Yahoo answered. + """ class FakeTicker: def __init__(self, *a, **k): pass @@ -114,7 +118,17 @@ def _ticker_with(articles, monkeypatch): def get_news(self, count=20): return articles + searched = [] + + class FakeSearch: + def __init__(self, query, **k): + searched.append(query) + self.news = list(search_news) + self.quotes = [{"symbol": s} for s in search_quotes] + monkeypatch.setattr(ynews.yf, "Ticker", FakeTicker) + monkeypatch.setattr(ynews.yf, "Search", FakeSearch) + return searched @pytest.mark.unit @@ -163,17 +177,135 @@ def test_coverage_gap_boundaries(dates, expect_gap): @pytest.mark.unit def test_ticker_news_empty_feed_for_a_past_window_is_unavailable(monkeypatch): + from tradingagents.dataflows.errors import VendorUnavailableError + _ticker_with([], monkeypatch) - out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14") - assert "unavailable" in out and "not an absence" in out + with pytest.raises(VendorUnavailableError): + ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14") @pytest.mark.unit def test_ticker_news_null_feed_is_handled(monkeypatch): - # Yahoo can return None instead of a list; that is unavailability, not an error. + # Yahoo can return None instead of a list; that is unavailability, not a crash. + from tradingagents.dataflows.errors import VendorUnavailableError + _ticker_with(None, monkeypatch) - out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14") - assert "unavailable" in out and "Error" not in out + with pytest.raises(VendorUnavailableError): + ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14") + + +def _tagged(title, day, *tickers): + return {"title": title, "publisher": "P", "link": "l", + "providerPublishTime": _epoch(day), "relatedTickers": list(tickers)} + + +@pytest.mark.unit +def test_an_empty_quote_feed_falls_back_to_the_articles_search_tags_with_the_symbol(monkeypatch): + today = datetime.now(UTC).strftime("%Y-%m-%d") + searched = _ticker_with([], monkeypatch, search_news=[ + _tagged("ABOUT AAPL", today, "AAPL", "MSFT"), + _tagged("ABOUT GIS", today, "GIS"), + ]) + out = ynews.get_news_yfinance("AAPL", today, today) + assert searched == ["AAPL"] + assert "ABOUT AAPL" in out and "ABOUT GIS" not in out + + +@pytest.mark.unit +def test_a_quote_feed_with_articles_is_not_searched(monkeypatch): + today = datetime.now(UTC).strftime("%Y-%m-%d") + searched = _ticker_with([_tagged("FEED", today)], monkeypatch) + out = ynews.get_news_yfinance("AAPL", today, today) + assert "FEED" in out and searched == [] + + +@pytest.mark.unit +def test_news_neither_yahoo_source_answers_is_a_vendor_outage(monkeypatch): + from tradingagents.dataflows.errors import VendorUnavailableError + + today = datetime.now(UTC).strftime("%Y-%m-%d") + _ticker_with([], monkeypatch, search_quotes=()) + with pytest.raises(VendorUnavailableError): + ynews.get_news_yfinance("AAPL", today, today) + + +@pytest.mark.unit +def test_a_yahoo_news_outage_moves_to_the_next_configured_vendor(monkeypatch): + from unittest.mock import patch + + from tradingagents.dataflows import router + from tradingagents.dataflows.config import get_config, set_config + + _ticker_with([], monkeypatch, search_quotes=()) + calls = [] + + def other_vendor(*a, **k): + calls.append(a) + return "OTHER VENDOR NEWS" + + saved = dict(get_config()["data_vendors"]) + set_config({"data_vendors": {**saved, "news_data": "yfinance,alpha_vantage"}}) + try: + with patch.dict(router.VENDOR_METHODS, {"get_news": { + "yfinance": ynews.get_news_yfinance, "alpha_vantage": other_vendor, + }}): + out = router.route_to_vendor("get_news", "AAPL", "2026-10-01", "2026-10-02") + finally: + set_config({"data_vendors": saved}) + assert out == "OTHER VENDOR NEWS" and len(calls) == 1 + + +@pytest.mark.unit +def test_search_results_never_count_as_an_absence_of_news(monkeypatch): + # Search returns a sparse, relevance-picked sample: nothing in a window it + # reaches is still no proof that no news was published. + today = datetime.now(UTC).strftime("%Y-%m-%d") + _ticker_with([], monkeypatch, search_news=[ + _tagged("NEWER", today, "AAPL"), _tagged("OLDER", "2026-01-05", "AAPL"), + ]) + out = ynews.get_news_yfinance("AAPL", "2026-03-01", "2026-03-08") + assert "No news found" not in out + assert "unavailable" in out and "not an absence" in out + assert "NEWER" not in out and "OLDER" not in out + + +@pytest.mark.unit +def test_a_symbol_yahoo_has_no_articles_for_is_left_to_the_next_vendor(monkeypatch): + # With its quote feed down, Yahoo serving no article about a symbol it knows + # (every non-US listing today) says nothing about the company's news. + from tradingagents.dataflows.errors import VendorUnavailableError + + today = datetime.now(UTC).strftime("%Y-%m-%d") + _ticker_with([], monkeypatch, search_quotes=("SHEL.L",), + search_news=[_tagged("ABOUT SHEL", today, "SHEL")]) + with pytest.raises(VendorUnavailableError, match="SHEL.L"): + ynews.get_news_yfinance("SHEL.L", today, today) + + +@pytest.mark.unit +def test_search_tags_are_matched_whatever_their_case(monkeypatch): + today = datetime.now(UTC).strftime("%Y-%m-%d") + _ticker_with([], monkeypatch, search_news=[_tagged("ABOUT AAPL", today, "aapl")]) + assert "ABOUT AAPL" in ynews.get_news_yfinance("AAPL", today, today) + + +@pytest.mark.unit +def test_a_search_with_articles_has_answered_even_without_listing_the_symbol(monkeypatch): + today = datetime.now(UTC).strftime("%Y-%m-%d") + _ticker_with([], monkeypatch, search_quotes=(), + search_news=[_tagged("ABOUT AAPL", today, "AAPL")]) + assert "ABOUT AAPL" in ynews.get_news_yfinance("AAPL", today, today) + + +@pytest.mark.unit +def test_a_search_yahoo_answers_with_not_found_is_an_outage(monkeypatch): + from tradingagents.dataflows.errors import VendorUnavailableError + + _ticker_with([], monkeypatch) + monkeypatch.setattr(ynews, "yf_retry", lambda fn: [] if "get_news" in repr(fn.__code__.co_names) else None) + with pytest.raises(VendorUnavailableError): + ynews.get_news_yfinance("AAPL", "2026-10-01", "2026-10-02") + @pytest.mark.unit diff --git a/tests/test_symbol_normalization_paths.py b/tests/test_symbol_normalization_paths.py index eb8a4dc35..a1971445d 100644 --- a/tests/test_symbol_normalization_paths.py +++ b/tests/test_symbol_normalization_paths.py @@ -67,11 +67,20 @@ def test_news_lookup_normalizes_symbol(monkeypatch): def get_news(self, count): return [] + class FakeSearch: + def __init__(self, query, **k): + seen["searched"] = query + self.news = [{"title": "Gold", "publisher": "P", "link": "l", + "providerPublishTime": 1736150400, "relatedTickers": ["GC=F"]}] + self.quotes = [{"symbol": "GC=F"}] + monkeypatch.setattr(ynews.yf, "Ticker", FakeTicker) + monkeypatch.setattr(ynews.yf, "Search", FakeSearch) monkeypatch.setattr(ynews, "yf_retry", lambda fn: fn()) out = ynews.get_news_yfinance("XAUUSD", "2025-01-01", "2025-01-10") assert seen["symbol"] == "GC=F" # news queried with the canonical symbol + assert seen["searched"] == "GC=F" assert "XAUUSD" in out # the user's ticker stays in the report assert "GC=F" in out # provenance noted diff --git a/tradingagents/dataflows/vendors/yahoo/news.py b/tradingagents/dataflows/vendors/yahoo/news.py index 7328beca7..f41bca989 100644 --- a/tradingagents/dataflows/vendors/yahoo/news.py +++ b/tradingagents/dataflows/vendors/yahoo/news.py @@ -8,6 +8,7 @@ from dateutil.relativedelta import relativedelta from tradingagents.dataflows.config import get_config from tradingagents.dataflows.date_window import coverage_gap, in_window +from tradingagents.dataflows.errors import VendorUnavailableError from tradingagents.dataflows.symbols import normalize_symbol from tradingagents.dataflows.vendors.yahoo.common import yf_retry @@ -53,21 +54,48 @@ def _extract_article_data(article: dict) -> dict: } +def _search_news(canonical: str, limit: int) -> tuple[list, bool]: + """Yahoo search's articles tagged with ``canonical``, and whether it knows the symbol. + + Search answers when the quote feed is empty, but its articles are a + relevance-picked sample, and some are tagged loosely; only those tagged with + the symbol are about it. + """ + search = yf_retry(lambda: yf.Search(canonical, news_count=limit)) + if search is None: # Yahoo answered "not found": its search is down + return [], False + tagged = [ + a for a in (search.news or []) + if canonical in (t.upper() for t in a.get("relatedTickers") or []) + ] + knows = any((q.get("symbol") or "").upper() == canonical for q in search.quotes or []) + return tagged, knows or bool(search.news) + + +def _format_articles(articles: list[dict]) -> str: + out = "" + for data in articles: + out += f"### {data['title']} (source: {data['publisher']})\n" + if data["summary"]: + out += f"{data['summary']}\n" + if data["link"]: + out += f"Link: {data['link']}\n" + out += "\n" + return out + + def get_news_yfinance( ticker: str, start_date: str, end_date: str, ) -> str: - """ - Retrieve news for a specific stock ticker using yfinance. + """News about ``ticker`` published in the window, from Yahoo Finance. - Args: - ticker: Stock ticker symbol (e.g., "AAPL") - start_date: Start date in yyyy-mm-dd format - end_date: End date in yyyy-mm-dd format - - Returns: - Formatted string containing news articles + The symbol's quote feed comes first. When it is empty, Yahoo search serves + the articles tagged with the symbol; when it has none, or neither answers, + Yahoo news is unavailable, which is not an absence of news, and a configured + next vendor is tried. Search results are a sample, so finding none in the + window never reads as "no news". """ article_limit = get_config()["news_article_limit"] # Query Yahoo with the canonical symbol, like every other yfinance path — @@ -75,38 +103,35 @@ def get_news_yfinance( # returns no news. Keep the user's ticker in the report header. canonical = normalize_symbol(ticker) resolved = "" if canonical == ticker else f" (resolved to {canonical})" - news = yf_retry(lambda: yf.Ticker(canonical).get_news(count=article_limit)) or [] + subject = f"news for {ticker}{resolved}" + feed = yf_retry(lambda: yf.Ticker(canonical).get_news(count=article_limit)) or [] + articles = [_extract_article_data(a) for a in feed] + if not feed: + found, answered = _search_news(canonical, article_limit) + if not answered: + raise VendorUnavailableError(f"Yahoo Finance returned no news response for {canonical}") + if not found: + # With the quote feed empty, no tagged article says nothing about the + # company's news (Yahoo tags no article with most non-US listings). + raise VendorUnavailableError(f"Yahoo Finance has no news articles tagged {canonical}") + articles = [_extract_article_data(a) for a in found] start_dt = datetime.strptime(start_date, "%Y-%m-%d") end_dt = datetime.strptime(end_date, "%Y-%m-%d") + # Keep only articles within the requested window (look-ahead safe). + in_range = [a for a in articles if in_window(a["pub_date"], start_dt, end_dt)] + if in_range: + return f"## {ticker}{resolved} News, from {start_date} to {end_date}:\n\n{_format_articles(in_range)}" - news_str = "" - filtered_count = 0 - - for article in news: - data = _extract_article_data(article) - - # Keep only articles within the requested window (look-ahead safe). - if not in_window(data["pub_date"], start_dt, end_dt): - continue - - news_str += f"### {data['title']} (source: {data['publisher']})\n" - if data["summary"]: - news_str += f"{data['summary']}\n" - if data["link"]: - news_str += f"Link: {data['link']}\n" - news_str += "\n" - filtered_count += 1 - - if filtered_count == 0: - gap = coverage_gap( - (_extract_article_data(a)["pub_date"] for a in news), - start_date, end_date, "Yahoo Finance news", f"news for {ticker}{resolved}", - ) - return gap or f"No news found for {ticker}{resolved} between {start_date} and {end_date}" - - return f"## {ticker}{resolved} News, from {start_date} to {end_date}:\n\n{news_str}" - + if not feed: + return (f"") + gap = coverage_gap( + (a["pub_date"] for a in articles), + start_date, end_date, "Yahoo Finance news", subject, + ) + return gap or f"No news found for {ticker}{resolved} between {start_date} and {end_date}" def get_global_news_yfinance( @@ -163,14 +188,7 @@ def get_global_news_yfinance( if len(in_window_news) >= limit: break - news_str = "" - for data in in_window_news[:limit]: - news_str += f"### {data['title']} (source: {data['publisher']})\n" - if data["summary"]: - news_str += f"{data['summary']}\n" - if data["link"]: - news_str += f"Link: {data['link']}\n" - news_str += "\n" + news_str = _format_articles(in_window_news[:limit]) # Nothing fell inside the window -> say so rather than return an # empty-bodied report (#993).