From 9dceea6e9d321dff81769c8a86bd247a94530cf0 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Thu, 1 Oct 2026 01:46:11 +0000 Subject: [PATCH] fix(yahoo): read a news article whose fields are null (#1458) - each article field falls back to its default on a null as well as on a missing key, so one malformed article no longer loses a ticker's news --- tests/test_news_lookahead.py | 30 ++++++++++ tradingagents/dataflows/vendors/yahoo/news.py | 60 +++++++++---------- 2 files changed, 58 insertions(+), 32 deletions(-) diff --git a/tests/test_news_lookahead.py b/tests/test_news_lookahead.py index 9d96dfbab..366605075 100644 --- a/tests/test_news_lookahead.py +++ b/tests/test_news_lookahead.py @@ -255,3 +255,33 @@ def test_the_article_limit_still_caps_what_is_returned(monkeypatch): out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=3) assert out.count("### ") == 3 + + +@pytest.mark.unit +@pytest.mark.parametrize("field", ["title", "summary", "provider", "canonicalUrl", "pubDate"]) +def test_an_article_with_a_null_field_is_read_with_its_default(field): + """Yahoo sends some fields as null rather than leaving them out (#1458).""" + content = {"title": "T", "summary": "S", "provider": {"displayName": "P"}, + "canonicalUrl": {"url": "https://x"}, "pubDate": "2026-09-22T10:00:00Z", field: None} + data = ynews._extract_article_data({"content": content}) + assert isinstance(data["title"], str) and isinstance(data["summary"], str) + assert isinstance(data["publisher"], str) and isinstance(data["link"], str) + + +@pytest.mark.unit +@pytest.mark.parametrize("field", ["title", "summary", "publisher", "link"]) +def test_a_flat_article_with_a_null_field_is_read_with_its_default(field): + data = ynews._extract_article_data({"title": "T", "summary": "S", "publisher": "P", "link": "L", + "providerPublishTime": _epoch("2026-09-22"), field: None}) + assert all(isinstance(data[k], str) for k in ("title", "summary", "publisher", "link")) + + +@pytest.mark.unit +def test_one_malformed_article_does_not_lose_the_feed(monkeypatch): + good = {"content": {"title": "Kept", "summary": "", "provider": {"displayName": "Wire"}, + "pubDate": "2026-09-22T10:00:00Z"}} + bad = {"content": {"title": "Also kept", "summary": None, "provider": None, + "pubDate": "2026-09-22T11:00:00Z"}} + _ticker_with([good, bad], monkeypatch) + out = ynews.get_news_yfinance("ZS", "2026-09-20", "2026-09-23") + assert "Kept" in out and "Also kept" in out diff --git a/tradingagents/dataflows/vendors/yahoo/news.py b/tradingagents/dataflows/vendors/yahoo/news.py index 1671daa8b..7328beca7 100644 --- a/tradingagents/dataflows/vendors/yahoo/news.py +++ b/tradingagents/dataflows/vendors/yahoo/news.py @@ -13,48 +13,44 @@ from tradingagents.dataflows.vendors.yahoo.common import yf_retry def _extract_article_data(article: dict) -> dict: - """Extract article data from yfinance news format (handles nested 'content' structure).""" - if "content" in article: - content = article["content"] - title = content.get("title", "No title") - summary = content.get("summary", "") - provider = content.get("provider", {}) - publisher = provider.get("displayName", "Unknown") + """Extract article data from yfinance news format (handles nested 'content' structure). + Yahoo sends some fields as null rather than leaving them out, so each falls + back to its default on a null as well as on a missing key (#1458). + """ + content = article.get("content") + if isinstance(content, dict): url_obj = content.get("canonicalUrl") or content.get("clickThroughUrl") or {} - link = url_obj.get("url", "") - - pub_date_str = content.get("pubDate", "") + pub_date_str = content.get("pubDate") or "" pub_date = None if pub_date_str: with contextlib.suppress(ValueError, AttributeError): pub_date = datetime.fromisoformat(pub_date_str.replace("Z", "+00:00")) return { - "title": title, - "summary": summary, - "publisher": publisher, - "link": link, - "pub_date": pub_date, - } - else: - # Fallback for flat structure. Parse the epoch publish time so flat - # articles are date-filterable too (otherwise they bypass the - # historical window and leak future news, #992/#1007). - pub_date = None - ts = article.get("providerPublishTime") - if ts: - # Epoch seconds are UTC; parse them as UTC-aware so filtering does - # not shift with the host timezone (#1126). - with contextlib.suppress(ValueError, OSError, TypeError): - pub_date = datetime.fromtimestamp(ts, tz=UTC) - return { - "title": article.get("title", "No title"), - "summary": article.get("summary", ""), - "publisher": article.get("publisher", "Unknown"), - "link": article.get("link", ""), + "title": content.get("title") or "No title", + "summary": content.get("summary") or "", + "publisher": (content.get("provider") or {}).get("displayName") or "Unknown", + "link": url_obj.get("url") or "", "pub_date": pub_date, } + # Fallback for flat structure. Parse the epoch publish time so flat + # articles are date-filterable too (otherwise they bypass the + # historical window and leak future news, #992/#1007). + pub_date = None + ts = article.get("providerPublishTime") + if ts: + # Epoch seconds are UTC; parse them as UTC-aware so filtering does + # not shift with the host timezone (#1126). + with contextlib.suppress(ValueError, OSError, TypeError): + pub_date = datetime.fromtimestamp(ts, tz=UTC) + return { + "title": article.get("title") or "No title", + "summary": article.get("summary") or "", + "publisher": article.get("publisher") or "Unknown", + "link": article.get("link") or "", + "pub_date": pub_date, + } def get_news_yfinance(