fix(dataflows): make the Yahoo news window UTC and end-exclusive

- the upper bound was inclusive, so an article stamped exactly midnight after
  end_date leaked into a historical run
- flat epoch timestamps were parsed in host-local time and offset-aware stamps had
  tzinfo stripped without converting, making filtering machine-dependent
- normalize every operand to UTC and use a half-open [start, end + 1 day) #1126
This commit is contained in:
Yijia-Xiao
2026-07-18 06:28:38 +00:00
parent 01477f9afb
commit 40774ca042
2 changed files with 56 additions and 18 deletions

View File

@@ -2,10 +2,11 @@
into a historical window. into a historical window.
Regressions for #992 (flat articles bypassed the date filter), #1007 (global Regressions for #992 (flat articles bypassed the date filter), #1007 (global
news injected future articles), #993 (empty-after-filter returned a blank body). news injected future articles), #993 (empty-after-filter returned a blank body),
and #1126 (inclusive upper bound leaked the midnight-after article; host-local
timestamp parsing made filtering machine-dependent).
""" """
import time from datetime import datetime, timezone
from datetime import datetime
import pytest import pytest
@@ -13,17 +14,20 @@ import tradingagents.dataflows.yfinance_news as ynews
def _epoch(date_str): def _epoch(date_str):
return int(time.mktime(datetime.strptime(date_str, "%Y-%m-%d").timetuple())) """Epoch seconds for UTC midnight of ``date_str`` (host-timezone independent)."""
return int(datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc).timestamp())
@pytest.mark.unit @pytest.mark.unit
def test_flat_article_publish_time_is_parsed(): def test_flat_article_publish_time_is_parsed():
# #992: flat articles now carry a pub_date (was always None -> unfilterable). # #992: flat articles now carry a pub_date (was always None -> unfilterable).
# #1126: parsed as UTC-aware, so the date can't shift with the host timezone.
data = ynews._extract_article_data( data = ynews._extract_article_data(
{"title": "X", "publisher": "P", "link": "l", "providerPublishTime": _epoch("2025-05-09")} {"title": "X", "publisher": "P", "link": "l", "providerPublishTime": _epoch("2025-05-09")}
) )
assert data["pub_date"] is not None assert data["pub_date"] is not None
assert data["pub_date"].strftime("%Y-%m-%d") == "2025-05-09" assert data["pub_date"].tzinfo is not None
assert data["pub_date"] == datetime(2025, 5, 9, tzinfo=timezone.utc)
@pytest.mark.unit @pytest.mark.unit
@@ -40,9 +44,30 @@ def test_window_excludes_future_and_undated_in_backtest():
@pytest.mark.unit @pytest.mark.unit
def test_window_keeps_undated_in_live_window(): def test_window_keeps_undated_in_live_window():
# Live window (reaches today): undated articles can't be "future", so keep them. # Live window (reaches today): undated articles can't be "future", so keep them.
start = datetime.now() now = datetime.now(timezone.utc)
end = datetime.now() assert ynews._in_news_window(None, now, now) is True
assert ynews._in_news_window(None, start, end) is True
@pytest.mark.unit
def test_upper_bound_is_exclusive():
# #1126: an article stamped exactly midnight AFTER end_date leaked in under
# the old inclusive bound; the whole of end_date itself must still be kept.
start = datetime(2025, 5, 1)
end = datetime(2025, 5, 9)
midnight_after = datetime(2025, 5, 10, 0, 0, 0, tzinfo=timezone.utc)
last_moment = datetime(2025, 5, 9, 23, 59, 59, tzinfo=timezone.utc)
assert ynews._in_news_window(midnight_after, start, end) is False
assert ynews._in_news_window(last_moment, start, end) is True
@pytest.mark.unit
def test_offset_aware_timestamp_is_converted_not_truncated():
# #1126: 2025-05-10T01:00+05:00 is really 2025-05-09T20:00Z -> inside the
# window. Stripping tzinfo (old behavior) misread it as 05-10 and dropped it.
start = datetime(2025, 5, 1)
end = datetime(2025, 5, 9)
aware = datetime.fromisoformat("2025-05-10T01:00:00+05:00")
assert ynews._in_news_window(aware, start, end) is True
@pytest.mark.unit @pytest.mark.unit

View File

@@ -1,7 +1,7 @@
"""yfinance-based news data fetching functions.""" """yfinance-based news data fetching functions."""
import contextlib import contextlib
from datetime import datetime from datetime import datetime, timedelta, timezone
import yfinance as yf import yfinance as yf
from dateutil.relativedelta import relativedelta from dateutil.relativedelta import relativedelta
@@ -11,6 +11,16 @@ from .stockstats_utils import yf_retry
from .symbol_utils import normalize_symbol from .symbol_utils import normalize_symbol
def _as_utc(dt: datetime) -> datetime:
"""Normalize a datetime to UTC-aware; a naive value is assumed to be UTC.
Window bounds arrive naive (parsed from ``yyyy-mm-dd``) while article
timestamps may be offset-aware, so every operand is normalized before
comparison. Without this the filter depends on the host timezone (#1126).
"""
return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt.astimezone(timezone.utc)
def _extract_article_data(article: dict) -> dict: def _extract_article_data(article: dict) -> dict:
"""Extract article data from yfinance news format (handles nested 'content' structure).""" """Extract article data from yfinance news format (handles nested 'content' structure)."""
# Handle nested content structure # Handle nested content structure
@@ -46,8 +56,10 @@ def _extract_article_data(article: dict) -> dict:
pub_date = None pub_date = None
ts = article.get("providerPublishTime") ts = article.get("providerPublishTime")
if ts: if ts:
# Epoch seconds are UTC; parse them as UTC-aware so filtering does
# not shift with the host timezone (#1126).
with contextlib.suppress(ValueError, OSError, TypeError): with contextlib.suppress(ValueError, OSError, TypeError):
pub_date = datetime.fromtimestamp(ts) pub_date = datetime.fromtimestamp(ts, tz=timezone.utc)
return { return {
"title": article.get("title", "No title"), "title": article.get("title", "No title"),
"summary": article.get("summary", ""), "summary": article.get("summary", ""),
@@ -58,17 +70,18 @@ def _extract_article_data(article: dict) -> dict:
def _in_news_window(pub_date, start_dt, end_dt) -> bool: def _in_news_window(pub_date, start_dt, end_dt) -> bool:
"""Whether an article belongs in the [start_dt, end_dt] window. """Whether an article belongs in the half-open window ``[start, end + 1 day)``.
Dated articles are kept only if they fall in the window. An undated article Every operand is normalized to UTC, and the upper bound is exclusive so an
is kept only when the window reaches the present (live run) — in a article stamped exactly at midnight after ``end_dt`` cannot leak into a
historical/backtest window it's excluded, since we can't prove it isn't historical run (#1126). An undated article is kept only when the window
future news (look-ahead safety, #992/#1007). reaches the present (live run) — in a historical/backtest window it's
excluded, since we can't prove it isn't future news (#992/#1007).
""" """
end = _as_utc(end_dt)
if pub_date is not None: if pub_date is not None:
naive = pub_date.replace(tzinfo=None) if hasattr(pub_date, "replace") else pub_date return _as_utc(start_dt) <= _as_utc(pub_date) < end + timedelta(days=1)
return start_dt <= naive <= end_dt + relativedelta(days=1) return end >= datetime.now(timezone.utc) - timedelta(days=1)
return end_dt >= datetime.now() - relativedelta(days=1)
def get_news_yfinance( def get_news_yfinance(