fix(dataflows): make the Yahoo news window UTC and end-exclusive

- the upper bound was inclusive, so an article stamped exactly midnight after
  end_date leaked into a historical run
- flat epoch timestamps were parsed in host-local time and offset-aware stamps had
  tzinfo stripped without converting, making filtering machine-dependent
- normalize every operand to UTC and use a half-open [start, end + 1 day) #1126
This commit is contained in:
Yijia-Xiao
2026-07-18 06:28:38 +00:00
parent 01477f9afb
commit 40774ca042
2 changed files with 56 additions and 18 deletions

View File

@@ -2,10 +2,11 @@
into a historical window.
Regressions for #992 (flat articles bypassed the date filter), #1007 (global
news injected future articles), #993 (empty-after-filter returned a blank body).
news injected future articles), #993 (empty-after-filter returned a blank body),
and #1126 (inclusive upper bound leaked the midnight-after article; host-local
timestamp parsing made filtering machine-dependent).
"""
import time
from datetime import datetime
from datetime import datetime, timezone
import pytest
@@ -13,17 +14,20 @@ import tradingagents.dataflows.yfinance_news as ynews
def _epoch(date_str):
return int(time.mktime(datetime.strptime(date_str, "%Y-%m-%d").timetuple()))
"""Epoch seconds for UTC midnight of ``date_str`` (host-timezone independent)."""
return int(datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc).timestamp())
@pytest.mark.unit
def test_flat_article_publish_time_is_parsed():
# #992: flat articles now carry a pub_date (was always None -> unfilterable).
# #1126: parsed as UTC-aware, so the date can't shift with the host timezone.
data = ynews._extract_article_data(
{"title": "X", "publisher": "P", "link": "l", "providerPublishTime": _epoch("2025-05-09")}
)
assert data["pub_date"] is not None
assert data["pub_date"].strftime("%Y-%m-%d") == "2025-05-09"
assert data["pub_date"].tzinfo is not None
assert data["pub_date"] == datetime(2025, 5, 9, tzinfo=timezone.utc)
@pytest.mark.unit
@@ -40,9 +44,30 @@ def test_window_excludes_future_and_undated_in_backtest():
@pytest.mark.unit
def test_window_keeps_undated_in_live_window():
# Live window (reaches today): undated articles can't be "future", so keep them.
start = datetime.now()
end = datetime.now()
assert ynews._in_news_window(None, start, end) is True
now = datetime.now(timezone.utc)
assert ynews._in_news_window(None, now, now) is True
@pytest.mark.unit
def test_upper_bound_is_exclusive():
# #1126: an article stamped exactly midnight AFTER end_date leaked in under
# the old inclusive bound; the whole of end_date itself must still be kept.
start = datetime(2025, 5, 1)
end = datetime(2025, 5, 9)
midnight_after = datetime(2025, 5, 10, 0, 0, 0, tzinfo=timezone.utc)
last_moment = datetime(2025, 5, 9, 23, 59, 59, tzinfo=timezone.utc)
assert ynews._in_news_window(midnight_after, start, end) is False
assert ynews._in_news_window(last_moment, start, end) is True
@pytest.mark.unit
def test_offset_aware_timestamp_is_converted_not_truncated():
# #1126: 2025-05-10T01:00+05:00 is really 2025-05-09T20:00Z -> inside the
# window. Stripping tzinfo (old behavior) misread it as 05-10 and dropped it.
start = datetime(2025, 5, 1)
end = datetime(2025, 5, 9)
aware = datetime.fromisoformat("2025-05-10T01:00:00+05:00")
assert ynews._in_news_window(aware, start, end) is True
@pytest.mark.unit

View File

@@ -1,7 +1,7 @@
"""yfinance-based news data fetching functions."""
import contextlib
from datetime import datetime
from datetime import datetime, timedelta, timezone
import yfinance as yf
from dateutil.relativedelta import relativedelta
@@ -11,6 +11,16 @@ from .stockstats_utils import yf_retry
from .symbol_utils import normalize_symbol
def _as_utc(dt: datetime) -> datetime:
"""Normalize a datetime to UTC-aware; a naive value is assumed to be UTC.
Window bounds arrive naive (parsed from ``yyyy-mm-dd``) while article
timestamps may be offset-aware, so every operand is normalized before
comparison. Without this the filter depends on the host timezone (#1126).
"""
return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt.astimezone(timezone.utc)
def _extract_article_data(article: dict) -> dict:
"""Extract article data from yfinance news format (handles nested 'content' structure)."""
# Handle nested content structure
@@ -46,8 +56,10 @@ def _extract_article_data(article: dict) -> dict:
pub_date = None
ts = article.get("providerPublishTime")
if ts:
# Epoch seconds are UTC; parse them as UTC-aware so filtering does
# not shift with the host timezone (#1126).
with contextlib.suppress(ValueError, OSError, TypeError):
pub_date = datetime.fromtimestamp(ts)
pub_date = datetime.fromtimestamp(ts, tz=timezone.utc)
return {
"title": article.get("title", "No title"),
"summary": article.get("summary", ""),
@@ -58,17 +70,18 @@ def _extract_article_data(article: dict) -> dict:
def _in_news_window(pub_date, start_dt, end_dt) -> bool:
"""Whether an article belongs in the [start_dt, end_dt] window.
"""Whether an article belongs in the half-open window ``[start, end + 1 day)``.
Dated articles are kept only if they fall in the window. An undated article
is kept only when the window reaches the present (live run) — in a
historical/backtest window it's excluded, since we can't prove it isn't
future news (look-ahead safety, #992/#1007).
Every operand is normalized to UTC, and the upper bound is exclusive so an
article stamped exactly at midnight after ``end_dt`` cannot leak into a
historical run (#1126). An undated article is kept only when the window
reaches the present (live run) — in a historical/backtest window it's
excluded, since we can't prove it isn't future news (#992/#1007).
"""
end = _as_utc(end_dt)
if pub_date is not None:
naive = pub_date.replace(tzinfo=None) if hasattr(pub_date, "replace") else pub_date
return start_dt <= naive <= end_dt + relativedelta(days=1)
return end_dt >= datetime.now() - relativedelta(days=1)
return _as_utc(start_dt) <= _as_utc(pub_date) < end + timedelta(days=1)
return end >= datetime.now(timezone.utc) - timedelta(days=1)
def get_news_yfinance(