mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-19 19:25:24 +03:00
fix(dataflows): make the Yahoo news window UTC and end-exclusive
- the upper bound was inclusive, so an article stamped exactly midnight after end_date leaked into a historical run - flat epoch timestamps were parsed in host-local time and offset-aware stamps had tzinfo stripped without converting, making filtering machine-dependent - normalize every operand to UTC and use a half-open [start, end + 1 day) #1126
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
"""yfinance-based news data fetching functions."""
|
||||
|
||||
import contextlib
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import yfinance as yf
|
||||
from dateutil.relativedelta import relativedelta
|
||||
@@ -11,6 +11,16 @@ from .stockstats_utils import yf_retry
|
||||
from .symbol_utils import normalize_symbol
|
||||
|
||||
|
||||
def _as_utc(dt: datetime) -> datetime:
|
||||
"""Normalize a datetime to UTC-aware; a naive value is assumed to be UTC.
|
||||
|
||||
Window bounds arrive naive (parsed from ``yyyy-mm-dd``) while article
|
||||
timestamps may be offset-aware, so every operand is normalized before
|
||||
comparison. Without this the filter depends on the host timezone (#1126).
|
||||
"""
|
||||
return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _extract_article_data(article: dict) -> dict:
|
||||
"""Extract article data from yfinance news format (handles nested 'content' structure)."""
|
||||
# Handle nested content structure
|
||||
@@ -46,8 +56,10 @@ def _extract_article_data(article: dict) -> dict:
|
||||
pub_date = None
|
||||
ts = article.get("providerPublishTime")
|
||||
if ts:
|
||||
# Epoch seconds are UTC; parse them as UTC-aware so filtering does
|
||||
# not shift with the host timezone (#1126).
|
||||
with contextlib.suppress(ValueError, OSError, TypeError):
|
||||
pub_date = datetime.fromtimestamp(ts)
|
||||
pub_date = datetime.fromtimestamp(ts, tz=timezone.utc)
|
||||
return {
|
||||
"title": article.get("title", "No title"),
|
||||
"summary": article.get("summary", ""),
|
||||
@@ -58,17 +70,18 @@ def _extract_article_data(article: dict) -> dict:
|
||||
|
||||
|
||||
def _in_news_window(pub_date, start_dt, end_dt) -> bool:
|
||||
"""Whether an article belongs in the [start_dt, end_dt] window.
|
||||
"""Whether an article belongs in the half-open window ``[start, end + 1 day)``.
|
||||
|
||||
Dated articles are kept only if they fall in the window. An undated article
|
||||
is kept only when the window reaches the present (live run) — in a
|
||||
historical/backtest window it's excluded, since we can't prove it isn't
|
||||
future news (look-ahead safety, #992/#1007).
|
||||
Every operand is normalized to UTC, and the upper bound is exclusive so an
|
||||
article stamped exactly at midnight after ``end_dt`` cannot leak into a
|
||||
historical run (#1126). An undated article is kept only when the window
|
||||
reaches the present (live run) — in a historical/backtest window it's
|
||||
excluded, since we can't prove it isn't future news (#992/#1007).
|
||||
"""
|
||||
end = _as_utc(end_dt)
|
||||
if pub_date is not None:
|
||||
naive = pub_date.replace(tzinfo=None) if hasattr(pub_date, "replace") else pub_date
|
||||
return start_dt <= naive <= end_dt + relativedelta(days=1)
|
||||
return end_dt >= datetime.now() - relativedelta(days=1)
|
||||
return _as_utc(start_dt) <= _as_utc(pub_date) < end + timedelta(days=1)
|
||||
return end >= datetime.now(timezone.utc) - timedelta(days=1)
|
||||
|
||||
|
||||
def get_news_yfinance(
|
||||
|
||||
Reference in New Issue
Block a user