fix(dataflows): report windows a feed cannot observe as unavailable

- Yahoo news and the Reddit and StockTwits feeds serve only recent items, so a
  historical window trimmed to nothing was reported as "no news" or "no
  posts", and the sentiment analyst scored that silence as a neutral signal
- judge each empty window in one shared rule: it is a real absence only when
  the feed's coverage reaches the window's first day and the window ends by
  today; otherwise report it unavailable with where coverage starts
- coverage comes from the returned timestamps, which are newest-first on these
  feeds, plus Reddit's one-week search lookback; merged global-news searches
  prove no continuity and are bounded by the present alone
- state in the sentiment analyst that historical sentiment inputs are not
  guaranteed to be point-in-time
This commit is contained in:
Yijia-Xiao
2026-09-14 22:07:41 +00:00
parent 673abb3c68
commit 2e38b47dca
7 changed files with 300 additions and 43 deletions

View File

@@ -101,5 +101,115 @@ def test_global_news_empty_after_filter_is_informative(monkeypatch):
monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=10)
assert "No global news found" in out
assert "###" not in out # no empty article body
# Only a later article came back, so the feed does not reach this window.
assert "unavailable" in out and "not an absence" in out
def _ticker_with(articles, monkeypatch):
class FakeTicker:
def __init__(self, *a, **k):
pass
def get_news(self, count=20):
return articles
monkeypatch.setattr(ynews.yf, "Ticker", FakeTicker)
@pytest.mark.unit
def test_ticker_news_window_before_feed_coverage_is_unavailable(monkeypatch):
# Yahoo serves only recent articles: a historical window gets none of them,
# which must read as "cannot answer", not "no news happened".
recent = [{"title": "RECENT", "publisher": "P", "link": "l",
"providerPublishTime": _epoch("2026-09-10")}]
_ticker_with(recent, monkeypatch)
out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
assert "RECENT" not in out
assert "unavailable" in out and "not an absence" in out
assert "2026-09-10" in out # says how far back the feed actually reaches
@pytest.mark.unit
def test_ticker_news_covered_but_empty_window_is_a_real_absence(monkeypatch):
articles = [{"title": "RECENT", "publisher": "P", "link": "l",
"providerPublishTime": _epoch("2026-09-10")},
{"title": "OLDER", "publisher": "P", "link": "l",
"providerPublishTime": _epoch("2026-07-01")}]
_ticker_with(articles, monkeypatch)
out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
assert "No news found" in out
assert "unavailable" not in out
@pytest.mark.unit
@pytest.mark.parametrize("dates, expect_gap", [
([], True), # empty feed: covers at most now
([None], True), # undated only: same
([datetime(2026, 5, 20, tzinfo=timezone.utc)], True), # all after the window
([datetime(2026, 5, 4, tzinfo=timezone.utc)], True), # starts mid-window: partial
([datetime(2026, 5, 1, 18, tzinfo=timezone.utc)], False), # reaches the first day
([datetime(2026, 5, 20, tzinfo=timezone.utc),
datetime(2026, 4, 1, tzinfo=timezone.utc)], False), # coverage reaches back
])
def test_coverage_gap_boundaries(dates, expect_gap):
from tradingagents.dataflows.date_window import coverage_gap
out = coverage_gap(dates, "2026-05-01", "2026-05-08", "Feed", "items")
assert (out is not None) is expect_gap
if expect_gap:
assert "unavailable for 2026-05-01..2026-05-08" in out and "not an absence" in out
@pytest.mark.unit
def test_ticker_news_empty_feed_for_a_past_window_is_unavailable(monkeypatch):
_ticker_with([], monkeypatch)
out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
assert "unavailable" in out and "not an absence" in out
@pytest.mark.unit
def test_ticker_news_null_feed_is_handled(monkeypatch):
# Yahoo can return None instead of a list; that is unavailability, not an error.
_ticker_with(None, monkeypatch)
out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
assert "unavailable" in out and "Error" not in out
@pytest.mark.unit
def test_global_news_empty_feed_for_a_past_window_is_unavailable(monkeypatch):
class FakeSearch:
def __init__(self, *a, **k):
self.news = []
monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=10)
assert "unavailable" in out and "not an absence" in out
@pytest.mark.unit
def test_global_news_does_not_infer_coverage_from_a_stale_search_hit(monkeypatch):
# Global news merges fuzzy searches; one old hit before the window says
# nothing about the days in between, so the window stays unavailable.
stale = {"title": "STALE", "publisher": "P", "link": "l", "providerPublishTime": _epoch("2025-01-01")}
fresh = {"title": "FRESH", "publisher": "P", "link": "l", "providerPublishTime": _epoch("2025-06-01")}
class FakeSearch:
def __init__(self, *a, **k):
self.news = [fresh, stale]
monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=10)
assert "unavailable" in out and "No global news found" not in out
@pytest.mark.unit
def test_coverage_gap_future_window_is_unavailable():
from datetime import timedelta
from tradingagents.dataflows.date_window import coverage_gap
today = datetime.now(timezone.utc).date()
out = coverage_gap([], str(today), str(today + timedelta(days=3)), "Feed", "items")
assert out is not None and "past today" in out