mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-19 11:15:24 +03:00
fix(dataflows): report windows a feed cannot observe as unavailable
- Yahoo news and the Reddit and StockTwits feeds serve only recent items, so a historical window trimmed to nothing was reported as "no news" or "no posts", and the sentiment analyst scored that silence as a neutral signal - judge each empty window in one shared rule: it is a real absence only when the feed's coverage reaches the window's first day and the window ends by today; otherwise report it unavailable with where coverage starts - coverage comes from the returned timestamps, which are newest-first on these feeds, plus Reddit's one-week search lookback; merged global-news searches prove no continuity and are bounded by the present alone - state in the sentiment analyst that historical sentiment inputs are not guaranteed to be point-in-time
This commit is contained in:
Yijia-Xiao
@@ -101,5 +101,115 @@ def test_global_news_empty_after_filter_is_informative(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
|
||||
out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=10)
|
||||
assert "No global news found" in out
|
||||
assert "###" not in out # no empty article body
|
||||
# Only a later article came back, so the feed does not reach this window.
|
||||
assert "unavailable" in out and "not an absence" in out
|
||||
|
||||
|
||||
|
||||
def _ticker_with(articles, monkeypatch):
|
||||
class FakeTicker:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
def get_news(self, count=20):
|
||||
return articles
|
||||
|
||||
monkeypatch.setattr(ynews.yf, "Ticker", FakeTicker)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ticker_news_window_before_feed_coverage_is_unavailable(monkeypatch):
|
||||
# Yahoo serves only recent articles: a historical window gets none of them,
|
||||
# which must read as "cannot answer", not "no news happened".
|
||||
recent = [{"title": "RECENT", "publisher": "P", "link": "l",
|
||||
"providerPublishTime": _epoch("2026-09-10")}]
|
||||
_ticker_with(recent, monkeypatch)
|
||||
out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
|
||||
assert "RECENT" not in out
|
||||
assert "unavailable" in out and "not an absence" in out
|
||||
assert "2026-09-10" in out # says how far back the feed actually reaches
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ticker_news_covered_but_empty_window_is_a_real_absence(monkeypatch):
|
||||
articles = [{"title": "RECENT", "publisher": "P", "link": "l",
|
||||
"providerPublishTime": _epoch("2026-09-10")},
|
||||
{"title": "OLDER", "publisher": "P", "link": "l",
|
||||
"providerPublishTime": _epoch("2026-07-01")}]
|
||||
_ticker_with(articles, monkeypatch)
|
||||
out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
|
||||
assert "No news found" in out
|
||||
assert "unavailable" not in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@pytest.mark.parametrize("dates, expect_gap", [
|
||||
([], True), # empty feed: covers at most now
|
||||
([None], True), # undated only: same
|
||||
([datetime(2026, 5, 20, tzinfo=timezone.utc)], True), # all after the window
|
||||
([datetime(2026, 5, 4, tzinfo=timezone.utc)], True), # starts mid-window: partial
|
||||
([datetime(2026, 5, 1, 18, tzinfo=timezone.utc)], False), # reaches the first day
|
||||
([datetime(2026, 5, 20, tzinfo=timezone.utc),
|
||||
datetime(2026, 4, 1, tzinfo=timezone.utc)], False), # coverage reaches back
|
||||
])
|
||||
def test_coverage_gap_boundaries(dates, expect_gap):
|
||||
from tradingagents.dataflows.date_window import coverage_gap
|
||||
|
||||
out = coverage_gap(dates, "2026-05-01", "2026-05-08", "Feed", "items")
|
||||
assert (out is not None) is expect_gap
|
||||
if expect_gap:
|
||||
assert "unavailable for 2026-05-01..2026-05-08" in out and "not an absence" in out
|
||||
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ticker_news_empty_feed_for_a_past_window_is_unavailable(monkeypatch):
|
||||
_ticker_with([], monkeypatch)
|
||||
out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
|
||||
assert "unavailable" in out and "not an absence" in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_ticker_news_null_feed_is_handled(monkeypatch):
|
||||
# Yahoo can return None instead of a list; that is unavailability, not an error.
|
||||
_ticker_with(None, monkeypatch)
|
||||
out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
|
||||
assert "unavailable" in out and "Error" not in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_global_news_empty_feed_for_a_past_window_is_unavailable(monkeypatch):
|
||||
class FakeSearch:
|
||||
def __init__(self, *a, **k):
|
||||
self.news = []
|
||||
|
||||
monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
|
||||
out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=10)
|
||||
assert "unavailable" in out and "not an absence" in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_global_news_does_not_infer_coverage_from_a_stale_search_hit(monkeypatch):
|
||||
# Global news merges fuzzy searches; one old hit before the window says
|
||||
# nothing about the days in between, so the window stays unavailable.
|
||||
stale = {"title": "STALE", "publisher": "P", "link": "l", "providerPublishTime": _epoch("2025-01-01")}
|
||||
fresh = {"title": "FRESH", "publisher": "P", "link": "l", "providerPublishTime": _epoch("2025-06-01")}
|
||||
|
||||
class FakeSearch:
|
||||
def __init__(self, *a, **k):
|
||||
self.news = [fresh, stale]
|
||||
|
||||
monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
|
||||
out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=10)
|
||||
assert "unavailable" in out and "No global news found" not in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_coverage_gap_future_window_is_unavailable():
|
||||
from datetime import timedelta
|
||||
|
||||
from tradingagents.dataflows.date_window import coverage_gap
|
||||
today = datetime.now(timezone.utc).date()
|
||||
out = coverage_gap([], str(today), str(today + timedelta(days=3)), "Feed", "items")
|
||||
assert out is not None and "past today" in out
|
||||
|
||||
@@ -71,9 +71,10 @@ def test_stocktwits_historical_window_excludes_recent(monkeypatch):
|
||||
recent = [_msg("2026-08-30T12:00:00Z", "Bullish"), _msg("2026-08-29T09:00:00Z")]
|
||||
monkeypatch.setattr(stocktwits, "urlopen", lambda *a, **k: _JsonResp({"messages": recent}))
|
||||
out = stocktwits.fetch_stocktwits_messages("AAPL", start_date="2026-05-01", end_date="2026-05-08")
|
||||
assert "no StockTwits messages" in out
|
||||
assert "2026-05-01..2026-05-08" in out
|
||||
assert "Bullish: 1" not in out # the recent bullish message did not leak
|
||||
# Coverage starts after the window: unavailable, never a claim of silence.
|
||||
assert "unavailable" in out and "not an absence" in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -107,7 +108,7 @@ def test_reddit_historical_window_excludes_recent(monkeypatch):
|
||||
start_date="2026-05-01", end_date="2026-05-08",
|
||||
)
|
||||
assert "NOW" not in out
|
||||
assert "no posts" in out.lower() or "no reddit posts" in out.lower()
|
||||
assert "unavailable" in out and "not an absence" in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
@@ -119,3 +120,90 @@ def test_reddit_live_window_keeps_in_range(monkeypatch):
|
||||
start_date="2026-05-01", end_date="2026-05-08",
|
||||
)
|
||||
assert "INRANGE" in out
|
||||
|
||||
|
||||
# --- coverage vs absence --------------------------------------------------------
|
||||
# The public feeds only serve recent items. When everything fetched postdates the
|
||||
# window the source cannot answer for that date; reporting "no posts" there is a
|
||||
# claim about the market that was never observed.
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_stocktwits_covered_but_empty_window_is_a_real_absence(monkeypatch):
|
||||
# The stream reaches back before the window (an older message exists) yet
|
||||
# nothing falls inside it: that is genuine silence.
|
||||
msgs = [_msg("2026-08-30T12:00:00Z"), _msg("2026-04-20T12:00:00Z")]
|
||||
monkeypatch.setattr(stocktwits, "urlopen", lambda *a, **k: _JsonResp({"messages": msgs}))
|
||||
out = stocktwits.fetch_stocktwits_messages("AAPL", start_date="2026-05-01", end_date="2026-05-08")
|
||||
assert "no StockTwits messages" in out
|
||||
assert "unavailable" not in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_reddit_covered_but_empty_window_is_a_real_absence(monkeypatch):
|
||||
posts = [{"title": "NOW", "created_utc": _epoch("2026-08-30"), "source": "rss"},
|
||||
{"title": "OLD", "created_utc": _epoch("2026-04-20"), "source": "rss"}]
|
||||
monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: posts)
|
||||
out = reddit.fetch_reddit_posts(
|
||||
"AAPL", subreddits=("stocks",), inter_request_delay=0,
|
||||
start_date="2026-05-01", end_date="2026-05-08",
|
||||
)
|
||||
assert "no reddit posts" in out.lower()
|
||||
assert "unavailable" not in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_reddit_empty_feed_for_an_old_window_is_unavailable(monkeypatch):
|
||||
# Search is limited to the last week, so an empty response says nothing
|
||||
# about a window from months ago: there are no timestamps to go on, and the
|
||||
# lookback bound alone must decide.
|
||||
monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: [])
|
||||
out = reddit.fetch_reddit_posts(
|
||||
"AAPL", subreddits=("stocks",), inter_request_delay=0,
|
||||
start_date="2024-05-01", end_date="2024-05-08",
|
||||
)
|
||||
assert "unavailable" in out and "not an absence" in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_reddit_live_empty_feed_is_a_real_absence(monkeypatch):
|
||||
monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: [])
|
||||
out = reddit.fetch_reddit_posts("AAPL", subreddits=("stocks",), inter_request_delay=0)
|
||||
assert "no reddit posts" in out.lower() and "past 7 days" in out
|
||||
assert "unavailable" not in out
|
||||
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_stocktwits_empty_stream_for_a_past_window_is_unavailable(monkeypatch):
|
||||
monkeypatch.setattr(stocktwits, "urlopen", lambda *a, **k: _JsonResp({"messages": []}))
|
||||
out = stocktwits.fetch_stocktwits_messages("AAPL", start_date="2026-05-01", end_date="2026-05-08")
|
||||
assert "unavailable" in out and "not an absence" in out
|
||||
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_reddit_window_straddling_the_lookback_is_unavailable(monkeypatch):
|
||||
# Ten days ago through five days ago: the week-long search never reaches the
|
||||
# first three days, so an empty result cannot stand for the whole window.
|
||||
from datetime import timedelta
|
||||
today = datetime.now(timezone.utc).date()
|
||||
monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: [])
|
||||
out = reddit.fetch_reddit_posts(
|
||||
"AAPL", subreddits=("stocks",), inter_request_delay=0,
|
||||
start_date=str(today - timedelta(days=10)), end_date=str(today - timedelta(days=5)),
|
||||
)
|
||||
assert "unavailable" in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_reddit_standard_week_window_empty_is_a_real_absence(monkeypatch):
|
||||
# The graph's window is [trade_date - 7, trade_date]; the week-long search
|
||||
# covers it, so an empty result is genuine silence.
|
||||
from datetime import timedelta
|
||||
today = datetime.now(timezone.utc).date()
|
||||
monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: [])
|
||||
out = reddit.fetch_reddit_posts(
|
||||
"AAPL", subreddits=("stocks",), inter_request_delay=0,
|
||||
start_date=str(today - timedelta(days=7)), end_date=str(today),
|
||||
)
|
||||
assert "no reddit posts" in out.lower() and "unavailable" not in out
|
||||
|
||||
Reference in New Issue
Block a user