fix(dataflows): report windows a feed cannot observe as unavailable

- Yahoo news and the Reddit and StockTwits feeds serve only recent items, so a
  historical window trimmed to nothing was reported as "no news" or "no
  posts", and the sentiment analyst scored that silence as a neutral signal
- judge each empty window in one shared rule: it is a real absence only when
  the feed's coverage reaches the window's first day and the window ends by
  today; otherwise report it unavailable with where coverage starts
- coverage comes from the returned timestamps, which are newest-first on these
  feeds, plus Reddit's one-week search lookback; merged global-news searches
  prove no continuity and are bounded by the present alone
- state in the sentiment analyst that historical sentiment inputs are not
  guaranteed to be point-in-time

tests/test_news_lookahead.py

@@ -101,5 +101,115 @@ def test_global_news_empty_after_filter_is_informative(monkeypatch):
 
     monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
     out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=10)
-    assert "No global news found" in out
     assert "###" not in out  # no empty article body
+    # Only a later article came back, so the feed does not reach this window.
+    assert "unavailable" in out and "not an absence" in out
+
+
+
+def _ticker_with(articles, monkeypatch):
+    class FakeTicker:
+        def __init__(self, *a, **k):
+            pass
+
+        def get_news(self, count=20):
+            return articles
+
+    monkeypatch.setattr(ynews.yf, "Ticker", FakeTicker)
+
+
+@pytest.mark.unit
+def test_ticker_news_window_before_feed_coverage_is_unavailable(monkeypatch):
+    # Yahoo serves only recent articles: a historical window gets none of them,
+    # which must read as "cannot answer", not "no news happened".
+    recent = [{"title": "RECENT", "publisher": "P", "link": "l",
+               "providerPublishTime": _epoch("2026-09-10")}]
+    _ticker_with(recent, monkeypatch)
+    out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
+    assert "RECENT" not in out
+    assert "unavailable" in out and "not an absence" in out
+    assert "2026-09-10" in out  # says how far back the feed actually reaches
+
+
+@pytest.mark.unit
+def test_ticker_news_covered_but_empty_window_is_a_real_absence(monkeypatch):
+    articles = [{"title": "RECENT", "publisher": "P", "link": "l",
+                 "providerPublishTime": _epoch("2026-09-10")},
+                {"title": "OLDER", "publisher": "P", "link": "l",
+                 "providerPublishTime": _epoch("2026-07-01")}]
+    _ticker_with(articles, monkeypatch)
+    out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
+    assert "No news found" in out
+    assert "unavailable" not in out
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize("dates, expect_gap", [
+    ([], True),                                              # empty feed: covers at most now
+    ([None], True),                                          # undated only: same
+    ([datetime(2026, 5, 20, tzinfo=timezone.utc)], True),    # all after the window
+    ([datetime(2026, 5, 4, tzinfo=timezone.utc)], True),     # starts mid-window: partial
+    ([datetime(2026, 5, 1, 18, tzinfo=timezone.utc)], False),  # reaches the first day
+    ([datetime(2026, 5, 20, tzinfo=timezone.utc),
+      datetime(2026, 4, 1, tzinfo=timezone.utc)], False),    # coverage reaches back
+])
+def test_coverage_gap_boundaries(dates, expect_gap):
+    from tradingagents.dataflows.date_window import coverage_gap
+
+    out = coverage_gap(dates, "2026-05-01", "2026-05-08", "Feed", "items")
+    assert (out is not None) is expect_gap
+    if expect_gap:
+        assert "unavailable for 2026-05-01..2026-05-08" in out and "not an absence" in out
+
+
+
+@pytest.mark.unit
+def test_ticker_news_empty_feed_for_a_past_window_is_unavailable(monkeypatch):
+    _ticker_with([], monkeypatch)
+    out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
+    assert "unavailable" in out and "not an absence" in out
+
+
+@pytest.mark.unit
+def test_ticker_news_null_feed_is_handled(monkeypatch):
+    # Yahoo can return None instead of a list; that is unavailability, not an error.
+    _ticker_with(None, monkeypatch)
+    out = ynews.get_news_yfinance("AAPL", "2026-08-07", "2026-08-14")
+    assert "unavailable" in out and "Error" not in out
+
+
+@pytest.mark.unit
+def test_global_news_empty_feed_for_a_past_window_is_unavailable(monkeypatch):
+    class FakeSearch:
+        def __init__(self, *a, **k):
+            self.news = []
+
+    monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
+    out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=10)
+    assert "unavailable" in out and "not an absence" in out
+
+
+@pytest.mark.unit
+def test_global_news_does_not_infer_coverage_from_a_stale_search_hit(monkeypatch):
+    # Global news merges fuzzy searches; one old hit before the window says
+    # nothing about the days in between, so the window stays unavailable.
+    stale = {"title": "STALE", "publisher": "P", "link": "l", "providerPublishTime": _epoch("2025-01-01")}
+    fresh = {"title": "FRESH", "publisher": "P", "link": "l", "providerPublishTime": _epoch("2025-06-01")}
+
+    class FakeSearch:
+        def __init__(self, *a, **k):
+            self.news = [fresh, stale]
+
+    monkeypatch.setattr(ynews.yf, "Search", FakeSearch)
+    out = ynews.get_global_news_yfinance("2025-05-09", look_back_days=7, limit=10)
+    assert "unavailable" in out and "No global news found" not in out
+
+
+@pytest.mark.unit
+def test_coverage_gap_future_window_is_unavailable():
+    from datetime import timedelta
+
+    from tradingagents.dataflows.date_window import coverage_gap
+    today = datetime.now(timezone.utc).date()
+    out = coverage_gap([], str(today), str(today + timedelta(days=3)), "Feed", "items")
+    assert out is not None and "past today" in out

tests/test_social_lookahead.py

@@ -71,9 +71,10 @@ def test_stocktwits_historical_window_excludes_recent(monkeypatch):
     recent = [_msg("2026-08-30T12:00:00Z", "Bullish"), _msg("2026-08-29T09:00:00Z")]
     monkeypatch.setattr(stocktwits, "urlopen", lambda *a, **k: _JsonResp({"messages": recent}))
     out = stocktwits.fetch_stocktwits_messages("AAPL", start_date="2026-05-01", end_date="2026-05-08")
-    assert "no StockTwits messages" in out
     assert "2026-05-01..2026-05-08" in out
     assert "Bullish: 1" not in out  # the recent bullish message did not leak
+    # Coverage starts after the window: unavailable, never a claim of silence.
+    assert "unavailable" in out and "not an absence" in out
 
 
 @pytest.mark.unit
@@ -107,7 +108,7 @@ def test_reddit_historical_window_excludes_recent(monkeypatch):
         start_date="2026-05-01", end_date="2026-05-08",
     )
     assert "NOW" not in out
-    assert "no posts" in out.lower() or "no reddit posts" in out.lower()
+    assert "unavailable" in out and "not an absence" in out
 
 
 @pytest.mark.unit
@@ -119,3 +120,90 @@ def test_reddit_live_window_keeps_in_range(monkeypatch):
         start_date="2026-05-01", end_date="2026-05-08",
     )
     assert "INRANGE" in out
+
+
+# --- coverage vs absence --------------------------------------------------------
+# The public feeds only serve recent items. When everything fetched postdates the
+# window the source cannot answer for that date; reporting "no posts" there is a
+# claim about the market that was never observed.
+
+@pytest.mark.unit
+def test_stocktwits_covered_but_empty_window_is_a_real_absence(monkeypatch):
+    # The stream reaches back before the window (an older message exists) yet
+    # nothing falls inside it: that is genuine silence.
+    msgs = [_msg("2026-08-30T12:00:00Z"), _msg("2026-04-20T12:00:00Z")]
+    monkeypatch.setattr(stocktwits, "urlopen", lambda *a, **k: _JsonResp({"messages": msgs}))
+    out = stocktwits.fetch_stocktwits_messages("AAPL", start_date="2026-05-01", end_date="2026-05-08")
+    assert "no StockTwits messages" in out
+    assert "unavailable" not in out
+
+
+@pytest.mark.unit
+def test_reddit_covered_but_empty_window_is_a_real_absence(monkeypatch):
+    posts = [{"title": "NOW", "created_utc": _epoch("2026-08-30"), "source": "rss"},
+             {"title": "OLD", "created_utc": _epoch("2026-04-20"), "source": "rss"}]
+    monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: posts)
+    out = reddit.fetch_reddit_posts(
+        "AAPL", subreddits=("stocks",), inter_request_delay=0,
+        start_date="2026-05-01", end_date="2026-05-08",
+    )
+    assert "no reddit posts" in out.lower()
+    assert "unavailable" not in out
+
+
+@pytest.mark.unit
+def test_reddit_empty_feed_for_an_old_window_is_unavailable(monkeypatch):
+    # Search is limited to the last week, so an empty response says nothing
+    # about a window from months ago: there are no timestamps to go on, and the
+    # lookback bound alone must decide.
+    monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: [])
+    out = reddit.fetch_reddit_posts(
+        "AAPL", subreddits=("stocks",), inter_request_delay=0,
+        start_date="2024-05-01", end_date="2024-05-08",
+    )
+    assert "unavailable" in out and "not an absence" in out
+
+
+@pytest.mark.unit
+def test_reddit_live_empty_feed_is_a_real_absence(monkeypatch):
+    monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: [])
+    out = reddit.fetch_reddit_posts("AAPL", subreddits=("stocks",), inter_request_delay=0)
+    assert "no reddit posts" in out.lower() and "past 7 days" in out
+    assert "unavailable" not in out
+
+
+
+@pytest.mark.unit
+def test_stocktwits_empty_stream_for_a_past_window_is_unavailable(monkeypatch):
+    monkeypatch.setattr(stocktwits, "urlopen", lambda *a, **k: _JsonResp({"messages": []}))
+    out = stocktwits.fetch_stocktwits_messages("AAPL", start_date="2026-05-01", end_date="2026-05-08")
+    assert "unavailable" in out and "not an absence" in out
+
+
+
+@pytest.mark.unit
+def test_reddit_window_straddling_the_lookback_is_unavailable(monkeypatch):
+    # Ten days ago through five days ago: the week-long search never reaches the
+    # first three days, so an empty result cannot stand for the whole window.
+    from datetime import timedelta
+    today = datetime.now(timezone.utc).date()
+    monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: [])
+    out = reddit.fetch_reddit_posts(
+        "AAPL", subreddits=("stocks",), inter_request_delay=0,
+        start_date=str(today - timedelta(days=10)), end_date=str(today - timedelta(days=5)),
+    )
+    assert "unavailable" in out
+
+
+@pytest.mark.unit
+def test_reddit_standard_week_window_empty_is_a_real_absence(monkeypatch):
+    # The graph's window is [trade_date - 7, trade_date]; the week-long search
+    # covers it, so an empty result is genuine silence.
+    from datetime import timedelta
+    today = datetime.now(timezone.utc).date()
+    monkeypatch.setattr(reddit, "_fetch_subreddit", lambda *a, **k: [])
+    out = reddit.fetch_reddit_posts(
+        "AAPL", subreddits=("stocks",), inter_request_delay=0,
+        start_date=str(today - timedelta(days=7)), end_date=str(today),
+    )
+    assert "no reddit posts" in out.lower() and "unavailable" not in out

tradingagents/agents/analysts/sentiment_analyst.py

@@ -13,6 +13,10 @@ the LLM is invoked and injects them into the prompt as structured blocks:
                            user-labeled Bullish/Bearish sentiment tags
   3. Reddit posts        — r/wallstreetbets, r/stocks, r/investing
 
+Each source is trimmed to the analysis window. These text feeds serve recent
+items and are not archived as of a past date, so sentiment inputs for a
+historical run are not guaranteed to be point-in-time.
+
 The agent does not use tool-calling; the data is in the prompt from
 turn 0. Output uses the structured-output pattern (json_schema for
 OpenAI/xAI, response_schema for Gemini, tool-use for Anthropic), falling

tradingagents/dataflows/date_window.py

@@ -32,6 +32,33 @@ def in_window(pub_dt: datetime | None, start_dt: datetime, end_dt: datetime) ->
     return end >= datetime.now(timezone.utc) - timedelta(days=1)
 
 
+def coverage_gap(
+    dates, start_date: str, end_date: str, source: str, subject: str
+) -> str | None:
+    """Placeholder for a window a feed did not fully observe, else None.
+
+    Yahoo news and the Reddit and StockTwits feeds return their latest items
+    whatever window is asked for, so "none found" over a window they never
+    observed would claim an absence nobody saw. A window is observed when
+    coverage reaches its first day and it ends by today; an empty result is then
+    a real absence and this returns None.
+
+    ``dates`` are the returned items' timestamps, plus the lookback start for a
+    feed with a fixed lookback. The oldest one bounds coverage only for a feed
+    returned newest-first and unbroken in time; a merged or relevance-ranked
+    result passes no dates, leaving only the present as the bound.
+    """
+    now = datetime.now(timezone.utc)
+    oldest = min((to_utc(d) for d in dates if d is not None), default=now)
+    if datetime.strptime(end_date, "%Y-%m-%d").date() > now.date():
+        reason = "the window extends past today"
+    elif oldest.date() > datetime.strptime(start_date, "%Y-%m-%d").date():
+        reason = f"it only serves recent items (coverage starts {oldest:%Y-%m-%d})"
+    else:
+        return None
+    return f"<{source} unavailable for {start_date}..{end_date}: {reason}, so this is not an absence of {subject}>"
+
+
 def withhold_live_profile(curr_date: str | None, label: str) -> str | None:
     """Notice to serve instead of a live-only company profile, or None to serve it.
 

tradingagents/dataflows/reddit.py

@@ -30,12 +30,12 @@ import re
 import time
 import xml.etree.ElementTree as ET
 from collections.abc import Iterable
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from urllib.error import HTTPError
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
 
-from .date_window import in_window
+from .date_window import coverage_gap, in_window
 from .symbol_utils import crypto_base
 
 logger = logging.getLogger(__name__)
@@ -51,13 +51,20 @@ def _within_window(posts, start_date, end_date):
         return posts
     start_dt = datetime.strptime(start_date, "%Y-%m-%d")
     end_dt = datetime.strptime(end_date, "%Y-%m-%d")
-    kept = []
-    for p in posts:
-        ts = p.get("created_utc")
-        created = datetime.fromtimestamp(ts, tz=timezone.utc) if ts else None
-        if in_window(created, start_dt, end_dt):
-            kept.append(p)
-    return kept
+    return [p for p in posts if in_window(_posted_at(p), start_dt, end_dt)]
+
+
+def _posted_at(post) -> datetime | None:
+    """A post's ``created_utc`` epoch as a UTC datetime, or None when missing."""
+    ts = post.get("created_utc")
+    return datetime.fromtimestamp(ts, tz=timezone.utc) if ts else None
+
+
+def _coverage_dates(posts) -> list:
+    """Post dates plus the search lookback start: the query is limited to the
+    last week (``t=week``), so a window older than that is out of reach even
+    when the feed returns nothing."""
+    return [_posted_at(p) for p in posts] + [datetime.now(timezone.utc) - _SEARCH_LOOKBACK]
 
 _API = "https://www.reddit.com/r/{sub}/search.json?{qs}"
 _RSS = "https://www.reddit.com/r/{sub}/search.rss?{qs}"
@@ -74,6 +81,9 @@ _ATOM_NS = {"atom": "http://www.w3.org/2005/Atom"}
 DEFAULT_SUBREDDITS = ("wallstreetbets", "stocks", "investing")
 
 
+_SEARCH_LOOKBACK = timedelta(days=7)  # matches t=week below
+
+
 def _search_qs(ticker: str, limit: int) -> str:
     return urlencode({
         "q": ticker,
@@ -288,6 +298,7 @@ def fetch_reddit_posts(
     blocks = []
     total_posts = 0
     unavailable = []
+    fetched_posts = []
     allow_retry = True
     for i, sub in enumerate(subreddits):
         if i > 0 and inter_request_delay:
@@ -305,8 +316,14 @@ def fetch_reddit_posts(
             continue
         posts = _within_window(fetched, start_date, end_date)
         total_posts += len(posts)
+        fetched_posts.extend(fetched)
         if not posts:
-            blocks.append(f"r/{sub}: <no posts found mentioning {ticker.upper()} in the past 7 days>")
+            gap = start_date and end_date and coverage_gap(
+                _coverage_dates(fetched), start_date, end_date,
+                f"r/{sub}", f"discussion of {ticker.upper()}",
+            )
+            period = f"within {start_date}..{end_date}" if start_date and end_date else "in the past 7 days"
+            blocks.append(f"r/{sub}: {gap or f'<no posts found mentioning {ticker.upper()} {period}>'}")
             continue
 
         via_rss = any(p.get("source") == "rss" for p in posts)
@@ -345,9 +362,14 @@ def fetch_reddit_posts(
                 f"({', '.join(f'r/{s}' for s in unavailable)}); this is not an "
                 f"absence of discussion>"
             )
-        summary = (
+        gap = start_date and end_date and coverage_gap(
+            _coverage_dates(fetched_posts), start_date, end_date,
+            "Reddit search", f"discussion of {ticker.upper()}",
+        )
+        period = f"within {start_date}..{end_date}" if start_date and end_date else "in the past 7 days"
+        summary = gap or (
             f"<no Reddit posts found mentioning {ticker.upper()} across "
-            f"{', '.join(f'r/{s}' for s in searched)} in the past 7 days>"
+            f"{', '.join(f'r/{s}' for s in searched)} {period}>"
         )
         if unavailable:
             summary += (

tradingagents/dataflows/stocktwits.py

@@ -21,7 +21,7 @@ import logging
 from datetime import datetime
 from urllib.request import Request, urlopen
 
-from .date_window import in_window
+from .date_window import coverage_gap, in_window
 from .symbol_utils import crypto_base
 
 logger = logging.getLogger(__name__)
@@ -30,6 +30,16 @@ _API = "https://api.stocktwits.com/api/2/streams/symbol/{ticker}.json"
 _UA = "tradingagents/0.2 (+https://github.com/TauricResearch/TradingAgents)"
 
 
+def _created_at(message) -> datetime | None:
+    """Parse a message's ISO 8601 ``created_at``; None when missing or malformed."""
+    raw = message.get("created_at")
+    if not raw:
+        return None
+    with contextlib.suppress(ValueError, TypeError):
+        return datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
+    return None
+
+
 def _within_window(messages, start_date, end_date):
     """Keep only messages published in [start_date, end_date] (look-ahead safe).
 
@@ -41,16 +51,7 @@ def _within_window(messages, start_date, end_date):
         return messages
     start_dt = datetime.strptime(start_date, "%Y-%m-%d")
     end_dt = datetime.strptime(end_date, "%Y-%m-%d")
-    kept = []
-    for m in messages:
-        created = None
-        raw = m.get("created_at")
-        if raw:
-            with contextlib.suppress(ValueError, TypeError):
-                created = datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
-        if in_window(created, start_dt, end_dt):
-            kept.append(m)
-    return kept
+    return [m for m in messages if in_window(_created_at(m), start_dt, end_dt)]
 
 
 def _stocktwits_symbol(ticker: str) -> str:
@@ -75,9 +76,9 @@ def fetch_stocktwits_messages(
     formatted plaintext block ready for prompt injection.
 
     When ``start_date``/``end_date`` (yyyy-mm-dd) are given, messages are trimmed
-    to that window. The StockTwits public stream only serves recent messages, so
-    for a historical run they all fall after the window and a clear placeholder
-    is returned rather than leaking today's chatter into a backtest (#1220).
+    to that window, so a historical run never sees today's chatter (#1220). The
+    public stream only serves recent messages, so a window it cannot reach is
+    reported as unavailable rather than as silence.
 
     Returns a placeholder string when the endpoint is unreachable, the
     symbol has no messages, or the response shape is unexpected — the
@@ -94,13 +95,17 @@ def fetch_stocktwits_messages(
         logger.warning("StockTwits fetch failed for %s: %s", ticker, exc)
         return f"<stocktwits unavailable: {type(exc).__name__}>"
 
-    messages = data.get("messages", []) if isinstance(data, dict) else []
-    messages = _within_window(messages, start_date, end_date)
+    fetched = data.get("messages", []) if isinstance(data, dict) else []
+    messages = _within_window(fetched, start_date, end_date)
     if not messages:
         if start_date and end_date:
-            return (
+            gap = coverage_gap(
+                (_created_at(m) for m in fetched), start_date, end_date,
+                "StockTwits", f"messages about ${ticker.upper()}",
+            )
+            return gap or (
                 f"<no StockTwits messages for ${ticker.upper()} within "
-                f"{start_date}..{end_date} (public stream serves only recent messages)>"
+                f"{start_date}..{end_date}>"
             )
         return f"<no StockTwits messages found for ${ticker.upper()}>"
 

tradingagents/dataflows/yfinance_news.py

@@ -7,7 +7,7 @@ import yfinance as yf
 from dateutil.relativedelta import relativedelta
 
 from .config import get_config
-from .date_window import in_window
+from .date_window import coverage_gap, in_window
 from .stockstats_utils import yf_retry
 from .symbol_utils import normalize_symbol
 
@@ -84,10 +84,7 @@ def get_news_yfinance(
     resolved = "" if canonical == ticker else f" (resolved to {canonical})"
     try:
         stock = yf.Ticker(canonical)
-        news = yf_retry(lambda: stock.get_news(count=article_limit))
-
-        if not news:
-            return f"No news found for {ticker}{resolved}"
+        news = yf_retry(lambda: stock.get_news(count=article_limit)) or []
 
         # Parse date range for filtering
         start_dt = datetime.strptime(start_date, "%Y-%m-%d")
@@ -112,7 +109,11 @@ def get_news_yfinance(
             filtered_count += 1
 
         if filtered_count == 0:
-            return f"No news found for {ticker}{resolved} between {start_date} and {end_date}"
+            gap = coverage_gap(
+                (_extract_article_data(a)["pub_date"] for a in news),
+                start_date, end_date, "Yahoo Finance news", f"news for {ticker}{resolved}",
+            )
+            return gap or f"No news found for {ticker}{resolved} between {start_date} and {end_date}"
 
         return f"## {ticker}{resolved} News, from {start_date} to {end_date}:\n\n{news_str}"
 
@@ -173,9 +174,6 @@ def get_global_news_yfinance(
             if len(all_news) >= limit:
                 break
 
-        if not all_news:
-            return f"No global news found for {curr_date}"
-
         # Calculate date range
         curr_dt = datetime.strptime(curr_date, "%Y-%m-%d")
         start_dt = curr_dt - relativedelta(days=look_back_days)
@@ -200,7 +198,10 @@ def get_global_news_yfinance(
         # All candidates fell outside the window -> say so rather than return an
         # empty-bodied report (#993).
         if kept == 0:
-            return f"No global news found between {start_date} and {curr_date}"
+            # Results merge several fuzzy searches, so their timestamps prove no
+            # continuous coverage; judge the window against the present only.
+            gap = coverage_gap((), start_date, curr_date, "Yahoo Finance global news", "market news")
+            return gap or f"No global news found between {start_date} and {curr_date}"
 
         return f"## Global Market News, from {start_date} to {curr_date}:\n\n{news_str}"