fix(dataflows): report a failed Reddit fetch as unavailable, not silence

- a failed fetch and an empty search both returned [], so a 429 rendered as
  'no posts found' and the sentiment analyst read throttling as real silence;
  when every subreddit was throttled the summary asserted it outright
- a failed fetch now returns None and renders as unavailable, and the summary
  only claims silence for subreddits actually searched
- raise the headerless-429 back-off to 60s, which is where a retry starts
  succeeding; pay it at most once per run so three throttled subreddits do not
  stall the analysis, and match the Retry-After cap to it #1295
This commit is contained in:
Yijia-Xiao
2026-09-07 20:54:19 +00:00
parent 1c44dd1ffc
commit 7cc478ad07
2 changed files with 129 additions and 23 deletions

View File

@@ -86,9 +86,9 @@ class TestRssParsing:
assert posts[0]["created_utc"] > 0
assert "datacenter unit" in posts[0]["selftext"]
def test_malformed_xml_fails_open(self):
def test_malformed_xml_reports_unavailable(self):
with patch.object(reddit, "urlopen", return_value=_resp(lambda: b"<<not xml>>")):
assert reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0) == []
assert reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0) is None
@pytest.mark.unit
@@ -139,7 +139,7 @@ class TestRss429Backoff:
patch.object(reddit.time, "sleep"):
posts = reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0)
assert op.call_count == 2 # one retry, then gives up cleanly
assert posts == []
assert posts is None
def test_retry_after_header_is_honoured(self):
err = HTTPError("url", 429, "Too Many Requests", {"Retry-After": "12"}, None)
@@ -166,7 +166,7 @@ class TestRss429Backoff:
reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0)
slept.assert_called_once()
(wait,), _ = slept.call_args
assert 4.0 <= wait <= 6.0 # 5s +/-20% jitter
assert 48.0 <= wait <= 72.0 # 60s +/-20% jitter
@pytest.mark.unit
@@ -174,9 +174,9 @@ class TestChunkedTransferErrorsHandled:
"""IncompleteRead/RemoteDisconnected come from http.client and are NOT
OSErrors, so they were previously uncaught and crashed the pipeline (#1024)."""
def test_rss_incomplete_read_degrades_to_empty(self):
def test_rss_incomplete_read_reports_unavailable(self):
with patch.object(reddit, "urlopen", return_value=_raise(http.client.IncompleteRead(b""))):
assert reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0) == []
assert reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0) is None
def test_json_incomplete_read_falls_back_to_rss(self):
with patch.object(reddit, "urlopen", return_value=_raise(http.client.IncompleteRead(b""))), \
@@ -190,7 +190,7 @@ class TestChunkedTransferErrorsHandled:
big = _resp(lambda: b"x" * 100)
with patch.object(reddit, "_MAX_FEED_BYTES", 10), \
patch.object(reddit, "urlopen", return_value=big):
assert reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0) == []
assert reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0) is None
@pytest.mark.unit
@@ -228,7 +228,7 @@ class TestCryptoSearchTerm:
def _captured_ticker(self, ticker):
seen = {}
def fake_fetch(t, sub, limit, timeout):
def fake_fetch(t, sub, limit, timeout, **kwargs):
seen["ticker"] = t
return []
@@ -241,3 +241,64 @@ class TestCryptoSearchTerm:
def test_equity_passes_through(self):
assert self._captured_ticker("NVDA") == "NVDA"
@pytest.mark.unit
class TestFailedFetchIsNotSilence:
"""A throttled fetch must not be rendered as "no posts found" (#1295).
Returning [] for both a failed request and a genuinely empty search made the
sentiment analyst read rate limiting as real silence ("r/stocks and
r/investing are silent"), which is a signal that was never observed.
"""
_POST = {
"title": "NVDA pops", "score": None, "num_comments": None,
"created_utc": reddit._iso_to_timestamp("2026-05-20T14:30:00Z"),
"selftext": "", "source": "rss",
}
def _run(self, results):
"""Drive fetch_reddit_posts with a per-subreddit result sequence."""
subs = tuple(f"s{i}" for i in range(len(results)))
with patch.object(reddit, "_fetch_subreddit", side_effect=list(results)):
return reddit.fetch_reddit_posts(
"NVDA", subreddits=subs, inter_request_delay=0
)
def test_failed_subreddit_is_marked_unavailable_not_empty(self):
out = self._run([None, [self._POST]])
assert "unavailable" in out
assert "no posts found" not in out.split("unavailable")[0]
def test_all_sources_failing_does_not_claim_no_posts(self):
out = self._run([None, None])
assert "Reddit unavailable" in out
assert "no Reddit posts found" not in out
def test_mixed_failure_and_empty_only_claims_silence_for_searched_subs(self):
# s0 failed, s1 genuinely returned nothing: the "no posts" claim must
# cover only s1, with s0 reported separately as unavailable.
out = self._run([None, []])
assert "r/s1" in out.split("unavailable (fetch failed)")[0]
assert "unavailable (fetch failed): r/s0" in out
def test_genuine_empty_still_reports_no_posts(self):
out = self._run([[], []])
assert "no Reddit posts found" in out
assert "unavailable" not in out
def test_retry_is_not_spent_again_after_a_failure(self):
# The 60s back-off must be paid at most once per run, so subsequent
# subreddits are fetched with retry disabled rather than stalling.
seen = []
def record(t, sub, limit, timeout, _retry=True):
seen.append(_retry)
return None
with patch.object(reddit, "_fetch_subreddit", side_effect=record):
reddit.fetch_reddit_posts(
"NVDA", subreddits=("a", "b", "c"), inter_request_delay=0
)
assert seen == [True, False, False]