fix(dataflows): honour Reddit Retry-After: 0 and jitter 429 backoff

- a valid Retry-After: 0 means retry at once but was treated as absent
  (`or 5.0`) and waited 5s; honour it exactly now
- jitter our own headerless fallback and the inter-subreddit pacing so several
  analyses sharing an IP don't retry in lockstep and re-collide on the limit;
  keep the single-retry ceiling (more retries can't fix an exhausted IP budget) #1193
This commit is contained in:
Yijia-Xiao
2026-09-01 05:02:36 +00:00
parent 70b58c21dc
commit 2322dd9baa
2 changed files with 44 additions and 5 deletions

View File

@@ -147,6 +147,26 @@ class TestRss429Backoff:
reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0)
slept.assert_called_once_with(12.0)
def test_retry_after_zero_is_honoured_not_treated_as_absent(self):
# A valid "Retry-After: 0" means retry at once; it must not fall through
# to the fallback wait (the earlier `or 5.0` bug turned 0 into 5s).
err = HTTPError("url", 429, "Too Many Requests", {"Retry-After": "0"}, None)
with patch.object(reddit, "urlopen", side_effect=[err, _atom_resp()]), \
patch.object(reddit.time, "sleep") as slept:
reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0)
slept.assert_called_once_with(0.0)
def test_headerless_429_fallback_is_jittered(self):
# No Retry-After -> our own ~5s fallback, jittered so concurrent runs
# don't retry in lockstep (kept within a tight band).
err = HTTPError("url", 429, "Too Many Requests", {}, None)
with patch.object(reddit, "urlopen", side_effect=[err, _atom_resp()]), \
patch.object(reddit.time, "sleep") as slept:
reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0)
slept.assert_called_once()
(wait,), _ = slept.call_args
assert 4.0 <= wait <= 6.0 # 5s +/-20% jitter
@pytest.mark.unit
class TestChunkedTransferErrorsHandled: