mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-19 11:15:24 +03:00
harden(dataflows): bound the Reddit feed read before parsing
- ElementTree does not resolve external entities, so the reported XXE flag doesn't apply; the real residual is an unbounded read of untrusted network XML - cap both the RSS and JSON reads at 5 MiB; overflow degrades to empty / RSS fallback through the existing failure paths #1206 #1276
This commit is contained in:
@@ -36,8 +36,9 @@ def _resp(read_fn):
|
|||||||
def __exit__(self_inner, *a):
|
def __exit__(self_inner, *a):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def read(self_inner):
|
def read(self_inner, size=-1):
|
||||||
return read_fn()
|
data = read_fn()
|
||||||
|
return data if size is None or size < 0 else data[:size]
|
||||||
return _Resp()
|
return _Resp()
|
||||||
|
|
||||||
|
|
||||||
@@ -183,6 +184,14 @@ class TestChunkedTransferErrorsHandled:
|
|||||||
reddit._fetch_subreddit_json("NVDA", "stocks", 5, 5.0)
|
reddit._fetch_subreddit_json("NVDA", "stocks", 5, 5.0)
|
||||||
rss.assert_called_once()
|
rss.assert_called_once()
|
||||||
|
|
||||||
|
def test_oversized_rss_feed_is_refused_not_parsed(self):
|
||||||
|
# A hostile/misbehaving endpoint streaming an unbounded body must not be
|
||||||
|
# read into memory before parsing; overflow degrades to an empty feed.
|
||||||
|
big = _resp(lambda: b"x" * 100)
|
||||||
|
with patch.object(reddit, "_MAX_FEED_BYTES", 10), \
|
||||||
|
patch.object(reddit, "urlopen", return_value=big):
|
||||||
|
assert reddit._fetch_subreddit_rss("NVDA", "stocks", 5, 5.0) == []
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.unit
|
@pytest.mark.unit
|
||||||
class TestFormatterHandlesRssPosts:
|
class TestFormatterHandlesRssPosts:
|
||||||
|
|||||||
@@ -126,6 +126,23 @@ def _retry_after_seconds(exc: HTTPError) -> float | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Reddit search feeds are small (a page of results); cap the read so a
|
||||||
|
# compromised or misbehaving endpoint can't stream an unbounded body into
|
||||||
|
# memory before we parse it. Overflow raises http.client.HTTPException, which
|
||||||
|
# both fetch paths already treat as a failed fetch (degrade to empty / RSS).
|
||||||
|
_MAX_FEED_BYTES = 5 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def _read_capped(resp) -> bytes:
|
||||||
|
"""Read a response body bounded to ``_MAX_FEED_BYTES``, raising on overflow."""
|
||||||
|
data = resp.read(_MAX_FEED_BYTES + 1)
|
||||||
|
if len(data) > _MAX_FEED_BYTES:
|
||||||
|
raise http.client.HTTPException(
|
||||||
|
f"Reddit feed exceeded {_MAX_FEED_BYTES} bytes; refusing to parse"
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
def _fetch_subreddit_rss(
|
def _fetch_subreddit_rss(
|
||||||
ticker: str,
|
ticker: str,
|
||||||
sub: str,
|
sub: str,
|
||||||
@@ -144,7 +161,7 @@ def _fetch_subreddit_rss(
|
|||||||
req = Request(url, headers={"User-Agent": _UA})
|
req = Request(url, headers={"User-Agent": _UA})
|
||||||
try:
|
try:
|
||||||
with urlopen(req, timeout=timeout) as resp:
|
with urlopen(req, timeout=timeout) as resp:
|
||||||
root = ET.fromstring(resp.read())
|
root = ET.fromstring(_read_capped(resp))
|
||||||
except HTTPError as exc:
|
except HTTPError as exc:
|
||||||
if exc.code == 429 and _retry:
|
if exc.code == 429 and _retry:
|
||||||
# Honour a server-supplied Retry-After exactly (including 0); jitter
|
# Honour a server-supplied Retry-After exactly (including 0); jitter
|
||||||
@@ -201,7 +218,7 @@ def _fetch_subreddit_json(
|
|||||||
req = Request(url, headers={"User-Agent": _UA, "Accept": "application/json"})
|
req = Request(url, headers={"User-Agent": _UA, "Accept": "application/json"})
|
||||||
try:
|
try:
|
||||||
with urlopen(req, timeout=timeout) as resp:
|
with urlopen(req, timeout=timeout) as resp:
|
||||||
payload = json.loads(resp.read())
|
payload = json.loads(_read_capped(resp))
|
||||||
children = (payload.get("data") or {}).get("children") or []
|
children = (payload.get("data") or {}).get("children") or []
|
||||||
return [c.get("data", {}) for c in children if isinstance(c, dict)]
|
return [c.get("data", {}) for c in children if isinstance(c, dict)]
|
||||||
except (OSError, http.client.HTTPException, json.JSONDecodeError) as exc:
|
except (OSError, http.client.HTTPException, json.JSONDecodeError) as exc:
|
||||||
|
|||||||
Reference in New Issue
Block a user