diff --git a/tests/test_stocktwits_resilience.py b/tests/test_stocktwits_resilience.py index 6975b68b7..aad640455 100644 --- a/tests/test_stocktwits_resilience.py +++ b/tests/test_stocktwits_resilience.py @@ -114,3 +114,10 @@ class TestStockTwitsScreening: with patch.object(stocktwits, "urlopen", return_value=_stream("SPAM")): out = stocktwits.fetch_stocktwits_messages("NVDA", screen=_drop_spam) assert "none of the 1 StockTwits messages is about $NVDA" in out + + +@pytest.mark.unit +def test_html_entities_in_message_bodies_are_decoded(): + with patch.object(stocktwits, "urlopen", return_value=_stream("S&P wasn't up")): + out = stocktwits.fetch_stocktwits_messages("NVDA") + assert "S&P wasn't up" in out diff --git a/tradingagents/dataflows/vendors/stocktwits.py b/tradingagents/dataflows/vendors/stocktwits.py index b6985fc89..79073982b 100644 --- a/tradingagents/dataflows/vendors/stocktwits.py +++ b/tradingagents/dataflows/vendors/stocktwits.py @@ -15,6 +15,7 @@ network call succeeded. from __future__ import annotations import contextlib +import html import http.client import json import logging @@ -54,6 +55,11 @@ def _within_window(messages, start_date, end_date): return [m for m in messages if in_window(_created_at(m), start_dt, end_dt)] +def _body(message) -> str: + """The message text; the API serves it HTML-escaped (``&``, ``'``).""" + return html.unescape(message.get("body") or "") + + def _stocktwits_symbol(ticker: str) -> str: """Map a crypto pair to StockTwits' ``.X`` convention. @@ -115,7 +121,7 @@ def fetch_stocktwits_messages( note = "" if screen: - keep, note = screen([m.get("body") or "" for m in messages]) + keep, note = screen([_body(m) for m in messages]) screened = len(messages) messages = [m for m, kept in zip(messages, keep, strict=True) if kept] if not messages: @@ -129,7 +135,7 @@ def fetch_stocktwits_messages( entities = m.get("entities") or {} sentiment_obj = entities.get("sentiment") or {} sentiment = sentiment_obj.get("basic") if isinstance(sentiment_obj, dict) else None - body = (m.get("body") or "").replace("\n", " ").strip() + body = _body(m).replace("\n", " ").strip() if len(body) > 280: body = body[:280] + "…"