fix(stocktwits): decode HTML entities in message bodies

- bodies reach the prompt and the screen as plain text (S&P, wasn't) rather than escaped (S&P, wasn't)
This commit is contained in:
Yijia-Xiao
2026-09-24 05:00:36 +00:00
parent 24c38602e5
commit 4187716842
2 changed files with 15 additions and 2 deletions
+7
View File
@@ -114,3 +114,10 @@ class TestStockTwitsScreening:
with patch.object(stocktwits, "urlopen", return_value=_stream("SPAM")):
out = stocktwits.fetch_stocktwits_messages("NVDA", screen=_drop_spam)
assert "none of the 1 StockTwits messages is about $NVDA" in out
@pytest.mark.unit
def test_html_entities_in_message_bodies_are_decoded():
with patch.object(stocktwits, "urlopen", return_value=_stream("S&P wasn't up")):
out = stocktwits.fetch_stocktwits_messages("NVDA")
assert "S&P wasn't up" in out
+8 -2
View File
@@ -15,6 +15,7 @@ network call succeeded.
from __future__ import annotations
import contextlib
import html
import http.client
import json
import logging
@@ -54,6 +55,11 @@ def _within_window(messages, start_date, end_date):
return [m for m in messages if in_window(_created_at(m), start_dt, end_dt)]
def _body(message) -> str:
"""The message text; the API serves it HTML-escaped (``&``, ``'``)."""
return html.unescape(message.get("body") or "")
def _stocktwits_symbol(ticker: str) -> str:
"""Map a crypto pair to StockTwits' ``<BASE>.X`` convention.
@@ -115,7 +121,7 @@ def fetch_stocktwits_messages(
note = ""
if screen:
keep, note = screen([m.get("body") or "" for m in messages])
keep, note = screen([_body(m) for m in messages])
screened = len(messages)
messages = [m for m, kept in zip(messages, keep, strict=True) if kept]
if not messages:
@@ -129,7 +135,7 @@ def fetch_stocktwits_messages(
entities = m.get("entities") or {}
sentiment_obj = entities.get("sentiment") or {}
sentiment = sentiment_obj.get("basic") if isinstance(sentiment_obj, dict) else None
body = (m.get("body") or "").replace("\n", " ").strip()
body = _body(m).replace("\n", " ").strip()
if len(body) > 280:
body = body[:280] + "…"