fix(dataflows): map crypto to StockTwits/Reddit sentiment symbols

- crypto reached StockTwits as Yahoo's BTC-USD (404) instead of BTC.X, and Reddit
  searched the dashed pair that barely matches; both now resolve the base via a
  shared crypto_base() helper, restoring crypto sentiment
- also fixes a StockTwits resilience test class that pytest never collected #1113
This commit is contained in:
Yijia-Xiao
2026-07-05 14:29:07 +00:00
parent daf1da9c35
commit a102afa090
6 changed files with 114 additions and 13 deletions

View File

@@ -30,6 +30,8 @@ from urllib.error import HTTPError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from .symbol_utils import crypto_base
logger = logging.getLogger(__name__)
_API = "https://www.reddit.com/r/{sub}/search.json?{qs}"
@@ -200,6 +202,9 @@ def fetch_reddit_posts(
stay under Reddit's public per-IP rate limit; combined with the RSS-first
path it makes 429s rare even when several analyses run back-to-back.
"""
# Crypto reaches us as a Yahoo pair (BTC-USD); search Reddit for the base
# ("BTC") so the query actually matches discussion instead of near-nothing.
ticker = crypto_base(ticker) or ticker
blocks = []
total_posts = 0
for i, sub in enumerate(subreddits):

View File

@@ -19,12 +19,25 @@ import json
import logging
from urllib.request import Request, urlopen
from .symbol_utils import crypto_base
logger = logging.getLogger(__name__)
_API = "https://api.stocktwits.com/api/2/streams/symbol/{ticker}.json"
_UA = "tradingagents/0.2 (+https://github.com/TauricResearch/TradingAgents)"
def _stocktwits_symbol(ticker: str) -> str:
"""Map a crypto pair to StockTwits' ``<BASE>.X`` convention.
StockTwits lists crypto as ``BTC.X`` (Yahoo's ``BTC-USD`` form 404s), so any
crypto symbol resolves to its base plus ``.X``; other symbols pass through
upper-cased.
"""
base = crypto_base(ticker)
return f"{base}.X" if base else ticker.strip().upper()
def fetch_stocktwits_messages(ticker: str, limit: int = 30, timeout: float = 10.0) -> str:
"""Fetch recent StockTwits messages for ``ticker`` and return them as a
formatted plaintext block ready for prompt injection.
@@ -33,7 +46,7 @@ def fetch_stocktwits_messages(ticker: str, limit: int = 30, timeout: float = 10.
symbol has no messages, or the response shape is unexpected — the
caller never has to special-case None or exceptions.
"""
url = _API.format(ticker=ticker.upper())
url = _API.format(ticker=_stocktwits_symbol(ticker))
req = Request(url, headers={"User-Agent": _UA, "Accept": "application/json"})
try:
with urlopen(req, timeout=timeout) as resp:

View File

@@ -80,22 +80,27 @@ _YAHOO_SAFE = re.compile(r"^[A-Za-z0-9._\-\^=]+$")
_CRYPTO_QUOTES = ("USDT", "USDC", "USD")
def _normalize_crypto(s: str) -> str | None:
"""Return ``<BASE>-USD`` if ``s`` is a known crypto quoted in USD/USDT/USDC.
Accepts dashed or undashed forms: ``BTCUSD``, ``BTCUSDT``, ``BTC-USDT``,
``BTC-USDC`` all resolve to ``BTC-USD``. Returns None otherwise.
def crypto_base(raw: str) -> str | None:
"""Return the crypto base (e.g. ``BTC``) for a known USD/USDT/USDC-quoted
crypto symbol in any form the pipeline may hold — ``BTC-USD``, ``BTCUSD``,
``BTC-USDT`` — or None for non-crypto symbols. Purely syntactic.
"""
compact = s.replace("-", "")
if not isinstance(raw, str):
return None
compact = raw.strip().upper().rstrip("+").replace("-", "")
for quote in _CRYPTO_QUOTES:
if compact.endswith(quote):
base = compact[: -len(quote)]
if base in _CRYPTO_BASES:
return f"{base}-USD"
break
return base if base in _CRYPTO_BASES else None
return None
def _normalize_crypto(s: str) -> str | None:
"""Return ``<BASE>-USD`` for a known USD/USDT/USDC-quoted crypto, else None."""
base = crypto_base(s)
return f"{base}-USD" if base else None
def normalize_symbol(raw: str) -> str:
"""Map a user/broker symbol to its canonical Yahoo Finance symbol.