From a102afa090dad4912a2a445d294ccfc49cc0e1ce Mon Sep 17 00:00:00 2001
From: Yijia-Xiao <yijia-xiao@outlook.com>
Date: Sun, 5 Jul 2026 14:29:07 +0000
Subject: [PATCH] fix(dataflows): map crypto to StockTwits/Reddit sentiment
 symbols

- crypto reached StockTwits as Yahoo's BTC-USD (404) instead of BTC.X, and Reddit
  searched the dashed pair that barely matches; both now resolve the base via a
  shared crypto_base() helper, restoring crypto sentiment
- also fixes a StockTwits resilience test class that pytest never collected #1113
---
 tests/test_reddit_fallback.py           | 22 +++++++++++++
 tests/test_stocktwits_resilience.py     | 41 +++++++++++++++++++++++--
 tests/test_symbol_utils.py              | 21 +++++++++++++
 tradingagents/dataflows/reddit.py       |  5 +++
 tradingagents/dataflows/stocktwits.py   | 15 ++++++++-
 tradingagents/dataflows/symbol_utils.py | 23 ++++++++------
 6 files changed, 114 insertions(+), 13 deletions(-)

diff --git a/tests/test_reddit_fallback.py b/tests/test_reddit_fallback.py
index 7fd98a90c..dece01ca6 100644
--- a/tests/test_reddit_fallback.py
+++ b/tests/test_reddit_fallback.py
@@ -190,3 +190,25 @@ class TestFormatterHandlesRssPosts:
         assert "1234↑" in out
         assert "56c" in out
         assert "via RSS" not in out
+
+
+@pytest.mark.unit
+class TestCryptoSearchTerm:
+    """A crypto pair (BTC-USD) barely matches Reddit text; search the base (#1113)."""
+
+    def _captured_ticker(self, ticker):
+        seen = {}
+
+        def fake_fetch(t, sub, limit, timeout):
+            seen["ticker"] = t
+            return []
+
+        with patch.object(reddit, "_fetch_subreddit", side_effect=fake_fetch):
+            reddit.fetch_reddit_posts(ticker, subreddits=("stocks",), inter_request_delay=0)
+        return seen["ticker"]
+
+    def test_crypto_pair_searches_base(self):
+        assert self._captured_ticker("BTC-USD") == "BTC"
+
+    def test_equity_passes_through(self):
+        assert self._captured_ticker("NVDA") == "NVDA"
diff --git a/tests/test_stocktwits_resilience.py b/tests/test_stocktwits_resilience.py
index 145e2267d..8e963f5c5 100644
--- a/tests/test_stocktwits_resilience.py
+++ b/tests/test_stocktwits_resilience.py
@@ -1,5 +1,9 @@
-"""StockTwits fetch degrades (never raises) on transport errors, including the
-http.client chunked-transfer exceptions that are not OSErrors (#1024)."""
+"""StockTwits fetch: transport-error resilience (#1024) and crypto symbol
+mapping (#1113).
+
+StockTwits lists crypto under ``<BASE>.X`` (Yahoo's ``BTC-USD`` 404s), and any
+transport error must degrade to a placeholder rather than raise.
+"""
 
 from __future__ import annotations
 
@@ -26,7 +30,7 @@ def _raise(exc):
 
 
 @pytest.mark.unit
-class StockTwitsResilienceTests:
+class TestStockTwitsResilience:
     @pytest.mark.parametrize(
         "exc",
         [
@@ -40,3 +44,34 @@ class StockTwitsResilienceTests:
             out = stocktwits.fetch_stocktwits_messages("NVDA")
         assert "unavailable" in out.lower()
         assert out.startswith("<stocktwits unavailable")
+
+
+@pytest.mark.unit
+class TestStockTwitsCryptoSymbols:
+    @pytest.mark.parametrize(
+        ("ticker", "expected"),
+        [
+            ("BTC-USD", "BTC.X"),
+            ("eth-usd", "ETH.X"),
+            ("SOL-USD", "SOL.X"),
+            ("BTCUSD", "BTC.X"),      # undashed broker form
+            ("BTC-USDT", "BTC.X"),    # stablecoin quote
+            ("AMD", "AMD"),
+            ("BRK-B", "BRK-B"),       # dashed class share: untouched
+            ("GOLD", "GOLD"),         # real equity (aliases elsewhere): untouched here
+            ("XYZ-USD", "XYZ-USD"),   # unknown base: not treated as crypto
+        ],
+    )
+    def test_symbol_mapping(self, ticker, expected):
+        assert stocktwits._stocktwits_symbol(ticker) == expected
+
+    def test_crypto_pair_requests_dot_x_endpoint(self):
+        seen = {}
+
+        def fake_urlopen(req, timeout=None):
+            seen["url"] = req.full_url
+            raise TimeoutError("stop after capturing the URL")
+
+        with patch.object(stocktwits, "urlopen", side_effect=fake_urlopen):
+            stocktwits.fetch_stocktwits_messages("BTC-USD")
+        assert "/symbol/BTC.X.json" in seen["url"]
diff --git a/tests/test_symbol_utils.py b/tests/test_symbol_utils.py
index 774c0f9bf..9db3894ff 100644
--- a/tests/test_symbol_utils.py
+++ b/tests/test_symbol_utils.py
@@ -6,6 +6,7 @@ import pytest
 
 from tradingagents.dataflows.symbol_utils import (
     NoMarketDataError,
+    crypto_base,
     is_yahoo_safe,
     normalize_symbol,
 )
@@ -77,5 +78,25 @@ class TestIsYahooSafe(unittest.TestCase):
             self.assertFalse(is_yahoo_safe(sym))
 
 
+@pytest.mark.unit
+class TestCryptoBase(unittest.TestCase):
+    def test_resolves_known_crypto_forms(self):
+        for raw in ("BTC-USD", "BTCUSD", "btc-usdt", "BTC-USDC", "BTCUSD+"):
+            self.assertEqual(crypto_base(raw), "BTC")
+        self.assertEqual(crypto_base("ETH-USD"), "ETH")
+        self.assertEqual(crypto_base("sol-usd"), "SOL")
+
+    def test_non_crypto_returns_none(self):
+        # Plain equities, class shares, and real tickers that alias elsewhere
+        # (GOLD -> gold future on the Yahoo path) must NOT read as crypto.
+        for raw in ("AAPL", "BRK-B", "GOLD", "XYZ-USD", "EURUSD", "", None):
+            self.assertIsNone(crypto_base(raw))
+
+    def test_agrees_with_normalize_symbol(self):
+        # crypto_base is the shared primitive behind the -USD normalization.
+        self.assertEqual(normalize_symbol("BTCUSD"), "BTC-USD")
+        self.assertEqual(crypto_base("BTCUSD"), "BTC")
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tradingagents/dataflows/reddit.py b/tradingagents/dataflows/reddit.py
index bfb87700b..edb4aedfc 100644
--- a/tradingagents/dataflows/reddit.py
+++ b/tradingagents/dataflows/reddit.py
@@ -30,6 +30,8 @@ from urllib.error import HTTPError
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
 
+from .symbol_utils import crypto_base
+
 logger = logging.getLogger(__name__)
 
 _API = "https://www.reddit.com/r/{sub}/search.json?{qs}"
@@ -200,6 +202,9 @@ def fetch_reddit_posts(
     stay under Reddit's public per-IP rate limit; combined with the RSS-first
     path it makes 429s rare even when several analyses run back-to-back.
     """
+    # Crypto reaches us as a Yahoo pair (BTC-USD); search Reddit for the base
+    # ("BTC") so the query actually matches discussion instead of near-nothing.
+    ticker = crypto_base(ticker) or ticker
     blocks = []
     total_posts = 0
     for i, sub in enumerate(subreddits):
diff --git a/tradingagents/dataflows/stocktwits.py b/tradingagents/dataflows/stocktwits.py
index 1594de1d3..1c2fd02bb 100644
--- a/tradingagents/dataflows/stocktwits.py
+++ b/tradingagents/dataflows/stocktwits.py
@@ -19,12 +19,25 @@ import json
 import logging
 from urllib.request import Request, urlopen
 
+from .symbol_utils import crypto_base
+
 logger = logging.getLogger(__name__)
 
 _API = "https://api.stocktwits.com/api/2/streams/symbol/{ticker}.json"
 _UA = "tradingagents/0.2 (+https://github.com/TauricResearch/TradingAgents)"
 
 
+def _stocktwits_symbol(ticker: str) -> str:
+    """Map a crypto pair to StockTwits' ``<BASE>.X`` convention.
+
+    StockTwits lists crypto as ``BTC.X`` (Yahoo's ``BTC-USD`` form 404s), so any
+    crypto symbol resolves to its base plus ``.X``; other symbols pass through
+    upper-cased.
+    """
+    base = crypto_base(ticker)
+    return f"{base}.X" if base else ticker.strip().upper()
+
+
 def fetch_stocktwits_messages(ticker: str, limit: int = 30, timeout: float = 10.0) -> str:
     """Fetch recent StockTwits messages for ``ticker`` and return them as a
     formatted plaintext block ready for prompt injection.
@@ -33,7 +46,7 @@ def fetch_stocktwits_messages(ticker: str, limit: int = 30, timeout: float = 10.
     symbol has no messages, or the response shape is unexpected — the
     caller never has to special-case None or exceptions.
     """
-    url = _API.format(ticker=ticker.upper())
+    url = _API.format(ticker=_stocktwits_symbol(ticker))
     req = Request(url, headers={"User-Agent": _UA, "Accept": "application/json"})
     try:
         with urlopen(req, timeout=timeout) as resp:
diff --git a/tradingagents/dataflows/symbol_utils.py b/tradingagents/dataflows/symbol_utils.py
index af46a1f02..f91b4e149 100644
--- a/tradingagents/dataflows/symbol_utils.py
+++ b/tradingagents/dataflows/symbol_utils.py
@@ -80,22 +80,27 @@ _YAHOO_SAFE = re.compile(r"^[A-Za-z0-9._\-\^=]+$")
 _CRYPTO_QUOTES = ("USDT", "USDC", "USD")
 
 
-def _normalize_crypto(s: str) -> str | None:
-    """Return ``<BASE>-USD`` if ``s`` is a known crypto quoted in USD/USDT/USDC.
-
-    Accepts dashed or undashed forms: ``BTCUSD``, ``BTCUSDT``, ``BTC-USDT``,
-    ``BTC-USDC`` all resolve to ``BTC-USD``. Returns None otherwise.
+def crypto_base(raw: str) -> str | None:
+    """Return the crypto base (e.g. ``BTC``) for a known USD/USDT/USDC-quoted
+    crypto symbol in any form the pipeline may hold — ``BTC-USD``, ``BTCUSD``,
+    ``BTC-USDT`` — or None for non-crypto symbols. Purely syntactic.
     """
-    compact = s.replace("-", "")
+    if not isinstance(raw, str):
+        return None
+    compact = raw.strip().upper().rstrip("+").replace("-", "")
     for quote in _CRYPTO_QUOTES:
         if compact.endswith(quote):
             base = compact[: -len(quote)]
-            if base in _CRYPTO_BASES:
-                return f"{base}-USD"
-            break
+            return base if base in _CRYPTO_BASES else None
     return None
 
 
+def _normalize_crypto(s: str) -> str | None:
+    """Return ``<BASE>-USD`` for a known USD/USDT/USDC-quoted crypto, else None."""
+    base = crypto_base(s)
+    return f"{base}-USD" if base else None
+
+
 def normalize_symbol(raw: str) -> str:
     """Map a user/broker symbol to its canonical Yahoo Finance symbol.