fix: support commodity/forex/crypto tickers and never invent prices (#781)

Analyzing a symbol Yahoo Finance does not recognize (e.g. XAUUSD+) could
produce an invented price instead of an error. The agent now either prices
the correct instrument or clearly reports that data is unavailable.

Ticker support:
- Commodities/forex/crypto resolve to the symbol Yahoo actually serves, so
  you can enter the common form and it just works:
    XAUUSD / XAUUSD+ / GOLD  -> GC=F   (gold)
    USOIL                    -> CL=F   (WTI crude)
    EURUSD                   -> EURUSD=X
    BTCUSD                   -> BTC-USD
    SPX500 / NAS100          -> ^GSPC / ^NDX
  Native Yahoo symbols (AAPL, GC=F, ^GSPC) keep working unchanged. New
  instruments are added by extending the alias table.

Reliability:
- Unknown or delisted symbols now return a clear "data unavailable" result
  the agent reports verbatim, instead of a value the model fills in.
- A failed fetch no longer leaves a broken symbol cached until the cache is
  cleared by hand.
This commit is contained in:
Yijia-Xiao
2026-05-31 22:38:47 +00:00
parent 2f85be624e
commit 1ff3f07a73
10 changed files with 460 additions and 53 deletions

View File

@@ -0,0 +1,88 @@
"""Tests that empty vendor results never become fabricated data.
Covers two systematic fixes:
- load_ohlcv must not cache an empty download (cache poisoning), and must
raise NoMarketDataError instead of returning an empty frame.
- route_to_vendor must convert NoMarketDataError into a single explicit
"NO_DATA_AVAILABLE" sentinel after all vendors are exhausted.
"""
import os
import unittest
from unittest import mock
import pandas as pd
import pytest
from tradingagents.dataflows import stockstats_utils, interface
from tradingagents.dataflows.config import set_config
from tradingagents.dataflows.symbol_utils import NoMarketDataError
@pytest.mark.unit
class TestLoadOhlcvNoPoison(unittest.TestCase):
def setUp(self):
self._tmp = os.path.join(os.path.dirname(__file__), "_tmp_cache")
os.makedirs(self._tmp, exist_ok=True)
set_config({"data_cache_dir": self._tmp})
def tearDown(self):
for f in os.listdir(self._tmp):
os.remove(os.path.join(self._tmp, f))
os.rmdir(self._tmp)
def test_empty_download_raises_and_does_not_cache(self):
empty = pd.DataFrame()
with mock.patch.object(stockstats_utils.yf, "download", return_value=empty) as dl:
with self.assertRaises(NoMarketDataError):
stockstats_utils.load_ohlcv("FAKE", "2026-01-01")
# Nothing should have been written to the cache.
self.assertEqual(os.listdir(self._tmp), [])
# A second call must re-attempt the fetch (no poisoned cache served).
with mock.patch.object(stockstats_utils.yf, "download", return_value=empty) as dl2:
with self.assertRaises(NoMarketDataError):
stockstats_utils.load_ohlcv("FAKE", "2026-01-01")
self.assertTrue(dl2.called)
@pytest.mark.unit
class TestRouteToVendorSentinel(unittest.TestCase):
def test_no_data_from_all_vendors_returns_sentinel(self):
def raises_no_data(symbol, *a, **k):
raise NoMarketDataError(symbol, "GC=F", "no rows")
patched = {"yfinance": raises_no_data, "alpha_vantage": raises_no_data}
with mock.patch.dict(
interface.VENDOR_METHODS, {"get_stock_data": patched}, clear=False
):
result = interface.route_to_vendor(
"get_stock_data", "XAUUSD+", "2026-01-01", "2026-01-10"
)
self.assertIn("NO_DATA_AVAILABLE", result)
self.assertIn("XAUUSD+", result)
self.assertIn("GC=F", result)
self.assertIn("Do not estimate", result)
def test_unconfigured_fallback_does_not_mask_no_data(self):
# When the primary vendor reports no data and the fallback is simply
# unavailable (e.g. missing API key -> raises), the no-data sentinel
# must win rather than the fallback's incidental error crashing out.
def raises_no_data(symbol, *a, **k):
raise NoMarketDataError(symbol, symbol, "no rows")
def raises_unavailable(symbol, *a, **k):
raise ValueError("ALPHA_VANTAGE_API_KEY environment variable is not set.")
patched = {"yfinance": raises_no_data, "alpha_vantage": raises_unavailable}
with mock.patch.dict(
interface.VENDOR_METHODS, {"get_stock_data": patched}, clear=False
):
result = interface.route_to_vendor(
"get_stock_data", "FAKE", "2026-01-01", "2026-01-10"
)
self.assertIn("NO_DATA_AVAILABLE", result)
if __name__ == "__main__":
unittest.main()

View File

@@ -14,6 +14,11 @@ class TestSafeTickerComponent(unittest.TestCase):
for ticker in ("AAPL", "BRK-B", "BRK.A", "0700.HK", "7203.T", "BHP.AX", "^GSPC"):
self.assertEqual(safe_ticker_component(ticker), ticker)
def test_accepts_futures_and_forex_formats(self):
# Futures use '=' (GC=F gold, CL=F crude), forex/CFD symbols use '+'.
for ticker in ("GC=F", "CL=F", "ES=F", "XAUUSD+", "EURUSD+"):
self.assertEqual(safe_ticker_component(ticker), ticker)
def test_rejects_path_separators(self):
for bad in (".", "..", "../etc", "a/b", "a\\b", "/abs", "..\\..\\x"):
with self.assertRaises(ValueError):

View File

@@ -0,0 +1,81 @@
"""Tests for symbol normalization and the no-data routing sentinel."""
import unittest
import pytest
from tradingagents.dataflows.symbol_utils import (
NoMarketDataError,
normalize_symbol,
is_yahoo_safe,
)
@pytest.mark.unit
class TestNormalizeSymbol(unittest.TestCase):
def test_plain_equities_unchanged(self):
for sym in ("AAPL", "MSFT", "TSM", "BRK.B", "0700.HK", "^GSPC", "GC=F"):
self.assertEqual(normalize_symbol(sym), sym)
def test_lowercases_are_upper(self):
self.assertEqual(normalize_symbol("aapl"), "AAPL")
self.assertEqual(normalize_symbol(" msft "), "MSFT")
def test_metal_aliases_map_to_futures(self):
self.assertEqual(normalize_symbol("XAUUSD"), "GC=F")
self.assertEqual(normalize_symbol("XAUUSD+"), "GC=F") # broker CFD suffix
self.assertEqual(normalize_symbol("xauusd+"), "GC=F")
self.assertEqual(normalize_symbol("GOLD"), "GC=F")
self.assertEqual(normalize_symbol("XAGUSD"), "SI=F")
def test_energy_and_index_aliases(self):
self.assertEqual(normalize_symbol("USOIL"), "CL=F")
self.assertEqual(normalize_symbol("SPX500"), "^GSPC")
self.assertEqual(normalize_symbol("NAS100"), "^NDX")
self.assertEqual(normalize_symbol("US30"), "^DJI")
def test_forex_pairs_get_x_suffix(self):
self.assertEqual(normalize_symbol("EURUSD"), "EURUSD=X")
self.assertEqual(normalize_symbol("GBPJPY"), "GBPJPY=X")
self.assertEqual(normalize_symbol("eurusd"), "EURUSD=X")
def test_crypto_pairs_get_dash_usd(self):
self.assertEqual(normalize_symbol("BTCUSD"), "BTC-USD")
self.assertEqual(normalize_symbol("ETHUSD"), "ETH-USD")
def test_six_letter_non_currency_left_alone(self):
# GOOGLE-style 6-letter tickers that aren't two currency codes
# must not be mangled into a fake forex pair.
self.assertEqual(normalize_symbol("ABCDEF"), "ABCDEF")
def test_empty_input_passthrough(self):
self.assertEqual(normalize_symbol(""), "")
@pytest.mark.unit
class TestNoMarketDataError(unittest.TestCase):
def test_message_includes_resolution(self):
err = NoMarketDataError("XAUUSD+", "GC=F", "no rows")
self.assertIn("XAUUSD+", str(err))
self.assertIn("GC=F", str(err))
self.assertEqual(err.symbol, "XAUUSD+")
self.assertEqual(err.canonical, "GC=F")
def test_canonical_defaults_to_symbol(self):
err = NoMarketDataError("FOOBAR")
self.assertEqual(err.canonical, "FOOBAR")
@pytest.mark.unit
class TestIsYahooSafe(unittest.TestCase):
def test_accepts_structural_chars(self):
for sym in ("AAPL", "GC=F", "^GSPC", "BRK.B", "BTC-USD"):
self.assertTrue(is_yahoo_safe(sym))
def test_rejects_slash_and_space(self):
for sym in ("a/b", "AA PL", ""):
self.assertFalse(is_yahoo_safe(sym))
if __name__ == "__main__":
unittest.main()