mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-19 19:25:24 +03:00
fix(dataflows): quote the prices the vendor reported in the verification snapshot
- gap filling keeps indicators on a continuous series, but put the previous session's open, high and low under an unsettled bar's date - load_ohlcv takes fill_gaps, and the snapshot reads the frame as reported
This commit is contained in:
@@ -29,7 +29,7 @@ class TestVerifiedSnapshot:
|
|||||||
pd.DataFrame({"Date": [pd.Timestamp("2026-06-01")], "Open": [999.0],
|
pd.DataFrame({"Date": [pd.Timestamp("2026-06-01")], "Open": [999.0],
|
||||||
"High": [999.0], "Low": [999.0], "Close": [999.0], "Volume": [999]}),
|
"High": [999.0], "Low": [999.0], "Close": [999.0], "Volume": [999]}),
|
||||||
], ignore_index=True)
|
], ignore_index=True)
|
||||||
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: data)
|
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: data)
|
||||||
|
|
||||||
snap = validator.build_verified_market_snapshot("COF", "2026-05-13")
|
snap = validator.build_verified_market_snapshot("COF", "2026-05-13")
|
||||||
assert "Verified market data snapshot for COF" in snap
|
assert "Verified market data snapshot for COF" in snap
|
||||||
@@ -39,24 +39,24 @@ class TestVerifiedSnapshot:
|
|||||||
assert "boll_lb" in snap # indicators present
|
assert "boll_lb" in snap # indicators present
|
||||||
|
|
||||||
def test_uses_previous_trading_day_when_date_is_weekend(self, monkeypatch):
|
def test_uses_previous_trading_day_when_date_is_weekend(self, monkeypatch):
|
||||||
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: _sample_ohlcv())
|
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: _sample_ohlcv())
|
||||||
# 2026-05-16 is a Saturday; latest row should be Fri 2026-05-15
|
# 2026-05-16 is a Saturday; latest row should be Fri 2026-05-15
|
||||||
snap = validator.build_verified_market_snapshot("COF", "2026-05-16")
|
snap = validator.build_verified_market_snapshot("COF", "2026-05-16")
|
||||||
assert "Latest trading row used: 2026-05-15" in snap
|
assert "Latest trading row used: 2026-05-15" in snap
|
||||||
assert "Recent verified closes" in snap
|
assert "Recent verified closes" in snap
|
||||||
|
|
||||||
def test_raises_when_no_rows_on_or_before_date(self, monkeypatch):
|
def test_raises_when_no_rows_on_or_before_date(self, monkeypatch):
|
||||||
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: _sample_ohlcv())
|
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: _sample_ohlcv())
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
validator.build_verified_market_snapshot("COF", "2020-01-01")
|
validator.build_verified_market_snapshot("COF", "2020-01-01")
|
||||||
|
|
||||||
def test_raises_on_empty_data(self, monkeypatch):
|
def test_raises_on_empty_data(self, monkeypatch):
|
||||||
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: pd.DataFrame())
|
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: pd.DataFrame())
|
||||||
with pytest.raises(ValueError):
|
with pytest.raises(ValueError):
|
||||||
validator.build_verified_market_snapshot("COF", "2026-05-13")
|
validator.build_verified_market_snapshot("COF", "2026-05-13")
|
||||||
|
|
||||||
def test_look_back_window_capped_at_30(self, monkeypatch):
|
def test_look_back_window_capped_at_30(self, monkeypatch):
|
||||||
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: _sample_ohlcv())
|
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: _sample_ohlcv())
|
||||||
snap = validator.build_verified_market_snapshot("COF", "2026-05-20", look_back_days=999)
|
snap = validator.build_verified_market_snapshot("COF", "2026-05-20", look_back_days=999)
|
||||||
# last-N closes table has at most 30 data rows
|
# last-N closes table has at most 30 data rows
|
||||||
close_rows = [ln for ln in snap.splitlines() if ln.startswith("| 2026-")]
|
close_rows = [ln for ln in snap.splitlines() if ln.startswith("| 2026-")]
|
||||||
@@ -69,7 +69,7 @@ class TestTool:
|
|||||||
from tradingagents.agents.utils.market_data_validation_tools import (
|
from tradingagents.agents.utils.market_data_validation_tools import (
|
||||||
get_verified_market_snapshot,
|
get_verified_market_snapshot,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: _sample_ohlcv())
|
monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: _sample_ohlcv())
|
||||||
out = get_verified_market_snapshot.invoke(
|
out = get_verified_market_snapshot.invoke(
|
||||||
{"symbol": "COF", "curr_date": "2026-05-20"}
|
{"symbol": "COF", "curr_date": "2026-05-20"}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -167,3 +167,34 @@ def test_tz_aware_latest_bar_is_kept_at_the_cutoff(monkeypatch, tmp_path):
|
|||||||
out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
|
out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
|
||||||
assert out["Close"].iloc[-1] == 101.5
|
assert out["Close"].iloc[-1] == 101.5
|
||||||
assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-08")
|
assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-08")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.unit
|
||||||
|
def test_the_snapshot_does_not_present_a_filled_price_as_reported(monkeypatch, tmp_path):
|
||||||
|
"""Gap filling exists so indicators compute on a continuous series. The
|
||||||
|
verification snapshot is the one place a number must be what the vendor
|
||||||
|
reported, or the module built to stop invented prices supplies them."""
|
||||||
|
from tradingagents.dataflows import market_data_validator as mdv, stockstats_utils as su
|
||||||
|
|
||||||
|
frame = pd.DataFrame({
|
||||||
|
"Date": ["2026-05-06", "2026-05-07", "2026-05-08"],
|
||||||
|
"Open": [100.0, 104.5, ""], # the latest bar has not settled
|
||||||
|
"High": [101.0, 105.5, ""],
|
||||||
|
"Low": [99.0, 103.5, ""],
|
||||||
|
"Close": [100.5, 105.0, 106.0],
|
||||||
|
"Volume": [1000000, 1000000, ""],
|
||||||
|
})
|
||||||
|
today = pd.Timestamp("2026-05-08 12:00")
|
||||||
|
monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)})
|
||||||
|
monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today))
|
||||||
|
cache = tmp_path / "AAPL-YFin-data.csv"
|
||||||
|
cache.write_text(frame.to_csv(index=False))
|
||||||
|
os.utime(cache, (today.timestamp(), today.timestamp()))
|
||||||
|
monkeypatch.setattr(su.yf, "download", lambda *a, **k: (_ for _ in ()).throw(
|
||||||
|
AssertionError("should read the seeded cache")))
|
||||||
|
|
||||||
|
out = mdv.build_verified_market_snapshot("AAPL", "2026-05-08", 3)
|
||||||
|
|
||||||
|
row = out.split("Latest verified OHLCV row")[1].split("###")[0]
|
||||||
|
assert "104.50" not in row and "105.50" not in row # the previous session's numbers
|
||||||
|
assert "106.00" in row # the close the vendor did report
|
||||||
|
|||||||
@@ -32,7 +32,9 @@ def _verified_rows(symbol: str, curr_date: str) -> pd.DataFrame:
|
|||||||
look-ahead rows, but we re-apply the cutoff defensively — this is a
|
look-ahead rows, but we re-apply the cutoff defensively — this is a
|
||||||
verification path, so it must not trust its input to be pre-filtered.
|
verification path, so it must not trust its input to be pre-filtered.
|
||||||
"""
|
"""
|
||||||
data = load_ohlcv(symbol, curr_date)
|
# As reported: this snapshot is quoted by the agents as exact prices, so a
|
||||||
|
# gap-filled cell would put the previous session's number under this date.
|
||||||
|
data = load_ohlcv(symbol, curr_date, fill_gaps=False)
|
||||||
if data is None or data.empty:
|
if data is None or data.empty:
|
||||||
raise ValueError(f"No OHLCV data available for {symbol}.")
|
raise ValueError(f"No OHLCV data available for {symbol}.")
|
||||||
|
|
||||||
|
|||||||
@@ -179,12 +179,16 @@ def _cache_is_fresh(data_file, curr_date_dt, now) -> bool:
|
|||||||
return curr_date_dt.date() < now.date() or (now - written).total_seconds() <= OHLCV_CACHE_TTL_SECONDS
|
return curr_date_dt.date() < now.date() or (now - written).total_seconds() <= OHLCV_CACHE_TTL_SECONDS
|
||||||
|
|
||||||
|
|
||||||
def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:
|
def load_ohlcv(symbol: str, curr_date: str, fill_gaps: bool = True) -> pd.DataFrame:
|
||||||
"""Fetch OHLCV data with caching, filtered to prevent look-ahead bias.
|
"""Fetch OHLCV data with caching, filtered to prevent look-ahead bias.
|
||||||
|
|
||||||
Downloads 5 years of data up to today and caches per symbol. On
|
Downloads 5 years of data up to today and caches per symbol. On
|
||||||
subsequent calls the cache is reused. Rows after curr_date are
|
subsequent calls the cache is reused. Rows after curr_date are
|
||||||
filtered out so backtests never see future prices.
|
filtered out so backtests never see future prices.
|
||||||
|
|
||||||
|
``fill_gaps`` carries prices forward over gaps so indicators compute on a
|
||||||
|
continuous series. Pass ``False`` to read the values as the vendor reported
|
||||||
|
them, leaving a cell that was never reported empty.
|
||||||
"""
|
"""
|
||||||
# Resolve broker/forex symbols (XAUUSD+ -> GC=F) to Yahoo's convention,
|
# Resolve broker/forex symbols (XAUUSD+ -> GC=F) to Yahoo's convention,
|
||||||
# then reject values that would escape the cache directory when
|
# then reject values that would escape the cache directory when
|
||||||
@@ -262,7 +266,10 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:
|
|||||||
data["Date"].iloc[-1].date(), data["Date"].iloc[settled[-1]].date(),
|
data["Date"].iloc[-1].date(), data["Date"].iloc[settled[-1]].date(),
|
||||||
)
|
)
|
||||||
|
|
||||||
data = _fill_price_gaps(data)
|
# Indicators need a continuous series, so gaps are carried forward. A caller
|
||||||
|
# that reports the numbers themselves asks for the frame as it was reported:
|
||||||
|
# a filled cell is the previous session's price under this session's date.
|
||||||
|
data = _fill_price_gaps(data) if fill_gaps else data.dropna(subset=["Close"]).copy()
|
||||||
|
|
||||||
# Reject a stale frame (latest row far older than curr_date) rather than
|
# Reject a stale frame (latest row far older than curr_date) rather than
|
||||||
# feeding year-old prices into indicators (#1021).
|
# feeding year-old prices into indicators (#1021).
|
||||||
|
|||||||
Reference in New Issue
Block a user