From de7e43fc4a94682a40a06cb21967a6dc865eb689 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Thu, 17 Sep 2026 23:37:43 +0000 Subject: [PATCH] fix(dataflows): quote the prices the vendor reported in the verification snapshot - gap filling keeps indicators on a continuous series, but put the previous session's open, high and low under an unsettled bar's date - load_ohlcv takes fill_gaps, and the snapshot reads the frame as reported --- tests/test_market_data_validator.py | 12 +++---- tests/test_ohlcv_latest_bar.py | 31 +++++++++++++++++++ .../dataflows/market_data_validator.py | 4 ++- tradingagents/dataflows/stockstats_utils.py | 11 +++++-- 4 files changed, 49 insertions(+), 9 deletions(-) diff --git a/tests/test_market_data_validator.py b/tests/test_market_data_validator.py index 40b6349e2..c26d6f537 100644 --- a/tests/test_market_data_validator.py +++ b/tests/test_market_data_validator.py @@ -29,7 +29,7 @@ class TestVerifiedSnapshot: pd.DataFrame({"Date": [pd.Timestamp("2026-06-01")], "Open": [999.0], "High": [999.0], "Low": [999.0], "Close": [999.0], "Volume": [999]}), ], ignore_index=True) - monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: data) + monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: data) snap = validator.build_verified_market_snapshot("COF", "2026-05-13") assert "Verified market data snapshot for COF" in snap @@ -39,24 +39,24 @@ class TestVerifiedSnapshot: assert "boll_lb" in snap # indicators present def test_uses_previous_trading_day_when_date_is_weekend(self, monkeypatch): - monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: _sample_ohlcv()) + monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: _sample_ohlcv()) # 2026-05-16 is a Saturday; latest row should be Fri 2026-05-15 snap = validator.build_verified_market_snapshot("COF", "2026-05-16") assert "Latest trading row used: 2026-05-15" in snap assert "Recent verified closes" in snap def test_raises_when_no_rows_on_or_before_date(self, monkeypatch): - monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: _sample_ohlcv()) + monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: _sample_ohlcv()) with pytest.raises(ValueError): validator.build_verified_market_snapshot("COF", "2020-01-01") def test_raises_on_empty_data(self, monkeypatch): - monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: pd.DataFrame()) + monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: pd.DataFrame()) with pytest.raises(ValueError): validator.build_verified_market_snapshot("COF", "2026-05-13") def test_look_back_window_capped_at_30(self, monkeypatch): - monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: _sample_ohlcv()) + monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: _sample_ohlcv()) snap = validator.build_verified_market_snapshot("COF", "2026-05-20", look_back_days=999) # last-N closes table has at most 30 data rows close_rows = [ln for ln in snap.splitlines() if ln.startswith("| 2026-")] @@ -69,7 +69,7 @@ class TestTool: from tradingagents.agents.utils.market_data_validation_tools import ( get_verified_market_snapshot, ) - monkeypatch.setattr(validator, "load_ohlcv", lambda s, d: _sample_ohlcv()) + monkeypatch.setattr(validator, "load_ohlcv", lambda s, d, fill_gaps=True: _sample_ohlcv()) out = get_verified_market_snapshot.invoke( {"symbol": "COF", "curr_date": "2026-05-20"} ) diff --git a/tests/test_ohlcv_latest_bar.py b/tests/test_ohlcv_latest_bar.py index 5483e2d13..548264cc5 100644 --- a/tests/test_ohlcv_latest_bar.py +++ b/tests/test_ohlcv_latest_bar.py @@ -167,3 +167,34 @@ def test_tz_aware_latest_bar_is_kept_at_the_cutoff(monkeypatch, tmp_path): out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08") assert out["Close"].iloc[-1] == 101.5 assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-08") + + +@pytest.mark.unit +def test_the_snapshot_does_not_present_a_filled_price_as_reported(monkeypatch, tmp_path): + """Gap filling exists so indicators compute on a continuous series. The + verification snapshot is the one place a number must be what the vendor + reported, or the module built to stop invented prices supplies them.""" + from tradingagents.dataflows import market_data_validator as mdv, stockstats_utils as su + + frame = pd.DataFrame({ + "Date": ["2026-05-06", "2026-05-07", "2026-05-08"], + "Open": [100.0, 104.5, ""], # the latest bar has not settled + "High": [101.0, 105.5, ""], + "Low": [99.0, 103.5, ""], + "Close": [100.5, 105.0, 106.0], + "Volume": [1000000, 1000000, ""], + }) + today = pd.Timestamp("2026-05-08 12:00") + monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) + monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today)) + cache = tmp_path / "AAPL-YFin-data.csv" + cache.write_text(frame.to_csv(index=False)) + os.utime(cache, (today.timestamp(), today.timestamp())) + monkeypatch.setattr(su.yf, "download", lambda *a, **k: (_ for _ in ()).throw( + AssertionError("should read the seeded cache"))) + + out = mdv.build_verified_market_snapshot("AAPL", "2026-05-08", 3) + + row = out.split("Latest verified OHLCV row")[1].split("###")[0] + assert "104.50" not in row and "105.50" not in row # the previous session's numbers + assert "106.00" in row # the close the vendor did report diff --git a/tradingagents/dataflows/market_data_validator.py b/tradingagents/dataflows/market_data_validator.py index baa5efecf..05c062457 100644 --- a/tradingagents/dataflows/market_data_validator.py +++ b/tradingagents/dataflows/market_data_validator.py @@ -32,7 +32,9 @@ def _verified_rows(symbol: str, curr_date: str) -> pd.DataFrame: look-ahead rows, but we re-apply the cutoff defensively — this is a verification path, so it must not trust its input to be pre-filtered. """ - data = load_ohlcv(symbol, curr_date) + # As reported: this snapshot is quoted by the agents as exact prices, so a + # gap-filled cell would put the previous session's number under this date. + data = load_ohlcv(symbol, curr_date, fill_gaps=False) if data is None or data.empty: raise ValueError(f"No OHLCV data available for {symbol}.") diff --git a/tradingagents/dataflows/stockstats_utils.py b/tradingagents/dataflows/stockstats_utils.py index 63d7b2ff1..54df1d4ad 100644 --- a/tradingagents/dataflows/stockstats_utils.py +++ b/tradingagents/dataflows/stockstats_utils.py @@ -179,12 +179,16 @@ def _cache_is_fresh(data_file, curr_date_dt, now) -> bool: return curr_date_dt.date() < now.date() or (now - written).total_seconds() <= OHLCV_CACHE_TTL_SECONDS -def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: +def load_ohlcv(symbol: str, curr_date: str, fill_gaps: bool = True) -> pd.DataFrame: """Fetch OHLCV data with caching, filtered to prevent look-ahead bias. Downloads 5 years of data up to today and caches per symbol. On subsequent calls the cache is reused. Rows after curr_date are filtered out so backtests never see future prices. + + ``fill_gaps`` carries prices forward over gaps so indicators compute on a + continuous series. Pass ``False`` to read the values as the vendor reported + them, leaving a cell that was never reported empty. """ # Resolve broker/forex symbols (XAUUSD+ -> GC=F) to Yahoo's convention, # then reject values that would escape the cache directory when @@ -262,7 +266,10 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: data["Date"].iloc[-1].date(), data["Date"].iloc[settled[-1]].date(), ) - data = _fill_price_gaps(data) + # Indicators need a continuous series, so gaps are carried forward. A caller + # that reports the numbers themselves asks for the frame as it was reported: + # a filled cell is the previous session's price under this session's date. + data = _fill_price_gaps(data) if fill_gaps else data.dropna(subset=["Close"]).copy() # Reject a stale frame (latest row far older than curr_date) rather than # feeding year-old prices into indicators (#1021).