From ef383df8f40e010fb7f43a065c29688a9ed4964e Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Mon, 7 Sep 2026 21:42:25 +0000 Subject: [PATCH] fix(dataflows): don't report a symbol as unavailable over an unsettled bar - a newest bar with no close made load_ohlcv reject the whole frame, so the routing layer answered with its no-data sentinel: the caller lost the entire price history and was told the symbol may be invalid, delisted or not covered, when only the latest session had not settled - treat a closeless newest bar as an unsettled session instead. The gap fill already drops it, here and mid-series alike, so the frame ends at the last settled bar; only a range with no close anywhere is still no data - the staleness check keeps deciding whether what remains is recent enough, so falling back cannot resurrect a long-dead series - log which bars had no close and which date is being used as the latest close --- tests/test_ohlcv_latest_bar.py | 43 ++++++++++++++++++--- tradingagents/dataflows/stockstats_utils.py | 23 ++++++++--- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/tests/test_ohlcv_latest_bar.py b/tests/test_ohlcv_latest_bar.py index 7a26b0e00..cf9bca676 100644 --- a/tests/test_ohlcv_latest_bar.py +++ b/tests/test_ohlcv_latest_bar.py @@ -4,8 +4,12 @@ yfinance can return the newest in-range bar with a NaN close (an unsettled or glitched session). The old path parsed dates without normalizing timezone and dropped every NaN-close row before applying the curr_date cutoff, so the latest bar disappeared and the previous trading day looked like the latest. Now dates -are normalized, and a latest in-range bar with no close raises rather than -silently falling back. +are normalized before the cutoff, so the frame ends at the last settled bar +instead of carrying a fabricated close. + +Refusing the whole frame instead (the first attempt at #1201) reported a +tradable symbol as invalid or delisted (#1289), so only a range with no close +anywhere counts as no data and the staleness check judges the rest. """ from __future__ import annotations @@ -92,19 +96,46 @@ def _run_load(monkeypatch, tmp_path, frame, curr_date): def _fail_download(*a, **k): raise AssertionError("should use the seeded cache, not download") monkeypatch.setattr(su.yf, "download", _fail_download) - monkeypatch.setattr(su, "_assert_ohlcv_not_stale", lambda *a, **k: None) return su.load_ohlcv("AAPL", curr_date) @pytest.mark.unit -def test_latest_in_range_nan_close_raises_not_silent_fallback(monkeypatch, tmp_path): - # Newest bar (the curr_date) has no close -> raise, don't return Thursday. +def test_unsettled_latest_bar_is_served_as_the_last_settled_bar(monkeypatch, tmp_path): + # Newest bar (the curr_date) has no close: serve the last settled bar rather + # than reporting the whole symbol as unavailable (#1289). frame = pd.DataFrame({ "Date": ["2026-05-07", "2026-05-08"], "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0], "Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000], }) - with pytest.raises(NoMarketDataError, match="no closing price"): + out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08") + assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-07") + assert out["Close"].iloc[-1] == 100.5 + + +@pytest.mark.unit +def test_no_settled_bar_at_all_is_still_no_data(monkeypatch, tmp_path): + frame = pd.DataFrame({ + "Date": ["2026-05-07", "2026-05-08"], + "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0], + "Close": [float("nan"), float("nan")], "Volume": [1_000_000, 1_000_000], + }) + with pytest.raises(NoMarketDataError, match="no bar in range has a closing price"): + _run_load(monkeypatch, tmp_path, frame, "2026-05-08") + + +@pytest.mark.unit +def test_serving_the_last_settled_bar_does_not_bypass_the_staleness_check( + monkeypatch, tmp_path +): + # Falling back must not resurrect a long-dead series: once the closeless + # tail is gone, the remaining bar is judged on its age like any other. + frame = pd.DataFrame({ + "Date": ["2026-01-05", "2026-05-08"], + "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0], + "Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000], + }) + with pytest.raises(NoMarketDataError, match="stale"): _run_load(monkeypatch, tmp_path, frame, "2026-05-08") diff --git a/tradingagents/dataflows/stockstats_utils.py b/tradingagents/dataflows/stockstats_utils.py index 94ba07ea8..b29612ece 100644 --- a/tradingagents/dataflows/stockstats_utils.py +++ b/tradingagents/dataflows/stockstats_utils.py @@ -250,13 +250,24 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: # Filter to curr_date to prevent look-ahead bias in backtesting. data = data[data["Date"] <= curr_date_dt] - # Guard the latest in-range bar before dropping incomplete rows: a newest bar - # with no close is "not settled yet", not "does not exist". Silently dropping - # it would make the previous trading day look like the latest (#1201); raise - # instead so the router surfaces it rather than fabricating a fallback. + # A newest bar with no close is usually an unsettled session — mid-session, + # a holiday, or a thinly traded instrument — not a symbol without data. + # _fill_price_gaps below drops it, here and mid-series alike, so the frame + # ends at the last settled bar rather than carrying a fabricated close + # (#1201). Refusing the whole frame instead reported a tradable symbol as + # invalid or delisted (#1289), so only a range with no close anywhere is + # treated as no data; the staleness check decides whether what remains is + # recent enough for curr_date. if not data.empty and pd.isna(data["Close"].iloc[-1]): - raise NoMarketDataError( - symbol, canonical, "latest in-range OHLCV bar has no closing price" + settled = data["Close"].notna().to_numpy().nonzero()[0] + if settled.size == 0: + raise NoMarketDataError( + symbol, canonical, "no bar in range has a closing price" + ) + logger.warning( + "%s: %d trailing bar(s) through %s have no closing price; using %s " + "as the latest close.", canonical, len(data) - settled[-1] - 1, + data["Date"].iloc[-1].date(), data["Date"].iloc[settled[-1]].date(), ) data = _fill_price_gaps(data)