fix(dataflows): don't report a symbol as unavailable over an unsettled bar

- a newest bar with no close made load_ohlcv reject the whole frame, so the
  routing layer answered with its no-data sentinel: the caller lost the entire
  price history and was told the symbol may be invalid, delisted or not
  covered, when only the latest session had not settled
- treat a closeless newest bar as an unsettled session instead. The gap fill
  already drops it, here and mid-series alike, so the frame ends at the last
  settled bar; only a range with no close anywhere is still no data
- the staleness check keeps deciding whether what remains is recent enough, so
  falling back cannot resurrect a long-dead series
- log which bars had no close and which date is being used as the latest close
This commit is contained in:
Yijia-Xiao
2026-09-07 21:42:25 +00:00
parent d58b838081
commit ef383df8f4
2 changed files with 54 additions and 12 deletions

View File

@@ -4,8 +4,12 @@ yfinance can return the newest in-range bar with a NaN close (an unsettled or
glitched session). The old path parsed dates without normalizing timezone and glitched session). The old path parsed dates without normalizing timezone and
dropped every NaN-close row before applying the curr_date cutoff, so the latest dropped every NaN-close row before applying the curr_date cutoff, so the latest
bar disappeared and the previous trading day looked like the latest. Now dates bar disappeared and the previous trading day looked like the latest. Now dates
are normalized, and a latest in-range bar with no close raises rather than are normalized before the cutoff, so the frame ends at the last settled bar
silently falling back. instead of carrying a fabricated close.
Refusing the whole frame instead (the first attempt at #1201) reported a
tradable symbol as invalid or delisted (#1289), so only a range with no close
anywhere counts as no data and the staleness check judges the rest.
""" """
from __future__ import annotations from __future__ import annotations
@@ -92,19 +96,46 @@ def _run_load(monkeypatch, tmp_path, frame, curr_date):
def _fail_download(*a, **k): def _fail_download(*a, **k):
raise AssertionError("should use the seeded cache, not download") raise AssertionError("should use the seeded cache, not download")
monkeypatch.setattr(su.yf, "download", _fail_download) monkeypatch.setattr(su.yf, "download", _fail_download)
monkeypatch.setattr(su, "_assert_ohlcv_not_stale", lambda *a, **k: None)
return su.load_ohlcv("AAPL", curr_date) return su.load_ohlcv("AAPL", curr_date)
@pytest.mark.unit @pytest.mark.unit
def test_latest_in_range_nan_close_raises_not_silent_fallback(monkeypatch, tmp_path): def test_unsettled_latest_bar_is_served_as_the_last_settled_bar(monkeypatch, tmp_path):
# Newest bar (the curr_date) has no close -> raise, don't return Thursday. # Newest bar (the curr_date) has no close: serve the last settled bar rather
# than reporting the whole symbol as unavailable (#1289).
frame = pd.DataFrame({ frame = pd.DataFrame({
"Date": ["2026-05-07", "2026-05-08"], "Date": ["2026-05-07", "2026-05-08"],
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0], "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
"Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000], "Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000],
}) })
with pytest.raises(NoMarketDataError, match="no closing price"): out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-07")
assert out["Close"].iloc[-1] == 100.5
@pytest.mark.unit
def test_no_settled_bar_at_all_is_still_no_data(monkeypatch, tmp_path):
frame = pd.DataFrame({
"Date": ["2026-05-07", "2026-05-08"],
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
"Close": [float("nan"), float("nan")], "Volume": [1_000_000, 1_000_000],
})
with pytest.raises(NoMarketDataError, match="no bar in range has a closing price"):
_run_load(monkeypatch, tmp_path, frame, "2026-05-08")
@pytest.mark.unit
def test_serving_the_last_settled_bar_does_not_bypass_the_staleness_check(
monkeypatch, tmp_path
):
# Falling back must not resurrect a long-dead series: once the closeless
# tail is gone, the remaining bar is judged on its age like any other.
frame = pd.DataFrame({
"Date": ["2026-01-05", "2026-05-08"],
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
"Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000],
})
with pytest.raises(NoMarketDataError, match="stale"):
_run_load(monkeypatch, tmp_path, frame, "2026-05-08") _run_load(monkeypatch, tmp_path, frame, "2026-05-08")

View File

@@ -250,13 +250,24 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:
# Filter to curr_date to prevent look-ahead bias in backtesting. # Filter to curr_date to prevent look-ahead bias in backtesting.
data = data[data["Date"] <= curr_date_dt] data = data[data["Date"] <= curr_date_dt]
# Guard the latest in-range bar before dropping incomplete rows: a newest bar # A newest bar with no close is usually an unsettled session — mid-session,
# with no close is "not settled yet", not "does not exist". Silently dropping # a holiday, or a thinly traded instrument — not a symbol without data.
# it would make the previous trading day look like the latest (#1201); raise # _fill_price_gaps below drops it, here and mid-series alike, so the frame
# instead so the router surfaces it rather than fabricating a fallback. # ends at the last settled bar rather than carrying a fabricated close
# (#1201). Refusing the whole frame instead reported a tradable symbol as
# invalid or delisted (#1289), so only a range with no close anywhere is
# treated as no data; the staleness check decides whether what remains is
# recent enough for curr_date.
if not data.empty and pd.isna(data["Close"].iloc[-1]): if not data.empty and pd.isna(data["Close"].iloc[-1]):
settled = data["Close"].notna().to_numpy().nonzero()[0]
if settled.size == 0:
raise NoMarketDataError( raise NoMarketDataError(
symbol, canonical, "latest in-range OHLCV bar has no closing price" symbol, canonical, "no bar in range has a closing price"
)
logger.warning(
"%s: %d trailing bar(s) through %s have no closing price; using %s "
"as the latest close.", canonical, len(data) - settled[-1] - 1,
data["Date"].iloc[-1].date(), data["Date"].iloc[settled[-1]].date(),
) )
data = _fill_price_gaps(data) data = _fill_price_gaps(data)