fix(dataflows): don't report a symbol as unavailable over an unsettled bar

- a newest bar with no close made load_ohlcv reject the whole frame, so the
  routing layer answered with its no-data sentinel: the caller lost the entire
  price history and was told the symbol may be invalid, delisted or not
  covered, when only the latest session had not settled
- treat a closeless newest bar as an unsettled session instead. The gap fill
  already drops it, here and mid-series alike, so the frame ends at the last
  settled bar; only a range with no close anywhere is still no data
- the staleness check keeps deciding whether what remains is recent enough, so
  falling back cannot resurrect a long-dead series
- log which bars had no close and which date is being used as the latest close
This commit is contained in:
Yijia-Xiao
2026-09-07 21:42:25 +00:00
parent d58b838081
commit ef383df8f4
2 changed files with 54 additions and 12 deletions

View File

@@ -4,8 +4,12 @@ yfinance can return the newest in-range bar with a NaN close (an unsettled or
glitched session). The old path parsed dates without normalizing timezone and
dropped every NaN-close row before applying the curr_date cutoff, so the latest
bar disappeared and the previous trading day looked like the latest. Now dates
are normalized, and a latest in-range bar with no close raises rather than
silently falling back.
are normalized before the cutoff, so the frame ends at the last settled bar
instead of carrying a fabricated close.
Refusing the whole frame instead (the first attempt at #1201) reported a
tradable symbol as invalid or delisted (#1289), so only a range with no close
anywhere counts as no data and the staleness check judges the rest.
"""
from __future__ import annotations
@@ -92,19 +96,46 @@ def _run_load(monkeypatch, tmp_path, frame, curr_date):
def _fail_download(*a, **k):
raise AssertionError("should use the seeded cache, not download")
monkeypatch.setattr(su.yf, "download", _fail_download)
monkeypatch.setattr(su, "_assert_ohlcv_not_stale", lambda *a, **k: None)
return su.load_ohlcv("AAPL", curr_date)
@pytest.mark.unit
def test_latest_in_range_nan_close_raises_not_silent_fallback(monkeypatch, tmp_path):
# Newest bar (the curr_date) has no close -> raise, don't return Thursday.
def test_unsettled_latest_bar_is_served_as_the_last_settled_bar(monkeypatch, tmp_path):
# Newest bar (the curr_date) has no close: serve the last settled bar rather
# than reporting the whole symbol as unavailable (#1289).
frame = pd.DataFrame({
"Date": ["2026-05-07", "2026-05-08"],
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
"Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000],
})
with pytest.raises(NoMarketDataError, match="no closing price"):
out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-07")
assert out["Close"].iloc[-1] == 100.5
@pytest.mark.unit
def test_no_settled_bar_at_all_is_still_no_data(monkeypatch, tmp_path):
frame = pd.DataFrame({
"Date": ["2026-05-07", "2026-05-08"],
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
"Close": [float("nan"), float("nan")], "Volume": [1_000_000, 1_000_000],
})
with pytest.raises(NoMarketDataError, match="no bar in range has a closing price"):
_run_load(monkeypatch, tmp_path, frame, "2026-05-08")
@pytest.mark.unit
def test_serving_the_last_settled_bar_does_not_bypass_the_staleness_check(
monkeypatch, tmp_path
):
# Falling back must not resurrect a long-dead series: once the closeless
# tail is gone, the remaining bar is judged on its age like any other.
frame = pd.DataFrame({
"Date": ["2026-01-05", "2026-05-08"],
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
"Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000],
})
with pytest.raises(NoMarketDataError, match="stale"):
_run_load(monkeypatch, tmp_path, frame, "2026-05-08")

View File

@@ -250,13 +250,24 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:
# Filter to curr_date to prevent look-ahead bias in backtesting.
data = data[data["Date"] <= curr_date_dt]
# Guard the latest in-range bar before dropping incomplete rows: a newest bar
# with no close is "not settled yet", not "does not exist". Silently dropping
# it would make the previous trading day look like the latest (#1201); raise
# instead so the router surfaces it rather than fabricating a fallback.
# A newest bar with no close is usually an unsettled session — mid-session,
# a holiday, or a thinly traded instrument — not a symbol without data.
# _fill_price_gaps below drops it, here and mid-series alike, so the frame
# ends at the last settled bar rather than carrying a fabricated close
# (#1201). Refusing the whole frame instead reported a tradable symbol as
# invalid or delisted (#1289), so only a range with no close anywhere is
# treated as no data; the staleness check decides whether what remains is
# recent enough for curr_date.
if not data.empty and pd.isna(data["Close"].iloc[-1]):
raise NoMarketDataError(
symbol, canonical, "latest in-range OHLCV bar has no closing price"
settled = data["Close"].notna().to_numpy().nonzero()[0]
if settled.size == 0:
raise NoMarketDataError(
symbol, canonical, "no bar in range has a closing price"
)
logger.warning(
"%s: %d trailing bar(s) through %s have no closing price; using %s "
"as the latest close.", canonical, len(data) - settled[-1] - 1,
data["Date"].iloc[-1].date(), data["Date"].iloc[settled[-1]].date(),
)
data = _fill_price_gaps(data)