diff --git a/tests/test_ohlcv_latest_bar.py b/tests/test_ohlcv_latest_bar.py new file mode 100644 index 000000000..7a26b0e00 --- /dev/null +++ b/tests/test_ohlcv_latest_bar.py @@ -0,0 +1,136 @@ +"""The latest trading day's bar must not silently vanish (#1201). + +yfinance can return the newest in-range bar with a NaN close (an unsettled or +glitched session). The old path parsed dates without normalizing timezone and +dropped every NaN-close row before applying the curr_date cutoff, so the latest +bar disappeared and the previous trading day looked like the latest. Now dates +are normalized, and a latest in-range bar with no close raises rather than +silently falling back. +""" +from __future__ import annotations + +import pandas as pd +import pytest + +from tradingagents.dataflows import stockstats_utils as su +from tradingagents.dataflows.symbol_utils import NoMarketDataError + +# --- date normalization ----------------------------------------------------- + +@pytest.mark.unit +def test_normalize_dates_strips_tz_and_normalizes_to_midnight(): + aware = pd.Series(pd.to_datetime( + ["2026-05-08 09:30:00-04:00", "2026-05-09 16:00:00-04:00"] + )) + out = su._normalize_dates(aware) + assert out.dt.tz is None + assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")] + + +@pytest.mark.unit +def test_normalize_dates_leaves_naive_dates_at_midnight(): + naive = pd.Series(pd.to_datetime(["2026-05-08 14:30:00", "2026-05-09 00:00:00"])) + out = su._normalize_dates(naive) + assert out.dt.tz is None + assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")] + + +@pytest.mark.unit +def test_normalize_dates_handles_mixed_dst_offsets(): + # 5y of US bars span DST; via a cache CSV they arrive as mixed-offset + # strings, which pd.to_datetime can't unify. Each keeps its own local date. + mixed = pd.Series([ + "2026-01-08 00:00:00-05:00", # EST + "2026-06-08 00:00:00-04:00", # EDT + "not-a-date", # -> NaT + ]) + out = su._normalize_dates(mixed) + assert out.iloc[0] == pd.Timestamp("2026-01-08") + assert out.iloc[1] == pd.Timestamp("2026-06-08") + assert pd.isna(out.iloc[2]) + + +@pytest.mark.unit +def test_normalize_dates_keeps_positive_offset_local_date(): + # A Tokyo bar at local midnight (+09:00) must stay on its own calendar day, + # not shift to the previous UTC day (which utc=True parsing would cause). + jst = pd.Series(["2026-05-08 00:00:00+09:00"]) + assert su._normalize_dates(jst).iloc[0] == pd.Timestamp("2026-05-08") + + +# --- fill vs guard responsibilities ---------------------------------------- + +@pytest.mark.unit +def test_clean_dataframe_keeps_nan_close_for_the_caller_to_inspect(): + # _clean_dataframe normalizes but no longer drops the NaN close itself. + df = pd.DataFrame({"Date": ["2026-05-08", "2026-05-09"], "Close": [100.0, float("nan")]}) + cleaned = su._clean_dataframe(df) + assert len(cleaned) == 2 + assert pd.isna(cleaned["Close"].iloc[-1]) + + +@pytest.mark.unit +def test_fill_price_gaps_drops_nan_close_rows(): + df = pd.DataFrame({"Date": pd.to_datetime(["2026-05-07", "2026-05-08"]), + "Close": [float("nan"), 100.0]}) + filled = su._fill_price_gaps(df) + assert len(filled) == 1 + assert filled["Close"].iloc[0] == 100.0 + + +# --- load_ohlcv end-to-end (with a mocked cache read) ----------------------- + +def _run_load(monkeypatch, tmp_path, frame, curr_date): + """Drive load_ohlcv against a pre-seeded cache frame (no network).""" + monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) + today = pd.Timestamp(curr_date) + monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today)) + start = (today - pd.DateOffset(years=5)).strftime("%Y-%m-%d") + end = (today + pd.Timedelta(days=1)).strftime("%Y-%m-%d") + (tmp_path / f"AAPL-YFin-data-{start}-{end}.csv").write_text(frame.to_csv(index=False)) + + def _fail_download(*a, **k): + raise AssertionError("should use the seeded cache, not download") + monkeypatch.setattr(su.yf, "download", _fail_download) + monkeypatch.setattr(su, "_assert_ohlcv_not_stale", lambda *a, **k: None) + return su.load_ohlcv("AAPL", curr_date) + + +@pytest.mark.unit +def test_latest_in_range_nan_close_raises_not_silent_fallback(monkeypatch, tmp_path): + # Newest bar (the curr_date) has no close -> raise, don't return Thursday. + frame = pd.DataFrame({ + "Date": ["2026-05-07", "2026-05-08"], + "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0], + "Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000], + }) + with pytest.raises(NoMarketDataError, match="no closing price"): + _run_load(monkeypatch, tmp_path, frame, "2026-05-08") + + +@pytest.mark.unit +def test_older_nan_close_row_is_still_dropped(monkeypatch, tmp_path): + # A stale gap mid-series is dropped; the valid latest bar is served. + frame = pd.DataFrame({ + "Date": ["2026-05-06", "2026-05-07", "2026-05-08"], + "Open": [100.0, 101.0, 102.0], "High": [101.0, 102.0, 103.0], + "Low": [99.0, 100.0, 101.0], + "Close": [100.5, float("nan"), 102.5], "Volume": [1_000_000, 1_000_000, 1_000_000], + }) + out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08") + assert out["Close"].iloc[-1] == 102.5 + assert (out["Date"] == pd.Timestamp("2026-05-07")).sum() == 0 # the NaN row is gone + + +@pytest.mark.unit +def test_tz_aware_latest_bar_is_kept_at_the_cutoff(monkeypatch, tmp_path): + # A tz-aware/intraday latest bar on the cutoff day must not be filtered out + # by a naive-vs-aware comparison. + frame = pd.DataFrame({ + "Date": ["2026-05-07 09:30:00-04:00", "2026-05-08 09:30:00-04:00"], + "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0], + "Close": [100.5, 101.5], "Volume": [1_000_000, 1_000_000], + }) + out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08") + assert out["Close"].iloc[-1] == 101.5 + assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-08") diff --git a/tradingagents/dataflows/stockstats_utils.py b/tradingagents/dataflows/stockstats_utils.py index 4ffe80422..94ba07ea8 100644 --- a/tradingagents/dataflows/stockstats_utils.py +++ b/tradingagents/dataflows/stockstats_utils.py @@ -60,17 +60,53 @@ def _ensure_date_column(data: pd.DataFrame) -> pd.DataFrame: return data +def _local_midnight(value) -> pd.Timestamp: + """A single timestamp as its naive, midnight-normalized local date (or NaT).""" + if pd.isna(value): + return pd.NaT + try: + ts = pd.Timestamp(value) + except (ValueError, TypeError): + return pd.NaT + if ts.tzinfo is not None: + ts = ts.tz_localize(None) # drop tz, keep the local wall-clock date + return ts.normalize() + + +def _normalize_dates(dates) -> pd.Series: + """Parse to naive, midnight-normalized dates so tz-aware or intraday + timestamps compare correctly against the naive ``curr_date`` cutoff (#1201). + + Normalized per element: 5 years of yfinance bars span daylight-saving + changes (and cache CSVs round-trip the offsets as strings), so the series can + carry mixed UTC offsets that ``pd.to_datetime`` cannot unify without + ``utc=True`` — which would shift non-US (positive-offset) markets to the + previous day. Keeping each bar's own local date avoids both. + """ + return pd.to_datetime(pd.Series(dates).map(_local_midnight)) + + def _clean_dataframe(data: pd.DataFrame) -> pd.DataFrame: - """Normalize a stock DataFrame for stockstats: parse dates, drop invalid rows, fill price gaps.""" + """Normalize a stock DataFrame for stockstats: parse/normalize dates and + coerce prices to numeric (NaN where invalid). Dropping incomplete rows and + filling gaps is left to ``_fill_price_gaps`` so the caller can first inspect + the latest in-range bar (#1201).""" data = _ensure_date_column(data) - data["Date"] = pd.to_datetime(data["Date"], errors="coerce") + data["Date"] = _normalize_dates(data["Date"]) data = data.dropna(subset=["Date"]) price_cols = [c for c in ["Open", "High", "Low", "Close", "Volume"] if c in data.columns] data[price_cols] = data[price_cols].apply(pd.to_numeric, errors="coerce") - data = data.dropna(subset=["Close"]) - data[price_cols] = data[price_cols].ffill().bfill() + return data + +def _fill_price_gaps(data: pd.DataFrame) -> pd.DataFrame: + """Drop rows with no close and forward/back-fill remaining price gaps so + indicators compute on a continuous series.""" + price_cols = [c for c in ["Open", "High", "Low", "Close", "Volume"] if c in data.columns] + # copy() so a filtered (sliced) input is written to safely, not via a view. + data = data.dropna(subset=["Close"]).copy() + data[price_cols] = data[price_cols].ffill().bfill() return data @@ -159,7 +195,7 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: safe_symbol = safe_ticker_component(canonical) config = get_config() - curr_date_dt = pd.to_datetime(curr_date) + curr_date_dt = pd.to_datetime(curr_date).normalize() # Cache uses a fixed window (5y to today) so one file per symbol. today_date = pd.Timestamp.today() @@ -211,9 +247,20 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: data = _clean_dataframe(data) - # Filter to curr_date to prevent look-ahead bias in backtesting + # Filter to curr_date to prevent look-ahead bias in backtesting. data = data[data["Date"] <= curr_date_dt] + # Guard the latest in-range bar before dropping incomplete rows: a newest bar + # with no close is "not settled yet", not "does not exist". Silently dropping + # it would make the previous trading day look like the latest (#1201); raise + # instead so the router surfaces it rather than fabricating a fallback. + if not data.empty and pd.isna(data["Close"].iloc[-1]): + raise NoMarketDataError( + symbol, canonical, "latest in-range OHLCV bar has no closing price" + ) + + data = _fill_price_gaps(data) + # Reject a stale frame (latest row far older than curr_date) rather than # feeding year-old prices into indicators (#1021). _assert_ohlcv_not_stale(data, curr_date, symbol, canonical)