fix(dataflows): don't silently drop the latest OHLCV bar

- the latest in-range bar with a NaN close was dropped before the curr_date
  cutoff, so the previous trading day looked like the latest; dates were also
  compared without timezone normalization
- normalize bar dates and curr_date to naive midnight (per element, so 5-year
  ranges spanning DST and non-US positive-offset markets keep their local date),
  then raise NoMarketDataError on a missing latest close rather than falling back
- split the fill step (_fill_price_gaps) from date/price normalization so the
  latest bar can be inspected before incomplete rows are dropped #1201

tests/test_ohlcv_latest_bar.py

@@ -0,0 +1,136 @@
+"""The latest trading day's bar must not silently vanish (#1201).
+
+yfinance can return the newest in-range bar with a NaN close (an unsettled or
+glitched session). The old path parsed dates without normalizing timezone and
+dropped every NaN-close row before applying the curr_date cutoff, so the latest
+bar disappeared and the previous trading day looked like the latest. Now dates
+are normalized, and a latest in-range bar with no close raises rather than
+silently falling back.
+"""
+from __future__ import annotations
+
+import pandas as pd
+import pytest
+
+from tradingagents.dataflows import stockstats_utils as su
+from tradingagents.dataflows.symbol_utils import NoMarketDataError
+
+# --- date normalization -----------------------------------------------------
+
+@pytest.mark.unit
+def test_normalize_dates_strips_tz_and_normalizes_to_midnight():
+    aware = pd.Series(pd.to_datetime(
+        ["2026-05-08 09:30:00-04:00", "2026-05-09 16:00:00-04:00"]
+    ))
+    out = su._normalize_dates(aware)
+    assert out.dt.tz is None
+    assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")]
+
+
+@pytest.mark.unit
+def test_normalize_dates_leaves_naive_dates_at_midnight():
+    naive = pd.Series(pd.to_datetime(["2026-05-08 14:30:00", "2026-05-09 00:00:00"]))
+    out = su._normalize_dates(naive)
+    assert out.dt.tz is None
+    assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")]
+
+
+@pytest.mark.unit
+def test_normalize_dates_handles_mixed_dst_offsets():
+    # 5y of US bars span DST; via a cache CSV they arrive as mixed-offset
+    # strings, which pd.to_datetime can't unify. Each keeps its own local date.
+    mixed = pd.Series([
+        "2026-01-08 00:00:00-05:00",  # EST
+        "2026-06-08 00:00:00-04:00",  # EDT
+        "not-a-date",                 # -> NaT
+    ])
+    out = su._normalize_dates(mixed)
+    assert out.iloc[0] == pd.Timestamp("2026-01-08")
+    assert out.iloc[1] == pd.Timestamp("2026-06-08")
+    assert pd.isna(out.iloc[2])
+
+
+@pytest.mark.unit
+def test_normalize_dates_keeps_positive_offset_local_date():
+    # A Tokyo bar at local midnight (+09:00) must stay on its own calendar day,
+    # not shift to the previous UTC day (which utc=True parsing would cause).
+    jst = pd.Series(["2026-05-08 00:00:00+09:00"])
+    assert su._normalize_dates(jst).iloc[0] == pd.Timestamp("2026-05-08")
+
+
+# --- fill vs guard responsibilities ----------------------------------------
+
+@pytest.mark.unit
+def test_clean_dataframe_keeps_nan_close_for_the_caller_to_inspect():
+    # _clean_dataframe normalizes but no longer drops the NaN close itself.
+    df = pd.DataFrame({"Date": ["2026-05-08", "2026-05-09"], "Close": [100.0, float("nan")]})
+    cleaned = su._clean_dataframe(df)
+    assert len(cleaned) == 2
+    assert pd.isna(cleaned["Close"].iloc[-1])
+
+
+@pytest.mark.unit
+def test_fill_price_gaps_drops_nan_close_rows():
+    df = pd.DataFrame({"Date": pd.to_datetime(["2026-05-07", "2026-05-08"]),
+                       "Close": [float("nan"), 100.0]})
+    filled = su._fill_price_gaps(df)
+    assert len(filled) == 1
+    assert filled["Close"].iloc[0] == 100.0
+
+
+# --- load_ohlcv end-to-end (with a mocked cache read) -----------------------
+
+def _run_load(monkeypatch, tmp_path, frame, curr_date):
+    """Drive load_ohlcv against a pre-seeded cache frame (no network)."""
+    monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)})
+    today = pd.Timestamp(curr_date)
+    monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today))
+    start = (today - pd.DateOffset(years=5)).strftime("%Y-%m-%d")
+    end = (today + pd.Timedelta(days=1)).strftime("%Y-%m-%d")
+    (tmp_path / f"AAPL-YFin-data-{start}-{end}.csv").write_text(frame.to_csv(index=False))
+
+    def _fail_download(*a, **k):
+        raise AssertionError("should use the seeded cache, not download")
+    monkeypatch.setattr(su.yf, "download", _fail_download)
+    monkeypatch.setattr(su, "_assert_ohlcv_not_stale", lambda *a, **k: None)
+    return su.load_ohlcv("AAPL", curr_date)
+
+
+@pytest.mark.unit
+def test_latest_in_range_nan_close_raises_not_silent_fallback(monkeypatch, tmp_path):
+    # Newest bar (the curr_date) has no close -> raise, don't return Thursday.
+    frame = pd.DataFrame({
+        "Date": ["2026-05-07", "2026-05-08"],
+        "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
+        "Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000],
+    })
+    with pytest.raises(NoMarketDataError, match="no closing price"):
+        _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
+
+
+@pytest.mark.unit
+def test_older_nan_close_row_is_still_dropped(monkeypatch, tmp_path):
+    # A stale gap mid-series is dropped; the valid latest bar is served.
+    frame = pd.DataFrame({
+        "Date": ["2026-05-06", "2026-05-07", "2026-05-08"],
+        "Open": [100.0, 101.0, 102.0], "High": [101.0, 102.0, 103.0],
+        "Low": [99.0, 100.0, 101.0],
+        "Close": [100.5, float("nan"), 102.5], "Volume": [1_000_000, 1_000_000, 1_000_000],
+    })
+    out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
+    assert out["Close"].iloc[-1] == 102.5
+    assert (out["Date"] == pd.Timestamp("2026-05-07")).sum() == 0  # the NaN row is gone
+
+
+@pytest.mark.unit
+def test_tz_aware_latest_bar_is_kept_at_the_cutoff(monkeypatch, tmp_path):
+    # A tz-aware/intraday latest bar on the cutoff day must not be filtered out
+    # by a naive-vs-aware comparison.
+    frame = pd.DataFrame({
+        "Date": ["2026-05-07 09:30:00-04:00", "2026-05-08 09:30:00-04:00"],
+        "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
+        "Close": [100.5, 101.5], "Volume": [1_000_000, 1_000_000],
+    })
+    out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
+    assert out["Close"].iloc[-1] == 101.5
+    assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-08")

tradingagents/dataflows/stockstats_utils.py

@@ -60,17 +60,53 @@ def _ensure_date_column(data: pd.DataFrame) -> pd.DataFrame:
     return data
 
 
+def _local_midnight(value) -> pd.Timestamp:
+    """A single timestamp as its naive, midnight-normalized local date (or NaT)."""
+    if pd.isna(value):
+        return pd.NaT
+    try:
+        ts = pd.Timestamp(value)
+    except (ValueError, TypeError):
+        return pd.NaT
+    if ts.tzinfo is not None:
+        ts = ts.tz_localize(None)  # drop tz, keep the local wall-clock date
+    return ts.normalize()
+
+
+def _normalize_dates(dates) -> pd.Series:
+    """Parse to naive, midnight-normalized dates so tz-aware or intraday
+    timestamps compare correctly against the naive ``curr_date`` cutoff (#1201).
+
+    Normalized per element: 5 years of yfinance bars span daylight-saving
+    changes (and cache CSVs round-trip the offsets as strings), so the series can
+    carry mixed UTC offsets that ``pd.to_datetime`` cannot unify without
+    ``utc=True`` — which would shift non-US (positive-offset) markets to the
+    previous day. Keeping each bar's own local date avoids both.
+    """
+    return pd.to_datetime(pd.Series(dates).map(_local_midnight))
+
+
 def _clean_dataframe(data: pd.DataFrame) -> pd.DataFrame:
-    """Normalize a stock DataFrame for stockstats: parse dates, drop invalid rows, fill price gaps."""
+    """Normalize a stock DataFrame for stockstats: parse/normalize dates and
+    coerce prices to numeric (NaN where invalid). Dropping incomplete rows and
+    filling gaps is left to ``_fill_price_gaps`` so the caller can first inspect
+    the latest in-range bar (#1201)."""
     data = _ensure_date_column(data)
-    data["Date"] = pd.to_datetime(data["Date"], errors="coerce")
+    data["Date"] = _normalize_dates(data["Date"])
     data = data.dropna(subset=["Date"])
 
     price_cols = [c for c in ["Open", "High", "Low", "Close", "Volume"] if c in data.columns]
     data[price_cols] = data[price_cols].apply(pd.to_numeric, errors="coerce")
-    data = data.dropna(subset=["Close"])
-    data[price_cols] = data[price_cols].ffill().bfill()
+    return data
 
+
+def _fill_price_gaps(data: pd.DataFrame) -> pd.DataFrame:
+    """Drop rows with no close and forward/back-fill remaining price gaps so
+    indicators compute on a continuous series."""
+    price_cols = [c for c in ["Open", "High", "Low", "Close", "Volume"] if c in data.columns]
+    # copy() so a filtered (sliced) input is written to safely, not via a view.
+    data = data.dropna(subset=["Close"]).copy()
+    data[price_cols] = data[price_cols].ffill().bfill()
     return data
 
 
@@ -159,7 +195,7 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:
     safe_symbol = safe_ticker_component(canonical)
 
     config = get_config()
-    curr_date_dt = pd.to_datetime(curr_date)
+    curr_date_dt = pd.to_datetime(curr_date).normalize()
 
     # Cache uses a fixed window (5y to today) so one file per symbol.
     today_date = pd.Timestamp.today()
@@ -211,9 +247,20 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:
 
     data = _clean_dataframe(data)
 
-    # Filter to curr_date to prevent look-ahead bias in backtesting
+    # Filter to curr_date to prevent look-ahead bias in backtesting.
     data = data[data["Date"] <= curr_date_dt]
 
+    # Guard the latest in-range bar before dropping incomplete rows: a newest bar
+    # with no close is "not settled yet", not "does not exist". Silently dropping
+    # it would make the previous trading day look like the latest (#1201); raise
+    # instead so the router surfaces it rather than fabricating a fallback.
+    if not data.empty and pd.isna(data["Close"].iloc[-1]):
+        raise NoMarketDataError(
+            symbol, canonical, "latest in-range OHLCV bar has no closing price"
+        )
+
+    data = _fill_price_gaps(data)
+
     # Reject a stale frame (latest row far older than curr_date) rather than
     # feeding year-old prices into indicators (#1021).
     _assert_ohlcv_not_stale(data, curr_date, symbol, canonical)