mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-19 19:25:24 +03:00
ef383df8f4
- a newest bar with no close made load_ohlcv reject the whole frame, so the routing layer answered with its no-data sentinel: the caller lost the entire price history and was told the symbol may be invalid, delisted or not covered, when only the latest session had not settled - treat a closeless newest bar as an unsettled session instead. The gap fill already drops it, here and mid-series alike, so the frame ends at the last settled bar; only a range with no close anywhere is still no data - the staleness check keeps deciding whether what remains is recent enough, so falling back cannot resurrect a long-dead series - log which bars had no close and which date is being used as the latest close
168 lines
7.1 KiB
Python
168 lines
7.1 KiB
Python
"""The latest trading day's bar must not silently vanish (#1201).
|
|
|
|
yfinance can return the newest in-range bar with a NaN close (an unsettled or
|
|
glitched session). The old path parsed dates without normalizing timezone and
|
|
dropped every NaN-close row before applying the curr_date cutoff, so the latest
|
|
bar disappeared and the previous trading day looked like the latest. Now dates
|
|
are normalized before the cutoff, so the frame ends at the last settled bar
|
|
instead of carrying a fabricated close.
|
|
|
|
Refusing the whole frame instead (the first attempt at #1201) reported a
|
|
tradable symbol as invalid or delisted (#1289), so only a range with no close
|
|
anywhere counts as no data and the staleness check judges the rest.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import pandas as pd
|
|
import pytest
|
|
|
|
from tradingagents.dataflows import stockstats_utils as su
|
|
from tradingagents.dataflows.symbol_utils import NoMarketDataError
|
|
|
|
# --- date normalization -----------------------------------------------------
|
|
|
|
@pytest.mark.unit
|
|
def test_normalize_dates_strips_tz_and_normalizes_to_midnight():
|
|
aware = pd.Series(pd.to_datetime(
|
|
["2026-05-08 09:30:00-04:00", "2026-05-09 16:00:00-04:00"]
|
|
))
|
|
out = su._normalize_dates(aware)
|
|
assert out.dt.tz is None
|
|
assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_normalize_dates_leaves_naive_dates_at_midnight():
|
|
naive = pd.Series(pd.to_datetime(["2026-05-08 14:30:00", "2026-05-09 00:00:00"]))
|
|
out = su._normalize_dates(naive)
|
|
assert out.dt.tz is None
|
|
assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")]
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_normalize_dates_handles_mixed_dst_offsets():
|
|
# 5y of US bars span DST; via a cache CSV they arrive as mixed-offset
|
|
# strings, which pd.to_datetime can't unify. Each keeps its own local date.
|
|
mixed = pd.Series([
|
|
"2026-01-08 00:00:00-05:00", # EST
|
|
"2026-06-08 00:00:00-04:00", # EDT
|
|
"not-a-date", # -> NaT
|
|
])
|
|
out = su._normalize_dates(mixed)
|
|
assert out.iloc[0] == pd.Timestamp("2026-01-08")
|
|
assert out.iloc[1] == pd.Timestamp("2026-06-08")
|
|
assert pd.isna(out.iloc[2])
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_normalize_dates_keeps_positive_offset_local_date():
|
|
# A Tokyo bar at local midnight (+09:00) must stay on its own calendar day,
|
|
# not shift to the previous UTC day (which utc=True parsing would cause).
|
|
jst = pd.Series(["2026-05-08 00:00:00+09:00"])
|
|
assert su._normalize_dates(jst).iloc[0] == pd.Timestamp("2026-05-08")
|
|
|
|
|
|
# --- fill vs guard responsibilities ----------------------------------------
|
|
|
|
@pytest.mark.unit
|
|
def test_clean_dataframe_keeps_nan_close_for_the_caller_to_inspect():
|
|
# _clean_dataframe normalizes but no longer drops the NaN close itself.
|
|
df = pd.DataFrame({"Date": ["2026-05-08", "2026-05-09"], "Close": [100.0, float("nan")]})
|
|
cleaned = su._clean_dataframe(df)
|
|
assert len(cleaned) == 2
|
|
assert pd.isna(cleaned["Close"].iloc[-1])
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_fill_price_gaps_drops_nan_close_rows():
|
|
df = pd.DataFrame({"Date": pd.to_datetime(["2026-05-07", "2026-05-08"]),
|
|
"Close": [float("nan"), 100.0]})
|
|
filled = su._fill_price_gaps(df)
|
|
assert len(filled) == 1
|
|
assert filled["Close"].iloc[0] == 100.0
|
|
|
|
|
|
# --- load_ohlcv end-to-end (with a mocked cache read) -----------------------
|
|
|
|
def _run_load(monkeypatch, tmp_path, frame, curr_date):
|
|
"""Drive load_ohlcv against a pre-seeded cache frame (no network)."""
|
|
monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)})
|
|
today = pd.Timestamp(curr_date)
|
|
monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today))
|
|
start = (today - pd.DateOffset(years=5)).strftime("%Y-%m-%d")
|
|
end = (today + pd.Timedelta(days=1)).strftime("%Y-%m-%d")
|
|
(tmp_path / f"AAPL-YFin-data-{start}-{end}.csv").write_text(frame.to_csv(index=False))
|
|
|
|
def _fail_download(*a, **k):
|
|
raise AssertionError("should use the seeded cache, not download")
|
|
monkeypatch.setattr(su.yf, "download", _fail_download)
|
|
return su.load_ohlcv("AAPL", curr_date)
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_unsettled_latest_bar_is_served_as_the_last_settled_bar(monkeypatch, tmp_path):
|
|
# Newest bar (the curr_date) has no close: serve the last settled bar rather
|
|
# than reporting the whole symbol as unavailable (#1289).
|
|
frame = pd.DataFrame({
|
|
"Date": ["2026-05-07", "2026-05-08"],
|
|
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
|
|
"Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000],
|
|
})
|
|
out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
|
|
assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-07")
|
|
assert out["Close"].iloc[-1] == 100.5
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_no_settled_bar_at_all_is_still_no_data(monkeypatch, tmp_path):
|
|
frame = pd.DataFrame({
|
|
"Date": ["2026-05-07", "2026-05-08"],
|
|
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
|
|
"Close": [float("nan"), float("nan")], "Volume": [1_000_000, 1_000_000],
|
|
})
|
|
with pytest.raises(NoMarketDataError, match="no bar in range has a closing price"):
|
|
_run_load(monkeypatch, tmp_path, frame, "2026-05-08")
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_serving_the_last_settled_bar_does_not_bypass_the_staleness_check(
|
|
monkeypatch, tmp_path
|
|
):
|
|
# Falling back must not resurrect a long-dead series: once the closeless
|
|
# tail is gone, the remaining bar is judged on its age like any other.
|
|
frame = pd.DataFrame({
|
|
"Date": ["2026-01-05", "2026-05-08"],
|
|
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
|
|
"Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000],
|
|
})
|
|
with pytest.raises(NoMarketDataError, match="stale"):
|
|
_run_load(monkeypatch, tmp_path, frame, "2026-05-08")
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_older_nan_close_row_is_still_dropped(monkeypatch, tmp_path):
|
|
# A stale gap mid-series is dropped; the valid latest bar is served.
|
|
frame = pd.DataFrame({
|
|
"Date": ["2026-05-06", "2026-05-07", "2026-05-08"],
|
|
"Open": [100.0, 101.0, 102.0], "High": [101.0, 102.0, 103.0],
|
|
"Low": [99.0, 100.0, 101.0],
|
|
"Close": [100.5, float("nan"), 102.5], "Volume": [1_000_000, 1_000_000, 1_000_000],
|
|
})
|
|
out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
|
|
assert out["Close"].iloc[-1] == 102.5
|
|
assert (out["Date"] == pd.Timestamp("2026-05-07")).sum() == 0 # the NaN row is gone
|
|
|
|
|
|
@pytest.mark.unit
|
|
def test_tz_aware_latest_bar_is_kept_at_the_cutoff(monkeypatch, tmp_path):
|
|
# A tz-aware/intraday latest bar on the cutoff day must not be filtered out
|
|
# by a naive-vs-aware comparison.
|
|
frame = pd.DataFrame({
|
|
"Date": ["2026-05-07 09:30:00-04:00", "2026-05-08 09:30:00-04:00"],
|
|
"Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
|
|
"Close": [100.5, 101.5], "Volume": [1_000_000, 1_000_000],
|
|
})
|
|
out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
|
|
assert out["Close"].iloc[-1] == 101.5
|
|
assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-08")
|