From d78c698d0eeebbe630d662721f30a99149e77ff8 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Sat, 18 Jul 2026 06:28:38 +0000 Subject: [PATCH] fix(dataflows): refresh the same-day OHLCV cache - the per-day cache was reused unconditionally, so a run started before the day's bar was final served that snapshot to every later run, feeding a stale close into technical analysis - a present row is not sufficient either, since Yahoo publishes a partial intraday candle; a TTL now governs every current-day cache while historical caches stay immutable #1150 --- tests/test_ohlcv_cache_freshness.py | 109 ++++++++++++++++++++ tradingagents/dataflows/stockstats_utils.py | 31 +++++- 2 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 tests/test_ohlcv_cache_freshness.py diff --git a/tests/test_ohlcv_cache_freshness.py b/tests/test_ohlcv_cache_freshness.py new file mode 100644 index 000000000..65a417f26 --- /dev/null +++ b/tests/test_ohlcv_cache_freshness.py @@ -0,0 +1,109 @@ +"""Same-day OHLCV cache must not serve a stale snapshot all day (#1150). + +The cache file is keyed per day, so a run started before the day's bar was final +would be reused by every later run, feeding a stale close into technical +analysis. Two cases matter for a current-day request: the bar may be missing, or +present but still in progress (Yahoo publishes a partial daily candle intraday). +Refresh is bounded by a TTL so repeated runs cannot hammer the vendor. +""" +from __future__ import annotations + +import os +import time + +import pandas as pd +import pytest + +import tradingagents.dataflows.stockstats_utils as su + +TODAY = pd.Timestamp("2026-07-18") +STALE = su.OHLCV_CACHE_TTL_SECONDS + 60 + + +def _write(tmp_path, name="cache.csv", age_seconds=0.0, last_date="2026-07-17"): + f = tmp_path / name + pd.DataFrame({"Date": [last_date], "Close": [1.0]}).to_csv(f, index=False) + if age_seconds: + old = time.time() - age_seconds + os.utime(f, (old, old)) + return str(f) + + +@pytest.mark.unit +def test_current_day_cache_past_ttl_is_refreshed(tmp_path): + # Bar missing (rows stop at yesterday) and file older than the TTL -> refetch. + assert su._needs_same_day_refresh(_write(tmp_path, age_seconds=STALE), TODAY, TODAY) is True + + +@pytest.mark.unit +def test_partial_current_day_bar_is_still_refreshed(tmp_path): + # Today's row is present but may be an in-progress candle whose Close is not + # the closing price. Row inspection can't distinguish it, so the TTL governs. + f = _write(tmp_path, age_seconds=STALE, last_date="2026-07-18") + assert su._needs_same_day_refresh(f, TODAY, TODAY) is True + + +@pytest.mark.unit +def test_recent_cache_is_not_refetched(tmp_path): + # Written moments ago: don't hammer the vendor (weekend/holiday guard). + assert su._needs_same_day_refresh(_write(tmp_path), TODAY, TODAY) is False + + +@pytest.mark.unit +def test_historical_request_always_uses_cache(tmp_path): + # Past dates are immutable: never refetch, however old the file is. + past = pd.Timestamp("2026-05-01") + f = _write(tmp_path, age_seconds=STALE, last_date="2026-04-30") + assert su._needs_same_day_refresh(f, past, TODAY) is False + + +@pytest.mark.unit +def test_load_ohlcv_refetches_stale_same_day_cache(tmp_path, monkeypatch): + """End-to-end: the helper is actually wired into load_ohlcv's cache branch. + + Without this, the unit tests above would still pass if the helper were never + called from the real code path. + """ + monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) + monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: TODAY)) + + # Pre-seed the cache file load_ohlcv will look for, aged past the TTL. + start = (TODAY - pd.DateOffset(years=5)).strftime("%Y-%m-%d") + end = (TODAY + pd.Timedelta(days=1)).strftime("%Y-%m-%d") + cache_file = tmp_path / f"AAPL-YFin-data-{start}-{end}.csv" + pd.DataFrame({"Date": ["2026-07-17"], "Close": [100.0]}).to_csv(cache_file, index=False) + old = time.time() - STALE + os.utime(cache_file, (old, old)) + + calls = [] + + def _fake_download(*a, **k): + calls.append(1) + return pd.DataFrame( + {"Date": pd.to_datetime(["2026-07-17", "2026-07-18"]), "Close": [100.0, 222.0]} + ).set_index("Date") + + monkeypatch.setattr(su.yf, "download", _fake_download) + + out = su.load_ohlcv("AAPL", TODAY.strftime("%Y-%m-%d")) + + assert calls, "stale same-day cache must trigger a refetch" + assert 222.0 in out["Close"].values, "refreshed close must reach the caller" + + +@pytest.mark.unit +def test_load_ohlcv_reuses_fresh_same_day_cache(tmp_path, monkeypatch): + # Mirror image: a fresh cache must NOT trigger a download. + monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) + monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: TODAY)) + + start = (TODAY - pd.DateOffset(years=5)).strftime("%Y-%m-%d") + end = (TODAY + pd.Timedelta(days=1)).strftime("%Y-%m-%d") + cache_file = tmp_path / f"AAPL-YFin-data-{start}-{end}.csv" + pd.DataFrame({"Date": ["2026-07-18"], "Close": [100.0]}).to_csv(cache_file, index=False) + + def _fail_download(*a, **k): + raise AssertionError("fresh cache must not refetch") + + monkeypatch.setattr(su.yf, "download", _fail_download) + su.load_ohlcv("AAPL", TODAY.strftime("%Y-%m-%d")) diff --git a/tradingagents/dataflows/stockstats_utils.py b/tradingagents/dataflows/stockstats_utils.py index db880b046..4ffe80422 100644 --- a/tradingagents/dataflows/stockstats_utils.py +++ b/tradingagents/dataflows/stockstats_utils.py @@ -19,6 +19,12 @@ logger = logging.getLogger(__name__) # enough to catch the year-old frames yfinance occasionally returns (#1021). MAX_OHLCV_STALE_DAYS = 10 +# How long a same-day cache that does not yet reach the requested day may be +# reused before it is refetched (#1150). Short enough that an intraday run picks +# up today's close soon after it publishes, long enough that a day with no bar +# at all (weekend, holiday) cannot trigger a download on every call. +OHLCV_CACHE_TTL_SECONDS = 900 + def yf_retry(func, max_retries=3, base_delay=2.0): """Execute a yfinance call with exponential backoff on rate limits. @@ -122,6 +128,23 @@ def _assert_ohlcv_not_stale( ) +def _needs_same_day_refresh(data_file, curr_date_dt, today_date) -> bool: + """Whether a cached frame must be refetched to reflect the requested day. + + The cache file is keyed per day, so without this a run started before the + day's bar was final keeps serving that snapshot to every later run (#1150). + Two distinct staleness cases exist for a current-day request: the bar may be + missing entirely, or present but still in progress — Yahoo publishes a + partial daily candle during market hours, whose ``Close`` is not the closing + price. Row inspection cannot tell a partial bar from a final one, so the TTL + governs every current-day cache. Historical requests always reuse the cache, + since those rows are immutable. + """ + if curr_date_dt.date() < today_date.date(): + return False + return time.time() - os.path.getmtime(data_file) > OHLCV_CACHE_TTL_SECONDS + + def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: """Fetch OHLCV data with caching, filtered to prevent look-ahead bias. @@ -159,7 +182,13 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: data = None if os.path.exists(data_file): cached = pd.read_csv(data_file, on_bad_lines="skip", encoding="utf-8") - if not cached.empty and "Close" in cached.columns: + # Serve the cache only when it is usable and not a stale snapshot of the + # day being requested (#1150); otherwise fall through and refetch. + if ( + not cached.empty + and "Close" in cached.columns + and not _needs_same_day_refresh(data_file, curr_date_dt, today_date) + ): data = cached if data is None: