From 29e331a9afe40521d9e2a3d1f0b8f47247b923d3 Mon Sep 17 00:00:00 2001 From: Yijia-Xiao Date: Mon, 14 Sep 2026 23:51:48 +0000 Subject: [PATCH] fix(dataflows): keep one OHLCV cache file per symbol (#1330) - the cache file is keyed by symbol and serves only on the day it was written --- tests/test_ohlcv_cache_freshness.py | 118 ++++++++++---------- tests/test_ohlcv_latest_bar.py | 8 +- tradingagents/dataflows/stockstats_utils.py | 36 +++--- 3 files changed, 78 insertions(+), 84 deletions(-) diff --git a/tests/test_ohlcv_cache_freshness.py b/tests/test_ohlcv_cache_freshness.py index 65a417f26..a30dc78d7 100644 --- a/tests/test_ohlcv_cache_freshness.py +++ b/tests/test_ohlcv_cache_freshness.py @@ -1,80 +1,72 @@ -"""Same-day OHLCV cache must not serve a stale snapshot all day (#1150). +"""The OHLCV cache: one file per symbol, fresh only on the day it was written. -The cache file is keyed per day, so a run started before the day's bar was final -would be reused by every later run, feeding a stale close into technical -analysis. Two cases matter for a current-day request: the bar may be missing, or -present but still in progress (Yahoo publishes a partial daily candle intraday). -Refresh is bounded by a TTL so repeated runs cannot hammer the vendor. +A current-day request also refetches past a TTL, so a run started before the +day's bar was final is not served that snapshot all day (#1150). Keying the file +by symbol rather than by day keeps the cache from growing a file per symbol per +day (#1330). """ from __future__ import annotations import os -import time import pandas as pd import pytest import tradingagents.dataflows.stockstats_utils as su -TODAY = pd.Timestamp("2026-07-18") +NOW = pd.Timestamp("2026-07-18 12:00") STALE = su.OHLCV_CACHE_TTL_SECONDS + 60 -def _write(tmp_path, name="cache.csv", age_seconds=0.0, last_date="2026-07-17"): +def _write(tmp_path, name="AAPL-YFin-data.csv", age_seconds=0.0, last_date="2026-07-17"): f = tmp_path / name - pd.DataFrame({"Date": [last_date], "Close": [1.0]}).to_csv(f, index=False) - if age_seconds: - old = time.time() - age_seconds - os.utime(f, (old, old)) - return str(f) + pd.DataFrame({"Date": [last_date], "Close": [100.0]}).to_csv(f, index=False) + written = NOW.timestamp() - age_seconds + os.utime(f, (written, written)) + return f + + +def _load(tmp_path, monkeypatch, curr_date, download): + monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) + monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: NOW)) + monkeypatch.setattr(su.yf, "download", download) + return su.load_ohlcv("AAPL", curr_date) + + +def _fail_download(*a, **k): + raise AssertionError("fresh cache must not refetch") @pytest.mark.unit -def test_current_day_cache_past_ttl_is_refreshed(tmp_path): - # Bar missing (rows stop at yesterday) and file older than the TTL -> refetch. - assert su._needs_same_day_refresh(_write(tmp_path, age_seconds=STALE), TODAY, TODAY) is True - - -@pytest.mark.unit -def test_partial_current_day_bar_is_still_refreshed(tmp_path): - # Today's row is present but may be an in-progress candle whose Close is not - # the closing price. Row inspection can't distinguish it, so the TTL governs. +def test_current_day_cache_past_ttl_is_not_fresh(tmp_path): + # Today's bar missing or still in progress: row inspection can't tell, so the TTL governs. + assert su._cache_is_fresh(_write(tmp_path, age_seconds=STALE), NOW.normalize(), NOW) is False f = _write(tmp_path, age_seconds=STALE, last_date="2026-07-18") - assert su._needs_same_day_refresh(f, TODAY, TODAY) is True + assert su._cache_is_fresh(f, NOW.normalize(), NOW) is False @pytest.mark.unit -def test_recent_cache_is_not_refetched(tmp_path): +def test_recent_cache_is_fresh(tmp_path): # Written moments ago: don't hammer the vendor (weekend/holiday guard). - assert su._needs_same_day_refresh(_write(tmp_path), TODAY, TODAY) is False + assert su._cache_is_fresh(_write(tmp_path), NOW.normalize(), NOW) is True @pytest.mark.unit -def test_historical_request_always_uses_cache(tmp_path): - # Past dates are immutable: never refetch, however old the file is. - past = pd.Timestamp("2026-05-01") +def test_historical_request_uses_todays_cache_past_the_ttl(tmp_path): f = _write(tmp_path, age_seconds=STALE, last_date="2026-04-30") - assert su._needs_same_day_refresh(f, past, TODAY) is False + assert su._cache_is_fresh(f, pd.Timestamp("2026-05-01"), NOW) is True + + +@pytest.mark.unit +def test_a_download_from_an_earlier_day_is_not_fresh(tmp_path): + f = _write(tmp_path, age_seconds=13 * 3600) # yesterday 23:00 + assert su._cache_is_fresh(f, pd.Timestamp("2026-05-01"), NOW) is False @pytest.mark.unit def test_load_ohlcv_refetches_stale_same_day_cache(tmp_path, monkeypatch): - """End-to-end: the helper is actually wired into load_ohlcv's cache branch. - - Without this, the unit tests above would still pass if the helper were never - called from the real code path. - """ - monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) - monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: TODAY)) - - # Pre-seed the cache file load_ohlcv will look for, aged past the TTL. - start = (TODAY - pd.DateOffset(years=5)).strftime("%Y-%m-%d") - end = (TODAY + pd.Timedelta(days=1)).strftime("%Y-%m-%d") - cache_file = tmp_path / f"AAPL-YFin-data-{start}-{end}.csv" - pd.DataFrame({"Date": ["2026-07-17"], "Close": [100.0]}).to_csv(cache_file, index=False) - old = time.time() - STALE - os.utime(cache_file, (old, old)) - + """End-to-end: the freshness check is wired into load_ohlcv's cache branch.""" + _write(tmp_path, age_seconds=STALE) calls = [] def _fake_download(*a, **k): @@ -83,27 +75,31 @@ def test_load_ohlcv_refetches_stale_same_day_cache(tmp_path, monkeypatch): {"Date": pd.to_datetime(["2026-07-17", "2026-07-18"]), "Close": [100.0, 222.0]} ).set_index("Date") - monkeypatch.setattr(su.yf, "download", _fake_download) - - out = su.load_ohlcv("AAPL", TODAY.strftime("%Y-%m-%d")) - + out = _load(tmp_path, monkeypatch, "2026-07-18", _fake_download) assert calls, "stale same-day cache must trigger a refetch" assert 222.0 in out["Close"].values, "refreshed close must reach the caller" @pytest.mark.unit def test_load_ohlcv_reuses_fresh_same_day_cache(tmp_path, monkeypatch): - # Mirror image: a fresh cache must NOT trigger a download. + _write(tmp_path, last_date="2026-07-18") + _load(tmp_path, monkeypatch, "2026-07-18", _fail_download) + + +@pytest.mark.unit +def test_one_cache_file_per_symbol_across_days(tmp_path, monkeypatch): + """A later day's download replaces the symbol's file instead of adding one (#1330).""" monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) - monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: TODAY)) + frame = pd.DataFrame({"Date": pd.to_datetime(["2026-07-16", "2026-07-17"]), "Close": [1.0, 2.0]}) + downloads = [] + monkeypatch.setattr(su.yf, "download", lambda *a, **k: downloads.append(1) or frame.set_index("Date")) - start = (TODAY - pd.DateOffset(years=5)).strftime("%Y-%m-%d") - end = (TODAY + pd.Timedelta(days=1)).strftime("%Y-%m-%d") - cache_file = tmp_path / f"AAPL-YFin-data-{start}-{end}.csv" - pd.DataFrame({"Date": ["2026-07-18"], "Close": [100.0]}).to_csv(cache_file, index=False) + for day in ("2026-07-18 10:00", "2026-07-19 10:00", "2026-07-20 10:00"): + now = pd.Timestamp(day) + monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda now=now: now)) + su.load_ohlcv("AAPL", "2026-07-17") + written = list(tmp_path.glob("AAPL-*.csv")) + os.utime(written[0], (now.timestamp(), now.timestamp())) - def _fail_download(*a, **k): - raise AssertionError("fresh cache must not refetch") - - monkeypatch.setattr(su.yf, "download", _fail_download) - su.load_ohlcv("AAPL", TODAY.strftime("%Y-%m-%d")) + assert len(downloads) == 3, "each new day refetches" + assert [p.name for p in tmp_path.iterdir()] == ["AAPL-YFin-data.csv"] diff --git a/tests/test_ohlcv_latest_bar.py b/tests/test_ohlcv_latest_bar.py index cf9bca676..5483e2d13 100644 --- a/tests/test_ohlcv_latest_bar.py +++ b/tests/test_ohlcv_latest_bar.py @@ -13,6 +13,8 @@ anywhere counts as no data and the staleness check judges the rest. """ from __future__ import annotations +import os + import pandas as pd import pytest @@ -89,9 +91,9 @@ def _run_load(monkeypatch, tmp_path, frame, curr_date): monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)}) today = pd.Timestamp(curr_date) monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today)) - start = (today - pd.DateOffset(years=5)).strftime("%Y-%m-%d") - end = (today + pd.Timedelta(days=1)).strftime("%Y-%m-%d") - (tmp_path / f"AAPL-YFin-data-{start}-{end}.csv").write_text(frame.to_csv(index=False)) + cache_file = tmp_path / "AAPL-YFin-data.csv" + cache_file.write_text(frame.to_csv(index=False)) + os.utime(cache_file, (today.timestamp(), today.timestamp())) def _fail_download(*a, **k): raise AssertionError("should use the seeded cache, not download") diff --git a/tradingagents/dataflows/stockstats_utils.py b/tradingagents/dataflows/stockstats_utils.py index 1bdd02ab8..63d7b2ff1 100644 --- a/tradingagents/dataflows/stockstats_utils.py +++ b/tradingagents/dataflows/stockstats_utils.py @@ -164,21 +164,19 @@ def _assert_ohlcv_not_stale( ) -def _needs_same_day_refresh(data_file, curr_date_dt, today_date) -> bool: - """Whether a cached frame must be refetched to reflect the requested day. +def _cache_is_fresh(data_file, curr_date_dt, now) -> bool: + """Whether the symbol's cached download can serve this request. - The cache file is keyed per day, so without this a run started before the - day's bar was final keeps serving that snapshot to every later run (#1150). - Two distinct staleness cases exist for a current-day request: the bar may be - missing entirely, or present but still in progress — Yahoo publishes a - partial daily candle during market hours, whose ``Close`` is not the closing - price. Row inspection cannot tell a partial bar from a final one, so the TTL - governs every current-day cache. Historical requests always reuse the cache, - since those rows are immutable. + The file holds the download made on the day it was written, so it serves + only that day. A current-day request also refetches once the file is older + than the TTL: Yahoo publishes a partial daily candle during market hours, + whose ``Close`` is not the closing price, and row inspection cannot tell it + from a final one (#1150). """ - if curr_date_dt.date() < today_date.date(): + written = pd.Timestamp.fromtimestamp(os.path.getmtime(data_file)) + if written.date() != now.date(): return False - return time.time() - os.path.getmtime(data_file) > OHLCV_CACHE_TTL_SECONDS + return curr_date_dt.date() < now.date() or (now - written).total_seconds() <= OHLCV_CACHE_TTL_SECONDS def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: @@ -197,19 +195,19 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: config = get_config() curr_date_dt = pd.to_datetime(curr_date).normalize() - # Cache uses a fixed window (5y to today) so one file per symbol. - today_date = pd.Timestamp.today() - start_date = today_date - pd.DateOffset(years=5) + # One cache file per symbol, holding the latest 5y-to-today download. + now = pd.Timestamp.today() + start_date = now - pd.DateOffset(years=5) start_str = start_date.strftime("%Y-%m-%d") # yfinance ``end`` is EXCLUSIVE; request tomorrow so today's row is included # when curr_date is the current day (#986). Look-ahead is still prevented by # the curr_date filter below. - end_str = (today_date + pd.Timedelta(days=1)).strftime("%Y-%m-%d") + end_str = (now + pd.Timedelta(days=1)).strftime("%Y-%m-%d") os.makedirs(config["data_cache_dir"], exist_ok=True) data_file = os.path.join( config["data_cache_dir"], - f"{safe_symbol}-YFin-data-{start_str}-{end_str}.csv", + f"{safe_symbol}-YFin-data.csv", ) # A cached file may be empty if a prior fetch failed (unknown symbol, @@ -218,12 +216,10 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame: data = None if os.path.exists(data_file): cached = pd.read_csv(data_file, on_bad_lines="skip", encoding="utf-8") - # Serve the cache only when it is usable and not a stale snapshot of the - # day being requested (#1150); otherwise fall through and refetch. if ( not cached.empty and "Close" in cached.columns - and not _needs_same_day_refresh(data_file, curr_date_dt, today_date) + and _cache_is_fresh(data_file, curr_date_dt, now) ): data = cached