fix(dataflows): clamp the FRED vintage pin to FRED's own clock

- the unconditional realtime pin 400s when curr_date is ahead of FRED's
  US-Central date (a live run's local date), which the router then degrades to
  a silent DATA_UNAVAILABLE — an Asia/Pacific run loses macro data
- clamp realtime_start/end to min(curr_date, FRED-today) via pytz Chicago;
  a past curr_date pins unchanged, so historical look-ahead safety is preserved
- name the vintage in the empty-result message: widening the window can't fix a
  series with no vintage coverage #1275
This commit is contained in:
Yijia-Xiao
2026-09-01 04:58:53 +00:00
parent 2448d0a125
commit 70b58c21dc
2 changed files with 64 additions and 16 deletions

View File

@@ -151,22 +151,46 @@ class FredFormattingTests(unittest.TestCase):
self.assertEqual(obs_params["observation_start"], "2025-07-02") # 90d back
def test_requests_pin_the_data_vintage(self):
# #1275: both the metadata and observations requests must set
# realtime_start=realtime_end=curr_date, or FRED serves the latest
# revision and revision-prone series leak future information.
# #1275: both the metadata and observations requests must pin the vintage
# to curr_date (clamped to FRED's today), or FRED serves the latest
# revision and revision-prone series leak future information. A past
# curr_date sits below FRED's today, so it pins through unchanged.
captured = {}
def _capture(path, params):
captured[path] = params
return _META if path == "series" else _OBS
with mock.patch.object(fred, "_request", side_effect=_capture):
with mock.patch.object(fred, "_fred_today", return_value="2026-01-01"), \
mock.patch.object(fred, "_request", side_effect=_capture):
fred.get_macro_data("cpi", "2025-09-30", 90)
for path in ("series", "series/observations"):
self.assertEqual(captured[path]["realtime_start"], "2025-09-30", path)
self.assertEqual(captured[path]["realtime_end"], "2025-09-30", path)
def test_future_curr_date_clamps_vintage_to_fred_today(self):
# #1275 regression: on a live run curr_date is the caller's LOCAL date,
# which can be a day ahead of FRED's US-Central clock. Pinning the vintage
# to that future date 400s, and the routing layer then drops macro data
# silently. The pin must clamp to FRED's today; the observation window
# (future bars can't exist yet) stays at curr_date.
captured = {}
def _capture(path, params):
captured[path] = params
return _META if path == "series" else _OBS
with mock.patch.object(fred, "_fred_today", return_value="2026-08-31"), \
mock.patch.object(fred, "_request", side_effect=_capture):
fred.get_macro_data("cpi", "2026-09-01", 90) # local a day ahead of Chicago
for path in ("series", "series/observations"):
self.assertEqual(captured[path]["realtime_start"], "2026-08-31", path)
self.assertEqual(captured[path]["realtime_end"], "2026-08-31", path)
# the observation window still tracks curr_date, not the clamped vintage
self.assertEqual(captured["series/observations"]["observation_end"], "2026-09-01")
@pytest.mark.unit
class FredRoutingTests(unittest.TestCase):

View File

@@ -12,6 +12,7 @@ import logging
import os
from datetime import datetime, timedelta
import pytz
import requests
from .errors import VendorNotConfiguredError
@@ -20,6 +21,12 @@ logger = logging.getLogger(__name__)
FRED_API_BASE = "https://api.stlouisfed.org/fred"
# FRED's realtime clock runs on US Central (St. Louis Fed). It rejects a
# realtime date in its own future with a 400, so the vintage pin is clamped to
# this rather than the caller's local date (#1275). pytz (already a dependency)
# bundles its own tz database, so this works where system tzdata is absent.
FRED_TZ = pytz.timezone("America/Chicago")
# Network timeout (seconds) so a stalled request can't hang the agents,
# mirroring the Alpha Vantage client.
REQUEST_TIMEOUT = 30
@@ -115,6 +122,16 @@ def _resolve_series_id(indicator: str) -> str:
return candidate
def _fred_today() -> str:
"""FRED's current calendar date (US Central) as ``yyyy-mm-dd``.
The vintage pin is clamped to this: FRED rejects a ``realtime_start`` after
its own today with a 400, and ``curr_date`` on a live run comes from the
caller's local clock, which can already be tomorrow in Chicago.
"""
return datetime.now(FRED_TZ).strftime("%Y-%m-%d")
def _request(path: str, params: dict) -> dict:
"""GET a FRED endpoint, surfacing FRED's JSON error body on a bad request."""
api_params = {**params, "api_key": get_api_key(), "file_type": "json"}
@@ -144,11 +161,11 @@ def get_macro_data(
indicator: A friendly alias (e.g. "cpi", "unemployment", "10y_treasury")
or a raw FRED series ID (e.g. "CPIAUCSL", "DGS10").
curr_date: The as-of date (yyyy-mm-dd). It bounds the observation window
AND pins the data vintage: FRED is queried with
``realtime_start = realtime_end = curr_date`` so a historical run sees
the values that were actually published by that date, not later
revisions. Without this, revision-prone series (CPI, GDP, ...) would
leak future information into a backtest (#1275).
AND pins the data vintage: FRED is queried with the realtime bounds
set to ``curr_date`` (clamped to FRED's own today) so a historical
run sees the values that were actually published by that date, not
later revisions. Without this, revision-prone series (CPI, GDP, ...)
would leak future information into a backtest (#1275).
look_back_days: Trailing window length; ``None`` uses DEFAULT_LOOKBACK_DAYS.
Returns:
@@ -161,11 +178,16 @@ def get_macro_data(
end_dt = datetime.strptime(curr_date, "%Y-%m-%d")
start_date = (end_dt - timedelta(days=look_back_days)).strftime("%Y-%m-%d")
# Pin the data vintage to curr_date. FRED defaults both realtime bounds to
# today, which serves the LATEST revision of every observation; a single-day
# realtime interval asks for the values known as of curr_date instead. This
# is applied to both the metadata and observations requests (#1275).
realtime = {"realtime_start": curr_date, "realtime_end": curr_date}
# Pin the data vintage. FRED defaults both realtime bounds to today, serving
# the LATEST revision of every observation; a single-day realtime interval
# asks for the values known as of the pin instead, on both the metadata and
# observations requests (#1275). Clamp to FRED's today: on a live run
# curr_date is the caller's local date, which can be a day ahead of Chicago,
# and a realtime date in FRED's future 400s -> the routing layer would then
# drop macro data silently. A past curr_date is unaffected, so historical
# point-in-time behaviour is preserved.
pit = min(curr_date, _fred_today())
realtime = {"realtime_start": pit, "realtime_end": pit}
# Invalid LLM-supplied indicator: return guidance rather than raising, so a
# bad argument doesn't abort the run (the routing layer also degrades macro
@@ -215,8 +237,10 @@ def get_macro_data(
if not points:
return header + (
f"\nNo observations for {series_id} in this window. The series may "
f"report less frequently than the window length; widen look_back_days."
f"\nNo observations for {series_id} in this window at the {pit} "
f"vintage. The series may report less frequently than the window "
f"(try a longer look_back_days), or have no vintage published by "
f"then (unpublished as of {pit}, or before ALFRED coverage begins)."
)
first_date, first_val = points[0]