mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-22 20:55:33 +03:00
fix(memory): gate past-context lessons to point-in-time in backtests
- get_past_context returned every resolved lesson regardless of the run date, so a historical run could learn from an outcome that had not happened yet - record each resolved entry's resolution date (the last price bar used) and filter get_past_context(as_of=trade_date) on it for a historical run; a current-date run passes None so live behavior and pre-migration entries (no stored resolution date, conservatively excluded from backtests) are unaffected #1251
This commit is contained in:
@@ -67,9 +67,21 @@ class TradingMemoryLog:
|
||||
"""Return entries with outcome:pending (for Phase B)."""
|
||||
return [e for e in self.load_entries() if e.get("pending")]
|
||||
|
||||
def get_past_context(self, ticker: str, n_same: int = 5, n_cross: int = 3) -> str:
|
||||
"""Return formatted past context string for agent prompt injection."""
|
||||
def get_past_context(
|
||||
self, ticker: str, n_same: int = 5, n_cross: int = 3, as_of: str | None = None
|
||||
) -> str:
|
||||
"""Return formatted past context string for agent prompt injection.
|
||||
|
||||
When ``as_of`` (yyyy-mm-dd) is given, only lessons whose outcome was
|
||||
already known by that date are included — an entry is kept only if it
|
||||
stores a resolution date (``resolved:...``) that is on or before
|
||||
``as_of``. This keeps a historical/backtest run from learning from
|
||||
outcomes that had not happened yet (#1251). ``as_of=None`` disables the
|
||||
filter, so live runs and pre-migration entries are unaffected.
|
||||
"""
|
||||
entries = [e for e in self.load_entries() if not e.get("pending")]
|
||||
if as_of is not None:
|
||||
entries = [e for e in entries if e.get("resolved") and e["resolved"] <= as_of]
|
||||
if not entries:
|
||||
return ""
|
||||
|
||||
@@ -104,12 +116,14 @@ class TradingMemoryLog:
|
||||
alpha_return: float,
|
||||
holding_days: int,
|
||||
reflection: str,
|
||||
resolution_date: str | None = None,
|
||||
) -> None:
|
||||
"""Replace pending tag and append REFLECTION section using atomic write.
|
||||
|
||||
Finds the first pending entry matching (trade_date, ticker), updates
|
||||
its tag with return figures, and appends a REFLECTION section. Uses
|
||||
a temp-file + os.replace() so a crash mid-write never corrupts the log.
|
||||
its tag with return figures (and the ``resolution_date`` the outcome
|
||||
became known), and appends a REFLECTION section. Uses a temp-file +
|
||||
os.replace() so a crash mid-write never corrupts the log.
|
||||
"""
|
||||
if not self._log_path or not self._log_path.exists():
|
||||
return
|
||||
@@ -140,9 +154,8 @@ class TradingMemoryLog:
|
||||
# Parse rating from the existing pending tag
|
||||
fields = [f.strip() for f in tag_line[1:-1].split("|")]
|
||||
rating = fields[2]
|
||||
new_tag = (
|
||||
f"[{trade_date} | {ticker} | {rating}"
|
||||
f" | {raw_pct} | {alpha_pct} | {holding_days}d]"
|
||||
new_tag = self._resolved_tag(
|
||||
trade_date, ticker, rating, raw_pct, alpha_pct, holding_days, resolution_date
|
||||
)
|
||||
rest = "\n".join(lines[1:])
|
||||
new_blocks.append(
|
||||
@@ -194,9 +207,9 @@ class TradingMemoryLog:
|
||||
rating = fields[2]
|
||||
raw_pct = f"{upd['raw_return']:+.1%}"
|
||||
alpha_pct = f"{upd['alpha_return']:+.1%}"
|
||||
new_tag = (
|
||||
f"[{trade_date} | {ticker} | {rating}"
|
||||
f" | {raw_pct} | {alpha_pct} | {upd['holding_days']}d]"
|
||||
new_tag = self._resolved_tag(
|
||||
trade_date, ticker, rating, raw_pct, alpha_pct,
|
||||
upd["holding_days"], upd.get("resolution_date"),
|
||||
)
|
||||
rest = "\n".join(lines[1:])
|
||||
new_blocks.append(
|
||||
@@ -217,6 +230,21 @@ class TradingMemoryLog:
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
@staticmethod
|
||||
def _resolved_tag(
|
||||
trade_date, ticker, rating, raw_pct, alpha_pct, holding_days, resolution_date
|
||||
) -> str:
|
||||
"""Build a resolved entry tag, recording the outcome's known-by date.
|
||||
|
||||
``resolution_date`` (the date of the last price bar used for the return)
|
||||
is the point-in-time cutoff a later run filters on (#1251). Omitted when
|
||||
unavailable, keeping the legacy 6-field tag.
|
||||
"""
|
||||
tag = f"[{trade_date} | {ticker} | {rating} | {raw_pct} | {alpha_pct} | {holding_days}d"
|
||||
if resolution_date:
|
||||
tag += f" | resolved:{resolution_date}"
|
||||
return tag + "]"
|
||||
|
||||
def _apply_rotation(self, blocks: list[str]) -> list[str]:
|
||||
"""Drop oldest resolved blocks when their count exceeds max_entries.
|
||||
|
||||
@@ -264,6 +292,12 @@ class TradingMemoryLog:
|
||||
fields = [f.strip() for f in tag_line[1:-1].split("|")]
|
||||
if len(fields) < 4:
|
||||
return None
|
||||
# Optional trailing "resolved:YYYY-MM-DD" field records when the outcome
|
||||
# became known, for point-in-time filtering (#1251).
|
||||
resolved = None
|
||||
for f in fields[6:]:
|
||||
if f.startswith("resolved:"):
|
||||
resolved = f[len("resolved:"):].strip()
|
||||
entry = {
|
||||
"date": fields[0],
|
||||
"ticker": fields[1],
|
||||
@@ -272,6 +306,7 @@ class TradingMemoryLog:
|
||||
"raw": fields[3] if fields[3] != "pending" else None,
|
||||
"alpha": fields[4] if len(fields) > 4 else None,
|
||||
"holding": fields[5] if len(fields) > 5 else None,
|
||||
"resolved": resolved,
|
||||
}
|
||||
body = "\n".join(lines[1:]).strip()
|
||||
decision_match = self._DECISION_RE.search(body)
|
||||
|
||||
@@ -272,7 +272,7 @@ class TradingAgentsGraph:
|
||||
def _fetch_returns(
|
||||
self, ticker: str, trade_date: str, holding_days: int = 5,
|
||||
benchmark: str = "SPY",
|
||||
) -> tuple[float | None, float | None, int | None]:
|
||||
) -> tuple[float | None, float | None, int | None, str | None]:
|
||||
"""Fetch raw and alpha return for ticker over holding_days from trade_date.
|
||||
|
||||
``benchmark`` is the index used as the alpha baseline (resolved by the
|
||||
@@ -294,7 +294,7 @@ class TradingAgentsGraph:
|
||||
bench = yf.Ticker(benchmark).history(start=trade_date, end=end_str)
|
||||
|
||||
if len(stock) < 2 or len(bench) < 2:
|
||||
return None, None, None
|
||||
return None, None, None, None
|
||||
|
||||
actual_days = min(holding_days, len(stock) - 1, len(bench) - 1)
|
||||
raw = float(
|
||||
@@ -306,13 +306,16 @@ class TradingAgentsGraph:
|
||||
/ bench["Close"].iloc[0]
|
||||
)
|
||||
alpha = raw - bench_ret
|
||||
return raw, alpha, actual_days
|
||||
# The date of the last price bar used is when this outcome became
|
||||
# known — the point-in-time cutoff for injecting the lesson (#1251).
|
||||
resolution_date = stock.index[actual_days].strftime("%Y-%m-%d")
|
||||
return raw, alpha, actual_days, resolution_date
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Could not resolve outcome for %s on %s vs %s (will retry next run): %s",
|
||||
ticker, trade_date, benchmark, e,
|
||||
)
|
||||
return None, None, None
|
||||
return None, None, None, None
|
||||
|
||||
def _resolve_pending_entries(self, ticker: str) -> None:
|
||||
"""Resolve pending log entries for ticker at the start of a new run.
|
||||
@@ -331,7 +334,7 @@ class TradingAgentsGraph:
|
||||
benchmark = self._resolve_benchmark(ticker)
|
||||
updates = []
|
||||
for entry in pending:
|
||||
raw, alpha, days = self._fetch_returns(
|
||||
raw, alpha, days, resolution_date = self._fetch_returns(
|
||||
ticker, entry["date"], benchmark=benchmark,
|
||||
)
|
||||
if raw is None:
|
||||
@@ -349,6 +352,7 @@ class TradingAgentsGraph:
|
||||
"alpha_return": alpha,
|
||||
"holding_days": days,
|
||||
"reflection": reflection,
|
||||
"resolution_date": resolution_date,
|
||||
})
|
||||
|
||||
if updates:
|
||||
@@ -366,6 +370,17 @@ class TradingAgentsGraph:
|
||||
identity = resolve_instrument_identity(ticker)
|
||||
return build_instrument_context(ticker, asset_type, identity)
|
||||
|
||||
def _memory_as_of(self, trade_date) -> str | None:
|
||||
"""Point-in-time cutoff for past-context lessons (#1251).
|
||||
|
||||
A historical/backtest run (trade date before today) filters lessons to
|
||||
those already resolved by the trade date. A current-date run returns
|
||||
None, disabling the filter so live behavior and pre-migration entries
|
||||
(which have no stored resolution date) are unaffected.
|
||||
"""
|
||||
td = str(trade_date)
|
||||
return td if td < datetime.now().strftime("%Y-%m-%d") else None
|
||||
|
||||
def _run_signature(self, asset_type: str) -> str:
|
||||
"""Graph-shape inputs that must invalidate a checkpoint if changed.
|
||||
|
||||
@@ -476,8 +491,12 @@ class TradingAgentsGraph:
|
||||
checkpoint_thread_id: str | None = None):
|
||||
"""Execute the graph and write the resulting state to disk and memory log."""
|
||||
# Initialize state — inject memory log context for PM and the
|
||||
# deterministically resolved instrument identity for all agents.
|
||||
past_context = self.memory_log.get_past_context(company_name)
|
||||
# deterministically resolved instrument identity for all agents. On a
|
||||
# historical run, gate lessons to those whose outcome was known by the
|
||||
# trade date so a backtest can't learn from the future (#1251).
|
||||
past_context = self.memory_log.get_past_context(
|
||||
company_name, as_of=self._memory_as_of(trade_date)
|
||||
)
|
||||
instrument_context = self.resolve_instrument_context(company_name, asset_type)
|
||||
init_agent_state = self.propagator.create_initial_state(
|
||||
company_name,
|
||||
|
||||
Reference in New Issue
Block a user