fix(dataflows): withhold insider trades with no filing date from a past run

- Yahoo and Alpha Vantage date a trade by when it happened, not by its Form 4 filing, so a run dated in the past is told they are withheld instead of served trades that may not yet have been public
This commit is contained in:
Yijia-Xiao
2026-09-29 05:57:18 +00:00
parent 1cf16bd0e4
commit da6b35f22f
8 changed files with 62 additions and 89 deletions
+2
View File
@@ -284,6 +284,8 @@ US company statements come from SEC EDGAR, which records the date every figure w
Other companies' statements come from Yahoo Finance, which dates a statement by the period it covers rather than by when it was published. A run dated today reads them; a run dated in the past is told they are withheld, since Yahoo cannot say which figures were public by then.
Insider trades are dated by when they happened, not when they were filed, so a run dated in the past is told they are withheld as well.
SEC asks callers to identify themselves and refuses requests that carry no contact address, so a default one is sent. Set your own so SEC can reach you rather than the project:
```bash
+2 -1
View File
@@ -16,6 +16,7 @@ import pytest
from tradingagents.dataflows import router
from tradingagents.dataflows.config import set_config
from tradingagents.dataflows.date_window import get_current_date
from tradingagents.dataflows.errors import NoMarketDataError
from tradingagents.dataflows.vendors.yahoo import common, ohlcv
@@ -102,4 +103,4 @@ def test_an_unreachable_yahoo_is_not_reported_as_a_symbol_without_insider_data()
with mock.patch.object(fundamentals.yf, "Ticker", return_value=ticker), \
mock.patch.object(fundamentals, "vendor_reachable", return_value=False), \
pytest.raises(VendorUnavailableError):
fundamentals.get_insider_transactions("AAPL", as_of_date="2026-09-21")
fundamentals.get_insider_transactions("AAPL", as_of_date=get_current_date())
+18
View File
@@ -58,3 +58,21 @@ def test_statements_come_from_sec_edgar_first_by_default():
with mock.patch.dict(router.VENDOR_METHODS, {"get_balance_sheet": chain}):
router.route_to_vendor("get_balance_sheet", "AAPL", "quarterly", PAST)
assert served == ["sec_edgar"]
@pytest.mark.unit
def test_yahoo_withholds_insider_trades_from_a_past_run_without_asking():
"""A trade is dated when it happened; it became public with its Form 4, up to
two business days later, and Yahoo reports no filing date."""
with mock.patch.object(yahoo, "yf_retry", side_effect=AssertionError("requested")):
out = yahoo.get_insider_transactions("AAPL", PAST)
assert "withheld" in out and PAST in out
@pytest.mark.unit
def test_alpha_vantage_withholds_insider_trades_from_a_past_run_without_asking():
from tradingagents.dataflows.vendors.alpha_vantage import news as alpha_vantage_news
with mock.patch.object(alpha_vantage_news, "_make_api_request", side_effect=AssertionError("requested")):
out = alpha_vantage_news.get_insider_transactions("IBM", PAST)
assert "withheld" in out and PAST in out
+1 -49
View File
@@ -7,7 +7,6 @@ only live odds, so a historical run withholds them.
from __future__ import annotations
import json
from unittest import mock
import pandas as pd
@@ -16,7 +15,6 @@ import pytest
from tradingagents.agents import tools
from tradingagents.dataflows.date_window import get_current_date
from tradingagents.dataflows.vendors import polymarket
from tradingagents.dataflows.vendors.alpha_vantage import news as alpha_vantage_news
from tradingagents.dataflows.vendors.yahoo import (
fundamentals as yahoo_fundamentals,
market as yahoo_market,
@@ -37,37 +35,12 @@ def _yf_insider(frame, curr_date):
return yahoo_fundamentals.get_insider_transactions("AAPL", curr_date)
@pytest.mark.unit
def test_yfinance_insider_filings_after_the_date_are_dropped():
out = _yf_insider(_insider_frame("2026-09-08", "2025-06-02", "2025-05-30", "2025-01-10"), "2025-06-01")
assert "2026-09-08" not in out and "2025-06-02" not in out
assert "2025-05-30" in out and "2025-01-10" in out
@pytest.mark.unit
def test_yfinance_insider_date_before_coverage_is_unavailable_not_absent():
out = _yf_insider(_insider_frame("2026-09-08", "2025-06-02"), "2024-01-01")
assert "unavailable" in out and "No insider transactions reported" not in out
assert "2025-06-02" not in out # a transaction after the run date
@pytest.mark.unit
def test_yfinance_insider_without_a_date_is_unfiltered():
out = _yf_insider(_insider_frame("2026-09-08", "2025-01-10"), None)
assert "2026-09-08" in out and "2025-01-10" in out
@pytest.mark.unit
def test_alpha_vantage_insider_filings_after_the_date_are_dropped():
body = json.dumps({"data": [
{"transaction_date": "2026-09-08", "executive": "A"},
{"transaction_date": "2025-05-30", "executive": "B"},
]})
with mock.patch.object(alpha_vantage_news, "_make_api_request", return_value=body):
out = json.loads(alpha_vantage_news.get_insider_transactions("AAPL", "2025-06-01"))
assert [t["executive"] for t in out["data"]] == ["B"]
@pytest.mark.unit
def test_polymarket_withholds_live_odds_from_a_historical_run():
with mock.patch.object(polymarket, "_request", side_effect=AssertionError("must not fetch")):
@@ -121,27 +94,6 @@ def test_a_current_run_is_not_cluttered_with_a_vintage_note(monkeypatch):
assert "current name" not in today
@pytest.mark.unit
def test_insider_rows_are_dated_by_the_trade_not_the_filing():
"""yfinance reports the transaction date and carries no filing date. A trade
becomes public when the Form 4 is filed, up to two business days later, so a
run must not be told these rows were public on their transaction date."""
import pandas as pd
frame = pd.DataFrame({
"Shares": [100, 200],
"Text": ["Sale at price 10.00 per share.", "Sale at price 11.00 per share."],
"Start Date": pd.to_datetime(["2026-05-01", "2026-05-20"]),
})
ticker = mock.Mock(insider_transactions=frame)
with mock.patch.object(yahoo_market.yf, "Ticker", return_value=ticker):
out = yahoo_fundamentals.get_insider_transactions("AAPL", "2026-05-10")
assert "2026-05-01" in out and "2026-05-20" not in out # still bounded by the date
assert "transaction date" in out.lower() # and says what the date means
assert "filed" in out.lower() # and that filing comes later
@pytest.mark.unit
def test_an_indicator_that_could_not_be_read_is_not_shown_as_a_blank_value():
"""The per-day fallback returned an empty string for a failed read, so the
@@ -164,7 +116,7 @@ def test_an_indicator_that_could_not_be_read_is_not_shown_as_a_blank_value():
("get_balance_sheet", ("AAPL", "annual", get_current_date())),
("get_cashflow", ("AAPL", "annual", get_current_date())),
("get_income_statement", ("AAPL", "annual", get_current_date())),
("get_insider_transactions", ("AAPL", "2026-09-01")),
("get_insider_transactions", ("AAPL", get_current_date())),
])
def test_a_yfinance_failure_is_a_vendor_error_not_a_report(func, args):
"""Returning the failure as text makes the router count it as an answer, so
+3 -1
View File
@@ -21,6 +21,7 @@ from tradingagents.agents.tools import (
)
from tradingagents.dataflows import router
from tradingagents.dataflows.config import set_config
from tradingagents.dataflows.date_window import get_current_date
from tradingagents.dataflows.errors import VendorUnavailableError
from tradingagents.dataflows.vendors.yahoo import common, fundamentals, ohlcv
@@ -55,7 +56,8 @@ def test_a_rate_limit_that_outlasts_the_retries_is_reported_as_one(yahoo, tool,
yahoo.setattr(yf.Ticker, "insider_transactions", property(_rate_limited))
yahoo.setattr(yf, "Search", _rate_limited)
out = tool.func(*args, trade_date=DAY)
# Dated today: a past run is not served insider trades at all.
out = tool.func(*args, trade_date=get_current_date())
assert out.startswith("DATA_UNAVAILABLE"), out
assert "delisted" not in out
+19
View File
@@ -127,6 +127,25 @@ def withhold_live_profile(as_of_date: str | None, label: str) -> str | None:
)
def withhold_undisclosed_trades(as_of_date: str | None, label: str) -> str | None:
"""Notice to serve instead of insider trades with no filing date, or None to serve them.
A vendor that dates an insider trade by when it happened (Yahoo, Alpha
Vantage) cannot say when it became public: its Form 4 is filed up to two
business days later. A past run is told so rather than served trades that
may not yet have been disclosed.
"""
if not is_historical(as_of_date):
return None
return (
f"# Insider Transactions for {label}\n"
f"# Point-in-time as of: {as_of_date}\n\n"
f"Insider transactions are withheld for this date. This vendor dates a trade "
f"by when it happened and reports no filing date, so it cannot show which "
f"trades were public on {as_of_date}."
)
def withhold_undated_statements(as_of_date: str | None, label: str, title: str) -> str | None:
"""Notice to serve instead of a statement with no filing date, or None to serve it.
+6 -12
View File
@@ -1,6 +1,6 @@
import json
from tradingagents.dataflows.config import get_config
from tradingagents.dataflows.date_window import withhold_undisclosed_trades
from tradingagents.dataflows.vendors.alpha_vantage.common import (
_make_api_request,
format_datetime_for_api,
@@ -77,19 +77,13 @@ def get_insider_transactions(symbol: str, as_of_date: str | None = None) -> dict
Args:
symbol: Ticker symbol. Example: "IBM".
as_of_date: When given, only transactions on or before it (yyyy-mm-dd).
as_of_date: The analysis date (yyyy-mm-dd); a past one withholds them.
Returns:
Dictionary containing insider transaction data or JSON string.
"""
params = {
"symbol": symbol,
}
response = _make_api_request("INSIDER_TRANSACTIONS", params)
if not as_of_date:
return response
payload = json.loads(response)
payload["data"] = [t for t in payload["data"] if t["transaction_date"] <= as_of_date]
return json.dumps(payload)
withheld = withhold_undisclosed_trades(as_of_date, symbol)
if withheld:
return withheld
return _make_api_request("INSIDER_TRANSACTIONS", {"symbol": symbol})
+11 -26
View File
@@ -1,9 +1,12 @@
from typing import Annotated
import pandas as pd
import yfinance as yf
from tradingagents.dataflows.date_window import withhold_live_profile, withhold_undated_statements
from tradingagents.dataflows.date_window import (
withhold_live_profile,
withhold_undated_statements,
withhold_undisclosed_trades,
)
from tradingagents.dataflows.errors import VendorUnavailableError
from tradingagents.dataflows.net import vendor_reachable
from tradingagents.dataflows.symbols import normalize_symbol
@@ -125,23 +128,15 @@ def get_income_statement(
return _statement(ticker, freq, as_of_date, "Income Statement", "quarterly_income_stmt", "income_stmt")
# Rows are dated by the transaction, which is when the insider traded, not when
# the market learned of it: a Form 4 is filed up to two business days later and
# this vendor reports no filing date, so the most recent rows may not have been
# public on the analysis date.
_TRANSACTION_DATE_VINTAGE = (
"# Rows are dated by transaction date. A trade becomes public when its Form 4 "
"is filed, up to two business days later, so the newest rows may not have been "
"known on this date.\n\n"
)
def get_insider_transactions(
ticker: Annotated[str, "ticker symbol of the company"],
as_of_date: Annotated[str | None, "only transactions on or before this date, yyyy-mm-dd"] = None,
as_of_date: Annotated[str | None, "analysis date, yyyy-mm-dd"] = None,
):
"""Get insider transactions data from yfinance."""
"""Get insider transactions data from yfinance, for a run dated today."""
canonical = normalize_symbol(ticker)
withheld = withhold_undisclosed_trades(as_of_date, canonical)
if withheld:
return withheld
data = yf_retry(lambda: yf.Ticker(canonical).insider_transactions)
# Empty is normal here (many valid symbols have no insider filings),
@@ -151,17 +146,7 @@ def get_insider_transactions(
raise VendorUnavailableError("Yahoo Finance is unreachable; insider filings were not retrieved")
return f"No insider transactions reported for symbol '{canonical}'"
if as_of_date:
traded = data["Start Date"]
kept = data[traded <= pd.Timestamp(as_of_date)]
if kept.empty:
return (
f"<insider transactions unavailable for {canonical} as of {as_of_date}: "
"Yahoo serves recent transactions only>"
)
data = kept
return f"# Insider Transactions data for {canonical}\n" + _TRANSACTION_DATE_VINTAGE + data.to_csv()
return f"# Insider Transactions data for {canonical}\n" + data.to_csv()
def get_company_profile(ticker: str) -> dict: