mirror of
https://github.com/TauricResearch/TradingAgents.git
synced 2026-09-19 11:15:24 +03:00
feat(dataflows): serve US statements as filed, from SEC EDGAR
- statements are limited to facts filed by the run's date, at the values filed then - quarterly figures are picked by span, so a year to date total is not read as a quarter - a fourth quarter is reported unavailable rather than derived - opt in with sec_edgar in the fundamental_data chain; a non-filer falls through
This commit is contained in:
50
.env.example
50
.env.example
@@ -16,57 +16,43 @@ MOONSHOT_API_KEY=
|
||||
GROQ_API_KEY=
|
||||
NVIDIA_API_KEY=
|
||||
|
||||
# FRED (Federal Reserve macro data: rates, inflation, labor, growth). Free key: https://fred.stlouisfed.org/docs/api/api_key.html
|
||||
# SEC EDGAR (US company filings, point-in-time). No key; a contact address SEC can reach you at.
|
||||
#SEC_EDGAR_USER_AGENT=Your Name your@email.com
|
||||
|
||||
# FRED (Federal Reserve macro data). Free key: https://fred.stlouisfed.org/docs/api/api_key.html
|
||||
#FRED_API_KEY=
|
||||
|
||||
# Optional: a custom OpenAI-compatible endpoint (vLLM, LM Studio, llama.cpp,
|
||||
# relay). Select provider "openai_compatible" and set the base URL; the key is
|
||||
# optional (local servers need none).
|
||||
# Custom OpenAI-compatible endpoint (vLLM, LM Studio, llama.cpp). Local servers need no key.
|
||||
#OPENAI_COMPATIBLE_API_KEY=
|
||||
|
||||
# AWS Bedrock (provider "bedrock", install with: pip install ".[bedrock]").
|
||||
# Auth: either a Bedrock API key (bearer token, no AWS access keys) OR the AWS
|
||||
# credential chain (env keys / ~/.aws/credentials / IAM role / AWS_PROFILE). Set
|
||||
# the region either way; a bearer token takes precedence when both are present.
|
||||
# AWS Bedrock (pip install ".[bedrock]"). Bearer token, or the AWS credential chain; set the region either way.
|
||||
#AWS_BEARER_TOKEN_BEDROCK=
|
||||
#AWS_DEFAULT_REGION=us-west-2
|
||||
#AWS_PROFILE=
|
||||
|
||||
# Optional: point at a remote Ollama server. When unset, defaults to
|
||||
# the local instance at http://localhost:11434/v1. Convention follows
|
||||
# the broader Ollama ecosystem; both the CLI dropdown and programmatic
|
||||
# client pick this up.
|
||||
# Remote Ollama server. Unset uses http://localhost:11434/v1.
|
||||
#OLLAMA_BASE_URL=http://your-ollama-host:11434/v1
|
||||
|
||||
# Optional: override DEFAULT_CONFIG without editing code.
|
||||
# Any TRADINGAGENTS_* variable below, when set, replaces the matching key
|
||||
# in tradingagents/default_config.py. Values are coerced to the type of
|
||||
# the existing default (bool / int / str), so "true"/"3" work as expected.
|
||||
# In the CLI, setting the LLM provider / models / backend URL / language
|
||||
# also skips the matching interactive selection step (useful for
|
||||
# OpenAI-compatible endpoints like opencode or LM Studio, and unattended runs).
|
||||
# Override any DEFAULT_CONFIG key. In the CLI, a value set here skips its prompt.
|
||||
#TRADINGAGENTS_LLM_PROVIDER=openai
|
||||
#TRADINGAGENTS_DEEP_THINK_LLM=gpt-5.4
|
||||
#TRADINGAGENTS_QUICK_THINK_LLM=gpt-5.4-mini
|
||||
#TRADINGAGENTS_DEEP_THINK_LLM=gpt-5.6
|
||||
#TRADINGAGENTS_QUICK_THINK_LLM=gpt-5.6-luna
|
||||
#TRADINGAGENTS_LLM_BACKEND_URL=
|
||||
#TRADINGAGENTS_OUTPUT_LANGUAGE=English
|
||||
#TRADINGAGENTS_MAX_DEBATE_ROUNDS=1
|
||||
#TRADINGAGENTS_MAX_RISK_ROUNDS=1
|
||||
#TRADINGAGENTS_CHECKPOINT_ENABLED=false
|
||||
# Sampling temperature (lower = less run-to-run variation on models that
|
||||
# honor it). Unset leaves each provider at its default. See the README
|
||||
# "Reproducibility" note — no setting makes LLM output fully deterministic.
|
||||
|
||||
# Lower temperature means less run-to-run variation on models that honor it.
|
||||
#TRADINGAGENTS_TEMPERATURE=0.0
|
||||
# LLM SDK retry budget forwarded to every provider. Unset leaves each SDK at its
|
||||
# own default (usually 2). Raise it to ride out bursty 429 rate-limit throttling
|
||||
# on rate-limited deployments (e.g. Azure OpenAI) instead of aborting the run.
|
||||
|
||||
# Retry budget for every LLM SDK. Raise it to ride out 429 throttling.
|
||||
#TRADINGAGENTS_LLM_MAX_RETRIES=6
|
||||
# Cap on output tokens forwarded to every provider (Gemini's max_output_tokens
|
||||
# too). Unset leaves each provider at its default. Set it to bound a model that
|
||||
# emits unbounded reasoning/output and hangs or trips a gateway idle timeout.
|
||||
|
||||
# Cap output tokens to bound a model that runs long and trips a timeout.
|
||||
#TRADINGAGENTS_MAX_TOKENS=8192
|
||||
# Provider-specific reasoning/thinking depth (optional; unset = provider
|
||||
# default). Setting one also skips the matching interactive prompt.
|
||||
|
||||
# Reasoning depth per provider; setting one skips its prompt.
|
||||
#TRADINGAGENTS_OPENAI_REASONING_EFFORT=medium
|
||||
#TRADINGAGENTS_GOOGLE_THINKING_LEVEL=high
|
||||
#TRADINGAGENTS_ANTHROPIC_EFFORT=high
|
||||
|
||||
239
tests/test_sec_edgar.py
Normal file
239
tests/test_sec_edgar.py
Normal file
@@ -0,0 +1,239 @@
|
||||
"""SEC EDGAR fundamentals: statements as they were filed, not as they read today.
|
||||
|
||||
Every other fundamentals vendor serves the current value of a past period and
|
||||
cuts on the fiscal period end, so a run sees figures the company had not yet
|
||||
filed, and later restatements replace what was actually published. EDGAR carries
|
||||
the filing date of every fact, so a run can be limited to what was on file by its
|
||||
own date.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from tradingagents.dataflows import sec_edgar
|
||||
from tradingagents.dataflows.errors import NoMarketDataError
|
||||
|
||||
_REAL_FETCH = sec_edgar._fetch_json
|
||||
|
||||
TICKER_MAP = {"0": {"cik_str": 320193, "ticker": "AAPL", "title": "Apple Inc."}}
|
||||
|
||||
|
||||
def _fact(end, val, filed, form="10-K", fp="FY", start=None):
|
||||
fact = {"end": end, "val": val, "filed": filed, "form": form, "fy": int(end[:4]), "fp": fp}
|
||||
if start:
|
||||
fact["start"] = start
|
||||
return fact
|
||||
|
||||
|
||||
FACTS = {
|
||||
"cik": 320193,
|
||||
"entityName": "Apple Inc.",
|
||||
"facts": {"us-gaap": {
|
||||
"Assets": {"units": {"USD": [
|
||||
_fact("2008-09-27", 39_572_000_000, "2008-11-05"),
|
||||
_fact("2008-09-27", 36_171_000_000, "2010-01-25", form="10-K/A"),
|
||||
_fact("2022-03-26", 350_662_000_000, "2022-04-29", form="10-Q", fp="Q2"),
|
||||
_fact("2024-09-28", 364_980_000_000, "2024-11-01"),
|
||||
]}},
|
||||
"Liabilities": {"units": {"USD": [_fact("2024-09-28", 308_030_000_000, "2024-11-01")]}},
|
||||
"EarningsPerShareDiluted": {"units": {"USD/shares": [
|
||||
_fact("2024-09-28", 6.08, "2024-11-01", start="2023-09-30"),
|
||||
]}},
|
||||
"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
|
||||
# One filing reports the quarter and the year to date under one end date.
|
||||
_fact("2025-12-31", 81_300_000_000, "2026-01-29", form="10-Q", fp="Q2", start="2025-10-01"),
|
||||
_fact("2025-12-31", 158_900_000_000, "2026-01-29", form="10-Q", fp="Q2", start="2025-07-01"),
|
||||
_fact("2024-09-28", 391_035_000_000, "2024-11-01", start="2023-09-30"),
|
||||
]}},
|
||||
}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_network_or_cache(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(sec_edgar, "get_config", lambda: {"data_cache_dir": str(tmp_path)})
|
||||
monkeypatch.setattr(sec_edgar, "_fetch_json", lambda url: TICKER_MAP if "company_tickers" in url else FACTS)
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_us_filer_resolves_to_its_cik():
|
||||
assert sec_edgar.cik_for("AAPL") == "0000320193"
|
||||
assert sec_edgar.cik_for("aapl") == "0000320193"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_non_filer_is_reported_as_such_not_as_missing_data():
|
||||
with pytest.raises(NoMarketDataError, match="not a US SEC filer"):
|
||||
sec_edgar.get_balance_sheet("0700.HK", "annual", "2026-01-01")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_restated_figure_reads_as_it_did_at_the_time():
|
||||
"""The value published then, not the correction filed later."""
|
||||
as_filed = sec_edgar.get_balance_sheet("AAPL", "annual", "2009-06-30")
|
||||
restated = sec_edgar.get_balance_sheet("AAPL", "annual", "2011-01-01")
|
||||
assert "39572" in as_filed and "36171" not in as_filed
|
||||
assert "36171" in restated
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_period_that_ended_but_was_not_filed_yet_is_not_served():
|
||||
"""The fiscal year ended 2024-09-28; it reached the public on 2024-11-01."""
|
||||
before = sec_edgar.get_balance_sheet("AAPL", "annual", "2024-10-15")
|
||||
after = sec_edgar.get_balance_sheet("AAPL", "annual", "2024-11-15")
|
||||
assert "2024-09-28" not in before
|
||||
assert "2024-09-28" in after and "364980" in after
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_quarter_is_not_confused_with_the_year_to_date():
|
||||
"""One filing carries both spans under the same end date (#MSFT-shaped)."""
|
||||
out = sec_edgar.get_income_statement("AAPL", "quarterly", "2026-06-01")
|
||||
assert "81300" in out
|
||||
assert "158900" not in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_line_the_filer_does_not_tag_is_named_unavailable():
|
||||
out = sec_edgar.get_balance_sheet("AAPL", "annual", "2024-11-15")
|
||||
assert "Stockholders Equity" in out and "unavailable" in out
|
||||
assert "364980" in out # the rest of the statement still returns
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_report_states_the_vintage_rule():
|
||||
out = sec_edgar.get_balance_sheet("AAPL", "annual", "2024-11-15")
|
||||
assert "filed on or before 2024-11-15" in out
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_filer_with_no_usable_facts_reads_differently_from_a_non_filer(monkeypatch):
|
||||
monkeypatch.setattr(sec_edgar, "_fetch_json",
|
||||
lambda url: TICKER_MAP if "company_tickers" in url else {"facts": {}})
|
||||
with pytest.raises(NoMarketDataError, match="no us-gaap facts"):
|
||||
sec_edgar.get_balance_sheet("AAPL", "annual", "2026-01-01")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_company_facts_are_fetched_once_per_company_not_once_per_date(tmp_path, monkeypatch):
|
||||
"""A sweep asks for many dates; the filing history is the same file."""
|
||||
calls = []
|
||||
monkeypatch.setattr(sec_edgar, "_fetch_json",
|
||||
lambda url: calls.append(url) or (TICKER_MAP if "company_tickers" in url else FACTS))
|
||||
for date in ("2024-11-15", "2025-01-15", "2025-06-15"):
|
||||
sec_edgar.get_balance_sheet("AAPL", "annual", date)
|
||||
assert len([u for u in calls if "companyfacts" in u]) == 1
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_it_works_unconfigured_and_takes_the_caller_s_own_contact(monkeypatch):
|
||||
"""SEC returns 403 for a User-Agent with no contact address, so the default
|
||||
carries one; a caller who sets their own replaces it."""
|
||||
monkeypatch.delenv("SEC_EDGAR_USER_AGENT", raising=False)
|
||||
assert "@" in sec_edgar._user_agent()
|
||||
|
||||
monkeypatch.setenv("SEC_EDGAR_USER_AGENT", "MyDesk research@example.com")
|
||||
assert sec_edgar._user_agent() == "MyDesk research@example.com"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_throttle_lets_the_next_vendor_try(monkeypatch):
|
||||
"""SEC throttles by refusing the request; the router then tries yfinance."""
|
||||
import requests
|
||||
|
||||
from tradingagents.dataflows.errors import VendorRateLimitError
|
||||
|
||||
def _throttled(*a, **k):
|
||||
raise requests.HTTPError(response=mock.Mock(status_code=429))
|
||||
|
||||
monkeypatch.setattr(sec_edgar.requests, "get", _throttled)
|
||||
with pytest.raises(VendorRateLimitError):
|
||||
_REAL_FETCH("https://data.sec.gov/api/xbrl/companyfacts/CIK0000320193.json")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_values_do_not_break_the_columns():
|
||||
"""Figures run to the billions; a thousands separator would split the field."""
|
||||
out = sec_edgar.get_balance_sheet("AAPL", "annual", "2024-11-15")
|
||||
body = [row for row in out.splitlines() if row.startswith("Total Assets")][0]
|
||||
assert body.count(",") == out.splitlines()[3].count(",")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_per_share_figure_keeps_its_own_unit():
|
||||
"""Statements are reported in millions, but EPS is dollars per share: scaling
|
||||
it the same way prints a real figure as zero."""
|
||||
out = sec_edgar.get_income_statement("AAPL", "annual", "2024-11-15")
|
||||
row = [r for r in out.splitlines() if r.startswith("Diluted EPS")][0]
|
||||
assert "6.08" in row
|
||||
assert "USD/shares" in row or "per share" in row
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_every_row_has_one_cell_per_period():
|
||||
"""An untagged line still has to line up with the columns, or the table is
|
||||
misread by position."""
|
||||
out = sec_edgar.get_balance_sheet("AAPL", "annual", "2024-11-15")
|
||||
table = [r for r in out.splitlines() if r and not r.startswith("#")]
|
||||
widths = {row.count(",") for row in table}
|
||||
assert len(widths) == 1, table
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_server_error_lets_the_next_vendor_try(monkeypatch):
|
||||
import requests
|
||||
|
||||
from tradingagents.dataflows.errors import VendorError
|
||||
|
||||
def _server_error(*a, **k):
|
||||
raise requests.HTTPError(response=mock.Mock(status_code=503))
|
||||
|
||||
monkeypatch.setattr(sec_edgar.requests, "get", _server_error)
|
||||
with pytest.raises(VendorError): # not a bare HTTPError
|
||||
_REAL_FETCH("https://data.sec.gov/api/xbrl/companyfacts/CIK0000320193.json")
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_older_periods_fall_back_to_the_tag_the_filer_used_then():
|
||||
"""Filers renamed lines when the revenue standard changed, so one tag covers
|
||||
only recent years. Each period takes one tag, never a sum of two."""
|
||||
facts = {"Revenues": {"units": {"USD": [_fact("2015-09-26", 233_715_000_000, "2015-10-28",
|
||||
start="2014-09-28")]}},
|
||||
"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
|
||||
_fact("2024-09-28", 391_035_000_000, "2024-11-01", start="2023-09-30")]}}}
|
||||
values, unit = sec_edgar._as_of(facts, ("RevenueFromContractWithCustomerExcludingAssessedTax",
|
||||
"Revenues"), "2026-01-01", (300, 400))
|
||||
assert values == {"2015-09-26": 233_715_000_000, "2024-09-28": 391_035_000_000}
|
||||
assert unit == "USD"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_a_period_reported_under_two_tags_takes_the_preferred_one_not_both():
|
||||
facts = {"Revenues": {"units": {"USD": [_fact("2024-09-28", 111, "2024-11-01", start="2023-09-30")]}},
|
||||
"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": [
|
||||
_fact("2024-09-28", 999, "2024-11-01", start="2023-09-30")]}}}
|
||||
values, _ = sec_edgar._as_of(facts, ("RevenueFromContractWithCustomerExcludingAssessedTax",
|
||||
"Revenues"), "2026-01-01", (300, 400))
|
||||
assert values == {"2024-09-28": 999}
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_the_default_identification_tracks_the_installed_version(monkeypatch):
|
||||
"""A release should identify itself, not a version frozen in the source."""
|
||||
monkeypatch.delenv("SEC_EDGAR_USER_AGENT", raising=False)
|
||||
monkeypatch.setattr(sec_edgar.metadata, "version", lambda name: "9.9.9")
|
||||
assert sec_edgar._user_agent() == "TradingAgents/9.9.9 (contact@example.com)"
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
def test_an_uninstalled_checkout_still_identifies_itself(monkeypatch):
|
||||
monkeypatch.delenv("SEC_EDGAR_USER_AGENT", raising=False)
|
||||
|
||||
def _missing(name):
|
||||
raise sec_edgar.metadata.PackageNotFoundError(name)
|
||||
|
||||
monkeypatch.setattr(sec_edgar.metadata, "version", _missing)
|
||||
assert "@" in sec_edgar._user_agent()
|
||||
@@ -19,6 +19,11 @@ from .errors import (
|
||||
)
|
||||
from .fred import get_macro_data as get_fred_macro_data
|
||||
from .polymarket import get_prediction_markets as get_polymarket_prediction_markets
|
||||
from .sec_edgar import (
|
||||
get_balance_sheet as get_sec_edgar_balance_sheet,
|
||||
get_cashflow as get_sec_edgar_cashflow,
|
||||
get_income_statement as get_sec_edgar_income_statement,
|
||||
)
|
||||
from .y_finance import (
|
||||
get_balance_sheet as get_yfinance_balance_sheet,
|
||||
get_cashflow as get_yfinance_cashflow,
|
||||
@@ -79,6 +84,7 @@ TOOLS_CATEGORIES = {
|
||||
|
||||
VENDOR_LIST = [
|
||||
"yfinance",
|
||||
"sec_edgar",
|
||||
"fred",
|
||||
"polymarket",
|
||||
"alpha_vantage",
|
||||
@@ -110,14 +116,17 @@ VENDOR_METHODS = {
|
||||
},
|
||||
"get_balance_sheet": {
|
||||
"alpha_vantage": get_alpha_vantage_balance_sheet,
|
||||
"sec_edgar": get_sec_edgar_balance_sheet,
|
||||
"yfinance": get_yfinance_balance_sheet,
|
||||
},
|
||||
"get_cashflow": {
|
||||
"alpha_vantage": get_alpha_vantage_cashflow,
|
||||
"sec_edgar": get_sec_edgar_cashflow,
|
||||
"yfinance": get_yfinance_cashflow,
|
||||
},
|
||||
"get_income_statement": {
|
||||
"alpha_vantage": get_alpha_vantage_income_statement,
|
||||
"sec_edgar": get_sec_edgar_income_statement,
|
||||
"yfinance": get_yfinance_income_statement,
|
||||
},
|
||||
# news_data
|
||||
|
||||
229
tradingagents/dataflows/sec_edgar.py
Normal file
229
tradingagents/dataflows/sec_edgar.py
Normal file
@@ -0,0 +1,229 @@
|
||||
"""Company statements as they were filed, from SEC EDGAR.
|
||||
|
||||
Every other fundamentals vendor serves a period's current value and cuts the
|
||||
statement at the fiscal period end. That is two claims a run should not make: a
|
||||
period that has ended is not public until the company files, weeks later, and a
|
||||
figure that was later restated is not what investors saw at the time.
|
||||
|
||||
EDGAR reports every fact with the date it was filed, so a run dated ``curr_date``
|
||||
serves exactly what was on file by then, restatements included at the vintage
|
||||
that was current. Apple's 2008 total assets read as 39.6B until the 2010
|
||||
amendment; a 2009 run should see 39.6B, and does.
|
||||
|
||||
Access needs no key or account, only a User-Agent identifying the caller, which
|
||||
SEC requires and refuses requests without. US filers only: anything absent from
|
||||
EDGAR's ticker map falls through to the next configured vendor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import date, datetime
|
||||
from importlib import metadata
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from .config import get_config
|
||||
from .errors import NoMarketDataError, VendorRateLimitError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_TICKERS_URL = "https://www.sec.gov/files/company_tickers.json"
|
||||
_FACTS_URL = "https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json"
|
||||
|
||||
# A filing history only changes when something new is filed, so one fetch per
|
||||
# company per day serves every date a run asks about.
|
||||
_CACHE_TTL_SECONDS = 24 * 60 * 60
|
||||
|
||||
# Line items, each with the tags filers use for it, best first. First match wins
|
||||
# and values are never summed across tags: a company reporting revenue under two
|
||||
# tags would otherwise be counted twice.
|
||||
_STATEMENTS: dict[str, list[tuple[str, tuple[str, ...]]]] = {
|
||||
"balance_sheet": [
|
||||
("Total Assets", ("Assets",)),
|
||||
("Current Assets", ("AssetsCurrent",)),
|
||||
("Cash and Equivalents", ("CashAndCashEquivalentsAtCarryingValue",)),
|
||||
("Total Liabilities", ("Liabilities",)),
|
||||
("Current Liabilities", ("LiabilitiesCurrent",)),
|
||||
("Stockholders Equity", ("StockholdersEquity",
|
||||
"StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest")),
|
||||
],
|
||||
"income_statement": [
|
||||
("Revenue", ("RevenueFromContractWithCustomerExcludingAssessedTax", "Revenues",
|
||||
"SalesRevenueNet")),
|
||||
("Cost of Revenue", ("CostOfRevenue", "CostOfGoodsAndServicesSold")),
|
||||
("Gross Profit", ("GrossProfit",)),
|
||||
("Operating Income", ("OperatingIncomeLoss",)),
|
||||
("Net Income", ("NetIncomeLoss",)),
|
||||
("Diluted EPS", ("EarningsPerShareDiluted",)),
|
||||
],
|
||||
"cashflow": [
|
||||
("Operating Cash Flow", ("NetCashProvidedByUsedInOperatingActivities",
|
||||
"NetCashProvidedByUsedInOperatingActivitiesContinuingOperations")),
|
||||
("Investing Cash Flow", ("NetCashProvidedByUsedInInvestingActivities",)),
|
||||
("Financing Cash Flow", ("NetCashProvidedByUsedInFinancingActivities",)),
|
||||
("Capital Expenditure", ("PaymentsToAcquirePropertyPlantAndEquipment",)),
|
||||
],
|
||||
}
|
||||
|
||||
# A statement's figures cover a span: a quarter is about 90 days, a year about
|
||||
# 365. One filing reports both the quarter and the year to date under the same
|
||||
# end date, so a match on the end date alone can report half a year as a quarter.
|
||||
_SPANS = {"quarterly": (60, 115), "annual": (300, 400)}
|
||||
|
||||
|
||||
def _user_agent() -> str:
|
||||
"""Who SEC sees. No account or key exists; callers identify themselves.
|
||||
|
||||
www.sec.gov, which serves the ticker map, refuses a User-Agent carrying no
|
||||
contact address: a client name alone or with a project URL gets 403, one
|
||||
with an address gets 200. So the default carries a placeholder address and
|
||||
the package version. Set SEC_EDGAR_USER_AGENT to your own name and address
|
||||
so SEC can reach you about your traffic rather than the project.
|
||||
"""
|
||||
configured = os.getenv("SEC_EDGAR_USER_AGENT", "").strip()
|
||||
return configured or f"TradingAgents/{_version()} (contact@example.com)"
|
||||
|
||||
|
||||
def _version() -> str:
|
||||
"""The installed package version, so a release identifies itself correctly."""
|
||||
try:
|
||||
return metadata.version("tradingagents")
|
||||
except metadata.PackageNotFoundError:
|
||||
return "dev"
|
||||
|
||||
|
||||
def _fetch_json(url: str) -> dict:
|
||||
"""Read a public EDGAR document, respecting SEC's identification rule."""
|
||||
try:
|
||||
response = requests.get(url, headers={"User-Agent": _user_agent()}, timeout=30)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except requests.RequestException as exc:
|
||||
status = getattr(getattr(exc, "response", None), "status_code", None)
|
||||
# Every failure here is "this vendor cannot serve it now", so the router
|
||||
# moves on instead of seeing a transport exception it has no rule for.
|
||||
raise VendorRateLimitError(f"SEC EDGAR request failed ({status or type(exc).__name__})") from exc
|
||||
except ValueError as exc:
|
||||
raise VendorRateLimitError("SEC EDGAR returned an unreadable response") from exc
|
||||
|
||||
|
||||
def _cached_json(url: str, name: str) -> dict:
|
||||
path = Path(get_config()["data_cache_dir"]) / "sec_edgar" / name
|
||||
if path.exists() and time.time() - path.stat().st_mtime < _CACHE_TTL_SECONDS:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except ValueError:
|
||||
pass # a truncated file is a miss, not a failure
|
||||
data = _fetch_json(url)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp = path.with_suffix(".tmp")
|
||||
temp.write_text(json.dumps(data), encoding="utf-8")
|
||||
os.replace(temp, path)
|
||||
return data
|
||||
|
||||
|
||||
def cik_for(ticker: str) -> str | None:
|
||||
"""The filer's CIK, or None when the ticker is not a US filer."""
|
||||
table = _cached_json(_TICKERS_URL, "company_tickers.json")
|
||||
wanted = ticker.strip().upper()
|
||||
for entry in table.values():
|
||||
if entry.get("ticker", "").upper() == wanted:
|
||||
return f"{int(entry['cik_str']):010d}"
|
||||
return None
|
||||
|
||||
|
||||
def _as_of(facts: dict, tags: tuple[str, ...], curr_date: str, span: tuple[int, int]) -> tuple[dict, str]:
|
||||
"""({period end: value}, unit) for the first tag the filer reports, as known then.
|
||||
|
||||
A period reported more than once takes its latest filing on or before the
|
||||
date, so an amendment counts from the day it was filed and not before. The
|
||||
unit comes from the filing: most lines are USD, earnings per share are
|
||||
USD/shares, and scaling those alike would print a real figure as zero.
|
||||
"""
|
||||
low, high = span
|
||||
values: dict[str, float] = {}
|
||||
chosen_unit = "USD"
|
||||
# Tags are tried in order and a period keeps the first one that reports it:
|
||||
# filers renamed lines over the years, so one tag covers only part of the
|
||||
# history. Values are never added across tags, which would double count.
|
||||
for tag in tags:
|
||||
for unit, unit_values in ((facts.get(tag) or {}).get("units", {})).items():
|
||||
latest: dict[str, dict] = {}
|
||||
for fact in unit_values:
|
||||
if fact["filed"] > curr_date or fact["end"] in values:
|
||||
continue
|
||||
# A duration fact (revenue, cash flow) must cover the span asked
|
||||
# for. An instant fact (a balance) has no span and serves both.
|
||||
if "start" in fact:
|
||||
days = (date.fromisoformat(fact["end"]) - date.fromisoformat(fact["start"])).days
|
||||
if not low <= days <= high:
|
||||
continue
|
||||
seen = latest.get(fact["end"])
|
||||
if seen is None or fact["filed"] >= seen["filed"]:
|
||||
latest[fact["end"]] = fact
|
||||
if latest:
|
||||
chosen_unit = unit
|
||||
values.update({end: fact["val"] for end, fact in latest.items()})
|
||||
return dict(sorted(values.items())), chosen_unit
|
||||
|
||||
|
||||
def _statement(kind: str, ticker: str, freq: str, curr_date: str, title: str) -> str:
|
||||
curr_date = curr_date or datetime.now().strftime("%Y-%m-%d")
|
||||
cik = cik_for(ticker)
|
||||
if cik is None:
|
||||
raise NoMarketDataError(ticker, ticker, "not a US SEC filer")
|
||||
|
||||
facts = _cached_json(_FACTS_URL.format(cik=cik), f"CIK{cik}.json")
|
||||
us_gaap = (facts.get("facts") or {}).get("us-gaap")
|
||||
if not us_gaap:
|
||||
raise NoMarketDataError(ticker, ticker, "US filer with no us-gaap facts")
|
||||
|
||||
span = _SPANS["quarterly" if freq.lower() == "quarterly" else "annual"]
|
||||
lines = {label: _as_of(us_gaap, tags, curr_date, span) for label, tags in _STATEMENTS[kind]}
|
||||
periods = sorted({end for values, _ in lines.values() for end in values})
|
||||
if not periods:
|
||||
raise NoMarketDataError(ticker, ticker, f"no {freq} {title.lower()} filed by {curr_date}")
|
||||
|
||||
header = (
|
||||
f"# {title} for {ticker.upper()} ({freq}), USD in millions unless the row says otherwise\n"
|
||||
f"# SEC EDGAR facts filed on or before {curr_date}, at the values filed then\n\n"
|
||||
)
|
||||
rows = [",".join([""] + periods)]
|
||||
for label, (values, unit) in lines.items():
|
||||
# Every row spans the same columns, or a reader lines the table up wrong.
|
||||
if not values:
|
||||
rows.append(",".join([label] + ["unavailable (not tagged by this filer)"] * len(periods)))
|
||||
continue
|
||||
name = label if unit == "USD" else f"{label} ({unit})"
|
||||
# Plain numbers: a thousands separator would split the CSV field.
|
||||
cells = [
|
||||
(f"{values[p] / 1e6:.0f}" if unit == "USD" else f"{values[p]:.2f}")
|
||||
if p in values else "" for p in periods
|
||||
]
|
||||
rows.append(",".join([name] + cells))
|
||||
return header + "\n".join(rows) + "\n"
|
||||
|
||||
|
||||
def get_balance_sheet(ticker: str, freq: str = "quarterly", curr_date: str | None = None) -> str:
|
||||
"""Balance sheet as filed on or before ``curr_date``."""
|
||||
return _statement("balance_sheet", ticker, freq, curr_date, "Balance Sheet")
|
||||
|
||||
|
||||
def get_income_statement(ticker: str, freq: str = "quarterly", curr_date: str | None = None) -> str:
|
||||
"""Income statement as filed on or before ``curr_date``.
|
||||
|
||||
A fourth quarter is never derived: filers report it only inside the annual
|
||||
figure, and subtracting three separately filed quarters would invent a number
|
||||
with no filing date behind it.
|
||||
"""
|
||||
return _statement("income_statement", ticker, freq, curr_date, "Income Statement")
|
||||
|
||||
|
||||
def get_cashflow(ticker: str, freq: str = "quarterly", curr_date: str | None = None) -> str:
|
||||
"""Cash flow statement as filed on or before ``curr_date``."""
|
||||
return _statement("cashflow", ticker, freq, curr_date, "Cash Flow Statement")
|
||||
Reference in New Issue
Block a user