feat(analytics): доходы, ребалансировка, налоги, бенчмарки и цели — фаза 4
Второй источник выплат: sources/tinvest/sync_events.py (GetDividends, GetBondCoupons, GetBondEvents) и sources/moex/payouts.py (ISS bondization + dividends). Приоритет между ними — pricing/payouts.resolve_payouts, решается на чтении, а не на записи: corporate_action уникален по (instrument_id, kind, source, source_id), обе версии сосуществуют, и правило можно поменять без ресинка истории. Амортизация от MOEX идёт в bond_nominal_schedule, а не в corporate_action — этим типом безраздельно владеет ledger/corporate_actions.py. analytics/income.py — metric_income_monthly (факт) и metric_income_calendar (прошлое и прогноз) с basis paid/announced/history на каждой строке, три источника числа не смешиваются. analytics/rebalance.py — сделки по portfolio_target пропорционально внутри бакета, лоты только вниз, покупки не занимают у ещё не свершившихся продаж. analytics/tax.py — оценка, не замена справки брокера: дивиденды/купоны gross, реализованный результат из lot_disposal с переоценкой каждой ноги на свою дату. analytics/benchmarks.py — TWR индекса на сетке портфеля, kind (price/total_return) не скрывается. analytics/goals.py — прогресс цели и нужный взнос по trailing XIRR. Четыре шага зарегистрированы в register_steps: benchmarks после returns (общая сетка дат), rebalance после allocation (её веса, не пересчитывает), income и tax после lots (нужен lot_disposal).
This commit is contained in:
Dmitry
@@ -1,8 +1,10 @@
|
||||
"""The `moex` source: prices and bond schedules from the MOEX ISS."""
|
||||
|
||||
from fintracker.sources.moex.payouts import MoexPayoutsSource
|
||||
from fintracker.sources.moex.sync import MoexSource
|
||||
from fintracker.sources.registry import register
|
||||
|
||||
register(MoexSource())
|
||||
register(MoexPayoutsSource())
|
||||
|
||||
__all__ = ["MoexSource"]
|
||||
__all__ = ["MoexPayoutsSource", "MoexSource"]
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
"""The `moex_payouts` source: the second payout feed, from MOEX ISS (plan §Фаза 4).
|
||||
|
||||
Two endpoints, both free and keyless:
|
||||
|
||||
* `/iss/securities/{secid}/bondization.json` — the bond's registered schedule: every coupon
|
||||
to maturity and the amortisation plan. `MoexClient.bondization` already reads it.
|
||||
* `/iss/securities/{secid}/dividends.json` — the dividend register extract for a share.
|
||||
Only this module uses it, so the request lives here rather than on the client.
|
||||
|
||||
What it writes, and what it deliberately does not:
|
||||
|
||||
* coupons -> `corporate_action(kind=coupon, source='moex')`, alongside the T-Invest rows
|
||||
for the same coupons rather than instead of them. Which one an analytic reads is decided
|
||||
by `pricing/payouts.resolve_payouts`, at read time — see that module for why.
|
||||
* dividends -> `corporate_action(kind=dividend, source='moex')`, same arrangement.
|
||||
* amortisations -> `bond_nominal_schedule(source='moex')`, and NOT
|
||||
`corporate_action(kind=amortization)`: that kind belongs to `ledger/corporate_actions.py`,
|
||||
whose prune deletes every row in it the ledger does not imply.
|
||||
|
||||
**The amortisation plan is read as a run-out, not as a column.** ISS states `value` (repaid
|
||||
per bond) and `facevalue` per row, but which side of the payment `facevalue` stands on is not
|
||||
documented and differs between papers. Summing what is still to be repaid is unambiguous:
|
||||
everything the issuer will ever repay per bond is the nominal, so the nominal standing after
|
||||
a given amortisation is the sum of those after it. `facevalue` is used only as a cross-check,
|
||||
and a mismatch is a warning rather than a different answer.
|
||||
|
||||
**There is no raw tier here.** `raw_moex_doc` (plan §1.1) does not exist yet, and the MOEX
|
||||
source has never had one: ISS answers are cheap to re-fetch, unlike the rate-limited
|
||||
T-Invest feeds the raw tables exist for.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import today_local
|
||||
from fintracker.models import AssetClass, Event, EventStatus, Instrument
|
||||
from fintracker.models.pricing import CorporateActionKind, CorporateActionStatus
|
||||
from fintracker.pricing.payouts import (
|
||||
MOEX,
|
||||
NominalPoint,
|
||||
PayoutRow,
|
||||
upsert_nominals,
|
||||
upsert_payouts,
|
||||
)
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.sources.moex.client import (
|
||||
BASE,
|
||||
AmortisationRow,
|
||||
CouponRow,
|
||||
MoexClient,
|
||||
MoexError,
|
||||
_currency,
|
||||
_date,
|
||||
_decimal,
|
||||
_rows,
|
||||
new_http_client,
|
||||
)
|
||||
from fintracker.sources.moex.sync import _secid_candidates
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = MOEX
|
||||
NAME = "moex_payouts"
|
||||
|
||||
BOND_CLASSES = frozenset({AssetClass.bond})
|
||||
DIVIDEND_CLASSES = frozenset({AssetClass.share, AssetClass.etf, AssetClass.fund})
|
||||
|
||||
ZERO = Decimal(0)
|
||||
NOMINAL_TOLERANCE = Decimal("0.01")
|
||||
"""Below this the run-out and the stated face value are rounding each other."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MoexDividendRow:
|
||||
"""One row of `/iss/securities/{secid}/dividends.json`.
|
||||
|
||||
The extract is a register snapshot: it states when the register closed and how much per
|
||||
share was declared, and nothing about when the money is actually paid — which is the
|
||||
gap T-Invest fills, and one of the reasons dividends are read from T-Invest first.
|
||||
"""
|
||||
|
||||
secid: str
|
||||
registry_close_date: date | None
|
||||
value: Decimal | None
|
||||
currency: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Target:
|
||||
instrument_id: int
|
||||
secid: str
|
||||
asset_class: AssetClass
|
||||
currency: str
|
||||
|
||||
|
||||
async def fetch_dividends(client: httpx.AsyncClient, secid: str) -> list[MoexDividendRow]:
|
||||
"""The dividend register extract for one paper; an unlisted paper answers with nothing."""
|
||||
response = await client.get(
|
||||
f"{BASE}/securities/{secid}/dividends.json", params={"iss.meta": "off"}
|
||||
)
|
||||
if response.status_code == httpx.codes.NOT_FOUND:
|
||||
raise MoexError(f"dividends for {secid} not found")
|
||||
response.raise_for_status()
|
||||
return [
|
||||
MoexDividendRow(
|
||||
secid=str(row.get("secid") or secid),
|
||||
registry_close_date=_date(row.get("registryclosedate")),
|
||||
value=_decimal(row.get("value")),
|
||||
currency=_currency(row.get("currencyid")),
|
||||
)
|
||||
for row in _rows(response.json(), "dividends")
|
||||
]
|
||||
|
||||
|
||||
def coupon_payout(row: CouponRow, *, instrument_id: int, today: date) -> PayoutRow | None:
|
||||
"""One bondization coupon as a `corporate_action` row.
|
||||
|
||||
A coupon whose rate is not fixed yet arrives dated but with no `value`; it is kept, with
|
||||
a NULL amount, because the date is real and the calendar needs it. `resolve_payouts`
|
||||
treats a missing amount as a gap rather than a disagreement for exactly this case.
|
||||
"""
|
||||
if row.coupon_date is None:
|
||||
return None
|
||||
return PayoutRow(
|
||||
instrument_id=instrument_id,
|
||||
kind=CorporateActionKind.coupon,
|
||||
status=_status(row.coupon_date, today),
|
||||
source=SOURCE,
|
||||
source_id=f"cpn:{row.coupon_date.isoformat()}",
|
||||
pay_date=row.coupon_date,
|
||||
amount_per_unit=row.value,
|
||||
currency=(row.currency or "").upper() or None,
|
||||
)
|
||||
|
||||
|
||||
def dividend_payout(row: MoexDividendRow, *, instrument_id: int, today: date) -> PayoutRow | None:
|
||||
"""One register row as a `corporate_action` row.
|
||||
|
||||
The register-closing date is the record date, and it is the only date stated — so it is
|
||||
also what a date-based merge with the T-Invest row keys on.
|
||||
"""
|
||||
if row.registry_close_date is None:
|
||||
return None
|
||||
return PayoutRow(
|
||||
instrument_id=instrument_id,
|
||||
kind=CorporateActionKind.dividend,
|
||||
status=_status(row.registry_close_date, today),
|
||||
source=SOURCE,
|
||||
source_id=f"div:{row.registry_close_date.isoformat()}",
|
||||
record_date=row.registry_close_date,
|
||||
amount_per_unit=row.value,
|
||||
currency=(row.currency or "").upper() or None,
|
||||
)
|
||||
|
||||
|
||||
def nominal_schedule(
|
||||
amortisations: Sequence[AmortisationRow],
|
||||
*,
|
||||
instrument_id: int,
|
||||
currency: str,
|
||||
warnings: list[str],
|
||||
secid: str = "",
|
||||
) -> list[NominalPoint]:
|
||||
"""The nominal standing after each amortisation, run out from the total still to repay."""
|
||||
usable = sorted(
|
||||
(a for a in amortisations if a.amort_date is not None and a.value is not None),
|
||||
key=lambda a: a.amort_date or date.min,
|
||||
)
|
||||
if not usable:
|
||||
return []
|
||||
|
||||
total = sum((a.value or ZERO for a in usable), start=ZERO)
|
||||
stated = max((a.face_value for a in usable if a.face_value is not None), default=None)
|
||||
if stated is not None and abs(stated - total) > NOMINAL_TOLERANCE:
|
||||
warnings.append(
|
||||
f"{secid}: сумма амортизаций {total} расходится с номиналом {stated} "
|
||||
"— график построен по сумме выплат"
|
||||
)
|
||||
|
||||
remaining = total
|
||||
points: list[NominalPoint] = []
|
||||
for row in usable:
|
||||
remaining -= row.value or ZERO
|
||||
points.append(
|
||||
NominalPoint(
|
||||
instrument_id=instrument_id,
|
||||
effective_date=row.amort_date or date.min,
|
||||
nominal=remaining,
|
||||
currency=(row.currency or currency).upper(),
|
||||
source=SOURCE,
|
||||
)
|
||||
)
|
||||
return points
|
||||
|
||||
|
||||
def _status(day: date, today: date) -> CorporateActionStatus:
|
||||
"""Past dates are facts the issuer has settled; future ones are announcements."""
|
||||
return CorporateActionStatus.paid if day < today else CorporateActionStatus.announced
|
||||
|
||||
|
||||
class MoexPayoutsSource:
|
||||
"""`Source` protocol, same shape as `MoexSource`. Registration is done elsewhere."""
|
||||
|
||||
name = NAME
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult:
|
||||
session = ctx.session
|
||||
today = today_local()
|
||||
targets = await load_targets(session)
|
||||
if not targets:
|
||||
log.info("moex_payouts: no priceable instruments in the ledger yet")
|
||||
return SyncResult(cursor_after=today.isoformat(), counts={"payouts": 0}, changed=False)
|
||||
|
||||
counts = {"instruments": 0, "coupons": 0, "dividends": 0, "payouts": 0, "nominals": 0}
|
||||
warnings: list[str] = []
|
||||
payouts: list[PayoutRow] = []
|
||||
nominals: list[NominalPoint] = []
|
||||
|
||||
async with new_http_client() as http, MoexClient(http) as moex:
|
||||
for target in targets:
|
||||
counts["instruments"] += 1
|
||||
if target.asset_class in BOND_CLASSES:
|
||||
rows = await self._bond(moex, target, today, payouts, nominals, warnings)
|
||||
counts["coupons"] += rows
|
||||
else:
|
||||
counts["dividends"] += await self._dividends(
|
||||
http, target, today, payouts, warnings
|
||||
)
|
||||
|
||||
counts["payouts"] = await upsert_payouts(session, payouts)
|
||||
counts["nominals"] = await upsert_nominals(session, nominals)
|
||||
await session.commit()
|
||||
|
||||
log.info(
|
||||
"moex_payouts: %s instruments, %s payouts, %s nominal points",
|
||||
counts["instruments"],
|
||||
counts["payouts"],
|
||||
counts["nominals"],
|
||||
)
|
||||
return SyncResult(
|
||||
cursor_after=today.isoformat(),
|
||||
counts=counts,
|
||||
warnings=warnings,
|
||||
changed=bool(counts["payouts"] or counts["nominals"]),
|
||||
)
|
||||
|
||||
async def _bond(
|
||||
self,
|
||||
moex: MoexClient,
|
||||
target: Target,
|
||||
today: date,
|
||||
payouts: list[PayoutRow],
|
||||
nominals: list[NominalPoint],
|
||||
warnings: list[str],
|
||||
) -> int:
|
||||
coupons: list[CouponRow] = []
|
||||
amortisations: list[AmortisationRow] = []
|
||||
for secid in _secid_candidates(target.secid):
|
||||
try:
|
||||
coupons, amortisations = await moex.bondization(secid)
|
||||
except (MoexError, httpx.HTTPError) as err:
|
||||
warnings.append(f"{secid}: {err}")
|
||||
continue
|
||||
if coupons or amortisations:
|
||||
break
|
||||
payouts += [
|
||||
payout
|
||||
for coupon in coupons
|
||||
if (payout := coupon_payout(coupon, instrument_id=target.instrument_id, today=today))
|
||||
]
|
||||
nominals += nominal_schedule(
|
||||
amortisations,
|
||||
instrument_id=target.instrument_id,
|
||||
currency=target.currency,
|
||||
warnings=warnings,
|
||||
secid=target.secid,
|
||||
)
|
||||
return len(coupons)
|
||||
|
||||
async def _dividends(
|
||||
self,
|
||||
http: httpx.AsyncClient,
|
||||
target: Target,
|
||||
today: date,
|
||||
payouts: list[PayoutRow],
|
||||
warnings: list[str],
|
||||
) -> int:
|
||||
rows: list[MoexDividendRow] = []
|
||||
for secid in _secid_candidates(target.secid):
|
||||
try:
|
||||
rows = await fetch_dividends(http, secid)
|
||||
except (MoexError, httpx.HTTPError) as err:
|
||||
warnings.append(f"{secid}: {err}")
|
||||
continue
|
||||
if rows:
|
||||
break
|
||||
payouts += [
|
||||
payout
|
||||
for row in rows
|
||||
if (payout := dividend_payout(row, instrument_id=target.instrument_id, today=today))
|
||||
]
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def load_targets(session: AsyncSession) -> list[Target]:
|
||||
"""Instruments the confirmed ledger touches that MOEX can answer about at all."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
Instrument.id,
|
||||
Instrument.ticker,
|
||||
Instrument.asset_class,
|
||||
Instrument.currency,
|
||||
)
|
||||
.join(Event, Event.instrument_id == Instrument.id)
|
||||
.where(
|
||||
Event.status == EventStatus.confirmed,
|
||||
Instrument.ticker.is_not(None),
|
||||
Instrument.asset_class.in_(BOND_CLASSES | DIVIDEND_CLASSES),
|
||||
)
|
||||
.group_by(Instrument.id, Instrument.ticker, Instrument.asset_class, Instrument.currency)
|
||||
)
|
||||
).all()
|
||||
return [
|
||||
Target(
|
||||
instrument_id=r.id,
|
||||
secid=r.ticker,
|
||||
asset_class=r.asset_class,
|
||||
currency=r.currency,
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NAME",
|
||||
"MoexDividendRow",
|
||||
"MoexPayoutsSource",
|
||||
"Target",
|
||||
"coupon_payout",
|
||||
"dividend_payout",
|
||||
"fetch_dividends",
|
||||
"load_targets",
|
||||
"nominal_schedule",
|
||||
]
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
from fintracker.sources.registry import register
|
||||
from fintracker.sources.tinvest.sync import TinvestSource
|
||||
from fintracker.sources.tinvest.sync_events import TinvestEventsSource
|
||||
|
||||
register(TinvestSource())
|
||||
register(TinvestEventsSource())
|
||||
|
||||
__all__ = ["TinvestSource"]
|
||||
__all__ = ["TinvestEventsSource", "TinvestSource"]
|
||||
|
||||
@@ -155,6 +155,61 @@ class InstrumentInfo:
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DividendRow:
|
||||
"""One `Dividend` from GetDividends, flattened.
|
||||
|
||||
`instrument_uid` is the uid we ASKED for: the record itself carries no instrument id
|
||||
at all, so the only link back to the paper is the request.
|
||||
"""
|
||||
|
||||
instrument_uid: str
|
||||
amount: Decimal | None
|
||||
"""`dividend_net` — the per-share figure T-Invest publishes."""
|
||||
currency: str | None
|
||||
payment_date: datetime | None
|
||||
declared_date: datetime | None
|
||||
record_date: datetime | None
|
||||
last_buy_date: datetime | None
|
||||
"""Last day a purchase still earns the dividend; the ex-date is the next trading day."""
|
||||
dividend_type: str | None
|
||||
regularity: str | None
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BondCouponRow:
|
||||
"""One `Coupon` from GetBondCoupons."""
|
||||
|
||||
instrument_uid: str
|
||||
coupon_number: int | None
|
||||
coupon_date: datetime | None
|
||||
fix_date: datetime | None
|
||||
pay_one_bond: Decimal | None
|
||||
currency: str | None
|
||||
coupon_type: str
|
||||
"""The enum's NAME, e.g. COUPON_TYPE_CONSTANT — an unknown one is the caller's call."""
|
||||
coupon_period: int | None
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BondEventRow:
|
||||
"""One `BondEvent` from GetBondEvents (coupons, calls, redemptions)."""
|
||||
|
||||
instrument_uid: str
|
||||
event_type: str
|
||||
"""The enum's NAME: EVENT_TYPE_CPN | EVENT_TYPE_CALL | EVENT_TYPE_MTY | EVENT_TYPE_CONV."""
|
||||
event_number: int | None
|
||||
event_date: datetime | None
|
||||
fix_date: datetime | None
|
||||
pay_date: datetime | None
|
||||
pay_one_bond: Decimal | None
|
||||
"""Money paid per bond — for a redemption event this is the principal repaid."""
|
||||
currency: str | None
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
def _as_dict(message: Any) -> dict[str, Any]:
|
||||
"""A JSON-able dict for the `raw_*` tables.
|
||||
|
||||
@@ -436,6 +491,125 @@ class TinvestClient:
|
||||
country=info.country or match.country,
|
||||
)
|
||||
|
||||
async def dividends(
|
||||
self, instrument_uid: str, *, since: datetime, until: datetime
|
||||
) -> list[DividendRow]:
|
||||
"""GetDividends over [since, until] for one paper.
|
||||
|
||||
Only shares and depositary receipts have a dividend history here; asking about a
|
||||
bond, an ETF or a currency answers NOT_FOUND or an empty list, which is a fact
|
||||
about the paper rather than an error — hence the empty list instead of a raise.
|
||||
"""
|
||||
resp = await self._payout_call(
|
||||
"GetDividends",
|
||||
instrument_uid,
|
||||
lambda: self._client.instruments.get_dividends(
|
||||
instrument_id=instrument_uid, from_=since, to=until
|
||||
),
|
||||
)
|
||||
if resp is None:
|
||||
return []
|
||||
return [
|
||||
DividendRow(
|
||||
instrument_uid=instrument_uid,
|
||||
amount=_money(d.dividend_net),
|
||||
currency=_currency(d.dividend_net),
|
||||
payment_date=getattr(d, "payment_date", None),
|
||||
declared_date=getattr(d, "declared_date", None),
|
||||
record_date=getattr(d, "record_date", None),
|
||||
last_buy_date=getattr(d, "last_buy_date", None),
|
||||
dividend_type=getattr(d, "dividend_type", None) or None,
|
||||
regularity=getattr(d, "regularity", None) or None,
|
||||
payload=_as_dict(d),
|
||||
)
|
||||
for d in resp.dividends
|
||||
]
|
||||
|
||||
async def bond_coupons(
|
||||
self, instrument_uid: str, *, since: datetime, until: datetime
|
||||
) -> list[BondCouponRow]:
|
||||
"""GetBondCoupons: the coupon schedule T-Invest holds for a bond."""
|
||||
resp = await self._payout_call(
|
||||
"GetBondCoupons",
|
||||
instrument_uid,
|
||||
lambda: self._client.instruments.get_bond_coupons(
|
||||
instrument_id=instrument_uid, from_=since, to=until
|
||||
),
|
||||
)
|
||||
if resp is None:
|
||||
return []
|
||||
return [
|
||||
BondCouponRow(
|
||||
instrument_uid=instrument_uid,
|
||||
coupon_number=int(getattr(c, "coupon_number", 0) or 0) or None,
|
||||
coupon_date=getattr(c, "coupon_date", None),
|
||||
fix_date=getattr(c, "fix_date", None),
|
||||
pay_one_bond=_money(c.pay_one_bond),
|
||||
currency=_currency(c.pay_one_bond),
|
||||
coupon_type=getattr(getattr(c, "coupon_type", None), "name", "") or "",
|
||||
coupon_period=int(getattr(c, "coupon_period", 0) or 0) or None,
|
||||
payload=_as_dict(c),
|
||||
)
|
||||
for c in resp.events
|
||||
]
|
||||
|
||||
async def bond_events(
|
||||
self, instrument_uid: str, *, since: datetime, until: datetime, event_type: str
|
||||
) -> list[BondEventRow]:
|
||||
"""GetBondEvents of one `EventType` name (EVENT_TYPE_CPN, EVENT_TYPE_MTY, …).
|
||||
|
||||
The request takes exactly one type, so a caller after both coupons and redemptions
|
||||
pays two RPCs. Amortisation has no type of its own: a partially amortised bond
|
||||
reports several `EVENT_TYPE_MTY` events, each repaying a slice of the principal.
|
||||
"""
|
||||
# `t_tech.invest` re-exports neither of these — they live only in `.schemas`
|
||||
from t_tech.invest.schemas import EventType, GetBondEventsRequest
|
||||
|
||||
try:
|
||||
kind = EventType[event_type]
|
||||
except KeyError:
|
||||
raise ValueError(f"unknown bond EventType: {event_type}") from None
|
||||
resp = await self._payout_call(
|
||||
"GetBondEvents",
|
||||
instrument_uid,
|
||||
lambda: self._client.instruments.get_bond_events(
|
||||
GetBondEventsRequest(instrument_id=instrument_uid, from_=since, to=until, type=kind)
|
||||
),
|
||||
)
|
||||
if resp is None:
|
||||
return []
|
||||
return [
|
||||
BondEventRow(
|
||||
instrument_uid=instrument_uid,
|
||||
event_type=getattr(getattr(e, "event_type", None), "name", "") or "",
|
||||
event_number=int(getattr(e, "event_number", 0) or 0) or None,
|
||||
event_date=getattr(e, "event_date", None),
|
||||
fix_date=getattr(e, "fix_date", None),
|
||||
pay_date=getattr(e, "pay_date", None) or getattr(e, "real_pay_date", None),
|
||||
pay_one_bond=_money(e.pay_one_bond),
|
||||
currency=_currency(e.pay_one_bond),
|
||||
payload=_as_dict(e),
|
||||
)
|
||||
for e in resp.events
|
||||
]
|
||||
|
||||
async def _payout_call(
|
||||
self, rpc: str, instrument_uid: str, fn: Callable[[], Coroutine[Any, Any, T]]
|
||||
) -> T | None:
|
||||
"""`_call`, but a paper the RPC does not serve yields None instead of failing the run.
|
||||
|
||||
The payout RPCs are typed to an asset class: a share has no coupon schedule and an
|
||||
ETF has no dividend history in this feed, and both answer NOT_FOUND or
|
||||
INVALID_ARGUMENT. One such paper must not abort a sync over the whole portfolio.
|
||||
"""
|
||||
try:
|
||||
return await self._call(fn)
|
||||
except AioRequestError as err:
|
||||
if err.code in (StatusCode.NOT_FOUND, StatusCode.INVALID_ARGUMENT):
|
||||
log.info("tinvest: %s has no %s data", instrument_uid, rpc)
|
||||
return None
|
||||
raise
|
||||
|
||||
|
||||
def _silence_sdk_telemetry() -> None:
|
||||
"""Stop the SDK from reporting our errors to T-Bank's Sentry.
|
||||
|
||||
@@ -0,0 +1,463 @@
|
||||
"""The `tinvest_events` source: the payout calendar as T-Invest publishes it (plan §Фаза 4).
|
||||
|
||||
One run, per instrument the ledger has ever touched:
|
||||
|
||||
1. shares, ETFs -> `GetDividends` -> `raw_tinvest_event('dividend')` -> `corporate_action`
|
||||
2. bonds -> `GetBondCoupons` -> `raw_tinvest_event('coupon')` -> `corporate_action`
|
||||
3. bonds -> `GetBondEvents` -> `raw_tinvest_event('bond_event')` -> `bond_nominal_schedule`
|
||||
|
||||
**Scope comes from the ledger, not from the current portfolio.** A paper that was sold last
|
||||
year still paid dividends while it was held, and the income history has to keep them.
|
||||
|
||||
**The window is the paper's whole life, not an increment.** These feeds are schedules, not
|
||||
streams: a coupon plan is revised in place (a floating rate gets fixed, a date moves), and an
|
||||
incremental read would keep an obsolete row forever. The cost is bounded — one RPC per share,
|
||||
two per bond, on a portfolio of under a hundred papers — and `client._call` waits out the
|
||||
Instruments limit of 200/min on its own, so a full sweep is slow rather than fragile.
|
||||
|
||||
**Amortisation does not go into `corporate_action`.** `ledger/corporate_actions.py` owns the
|
||||
kinds `split`, `amortization` and `repayment` end to end: its `_prune` deletes every row in
|
||||
those kinds the ledger does not imply, so anything this module wrote there would survive
|
||||
until the next refresh and no longer. What a feed knows that the ledger cannot is the nominal
|
||||
the bond carries *before* the money arrives, and that is exactly what
|
||||
`bond_nominal_schedule` is for — the amortisation cash is the difference between two
|
||||
consecutive nominals. Dividends and coupons are untouched by that prune and are written
|
||||
normally, as `announced` ahead of the pay date and `paid` once it has passed.
|
||||
|
||||
**Redemption events carry no nominal, only money.** `GetBondEvents(EVENT_TYPE_MTY)` returns
|
||||
one event per partial redemption with `pay_one_bond` — the slice of principal repaid. The
|
||||
schedule is reconstructed by running that backwards from the total: everything the issuer
|
||||
will ever repay per bond IS the original nominal, so the nominal standing after each
|
||||
redemption is the sum of the redemptions still to come. Reading `instrument.nominal` instead
|
||||
would not work: T-Invest reports the *current* nominal, which is already amortised down.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from datetime import UTC, date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import today_local
|
||||
from fintracker.models import AssetClass, Event, EventStatus, Instrument, RawTinvestEvent
|
||||
from fintracker.models.pricing import CorporateActionKind, CorporateActionStatus
|
||||
from fintracker.pricing.payouts import (
|
||||
TINVEST,
|
||||
NominalPoint,
|
||||
PayoutRow,
|
||||
upsert_nominals,
|
||||
upsert_payouts,
|
||||
)
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.sources.tinvest.client import (
|
||||
BondCouponRow,
|
||||
BondEventRow,
|
||||
DividendRow,
|
||||
TinvestClient,
|
||||
)
|
||||
from fintracker.sources.tinvest.sync import TinvestAuthError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = TINVEST
|
||||
NAME = "tinvest_events"
|
||||
MSK = ZoneInfo("Europe/Moscow")
|
||||
|
||||
HISTORY_START = datetime(2015, 1, 1, tzinfo=UTC)
|
||||
"""Far enough back for any paper the portfolio has held; the API clamps to the issue date."""
|
||||
FORWARD_YEARS = 10
|
||||
"""How far ahead to ask. A coupon plan runs to maturity, and a long OFZ is a decade out."""
|
||||
|
||||
#: Which asset classes have a dividend history in this feed at all.
|
||||
DIVIDEND_CLASSES = frozenset({AssetClass.share, AssetClass.etf, AssetClass.fund})
|
||||
BOND_CLASSES = frozenset({AssetClass.bond})
|
||||
|
||||
#: Coupon types the API states. `UNSPECIFIED` is not one of them — it is the API declining
|
||||
#: to say, and a payout whose nature is unknown is warned about, never filed as "other".
|
||||
KNOWN_COUPON_TYPES = frozenset(
|
||||
{
|
||||
"COUPON_TYPE_CONSTANT",
|
||||
"COUPON_TYPE_FLOATING",
|
||||
"COUPON_TYPE_DISCOUNT",
|
||||
"COUPON_TYPE_MORTGAGE",
|
||||
"COUPON_TYPE_FIX",
|
||||
"COUPON_TYPE_VARIABLE",
|
||||
"COUPON_TYPE_OTHER",
|
||||
}
|
||||
)
|
||||
|
||||
REDEMPTION = "EVENT_TYPE_MTY"
|
||||
"""The only bond event type this sync asks for: partial and final redemptions."""
|
||||
|
||||
ZERO = Decimal(0)
|
||||
|
||||
|
||||
class Target:
|
||||
"""One instrument to ask about, with the identity and shape the feeds need."""
|
||||
|
||||
__slots__ = ("asset_class", "currency", "instrument_id", "ticker", "uid")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instrument_id: int,
|
||||
uid: str,
|
||||
asset_class: AssetClass,
|
||||
currency: str,
|
||||
ticker: str | None,
|
||||
) -> None:
|
||||
self.instrument_id = instrument_id
|
||||
self.uid = uid
|
||||
self.asset_class = asset_class
|
||||
self.currency = currency
|
||||
self.ticker = ticker
|
||||
|
||||
|
||||
def msk_date(value: datetime | None) -> date | None:
|
||||
"""A feed timestamp as the trading day it belongs to (conventions: trade dates in MSK).
|
||||
|
||||
The API stamps these at midnight UTC, which is the previous evening in Moscow — reading
|
||||
the date off the UTC value moves every payout one day earlier.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=UTC)
|
||||
return value.astimezone(MSK).date()
|
||||
|
||||
|
||||
def status_for(day: date | None, today: date) -> CorporateActionStatus:
|
||||
"""A payout whose date has passed is `paid`; one still ahead is `announced`.
|
||||
|
||||
The feed states the issuer's schedule, not our bank statement, so `paid` here means "the
|
||||
issuer paid it", which is what the calendar and the income forecast need. Whether the
|
||||
money reached a specific account is the ledger's answer, and the ledger's `paid` rows
|
||||
outrank these — see `pricing/payouts.py`.
|
||||
"""
|
||||
if day is not None and day < today:
|
||||
return CorporateActionStatus.paid
|
||||
return CorporateActionStatus.announced
|
||||
|
||||
|
||||
def dividend_payout(row: DividendRow, *, instrument_id: int, today: date) -> PayoutRow | None:
|
||||
"""One `GetDividends` record as a `corporate_action` row, or None when it says nothing.
|
||||
|
||||
`last_buy_date` is the last day a purchase still earns the dividend, which is the day
|
||||
BEFORE the ex-date — but it is the only ex-side date the feed states, and storing it as
|
||||
`ex_date` keeps the calendar honest to within one trading day. The record date is the
|
||||
one that decides entitlement, and it is stated exactly.
|
||||
"""
|
||||
record = msk_date(row.record_date)
|
||||
pay = msk_date(row.payment_date)
|
||||
declared = msk_date(row.declared_date)
|
||||
key = record or pay or declared
|
||||
if key is None:
|
||||
return None
|
||||
return PayoutRow(
|
||||
instrument_id=instrument_id,
|
||||
kind=CorporateActionKind.dividend,
|
||||
status=status_for(pay or record, today),
|
||||
source=SOURCE,
|
||||
source_id=f"div:{key.isoformat()}",
|
||||
record_date=record,
|
||||
ex_date=msk_date(row.last_buy_date),
|
||||
pay_date=pay,
|
||||
amount_per_unit=row.amount,
|
||||
currency=row.currency,
|
||||
)
|
||||
|
||||
|
||||
def coupon_payout(
|
||||
row: BondCouponRow, *, instrument_id: int, today: date, warnings: list[str]
|
||||
) -> PayoutRow | None:
|
||||
"""One `GetBondCoupons` record as a `corporate_action` row.
|
||||
|
||||
A zero payment is dropped without a warning: that is a discount bond's nominal coupon,
|
||||
not a payout. An unrecognised `coupon_type` IS warned about and dropped — filing it as
|
||||
a plain coupon would put money of an unknown nature into the income forecast.
|
||||
"""
|
||||
if row.coupon_type not in KNOWN_COUPON_TYPES:
|
||||
warnings.append(
|
||||
f"{row.instrument_uid}: незнакомый тип купона {row.coupon_type or '<пусто>'} — пропущен"
|
||||
)
|
||||
return None
|
||||
day = msk_date(row.coupon_date)
|
||||
if day is None:
|
||||
return None
|
||||
if row.pay_one_bond is not None and row.pay_one_bond == ZERO:
|
||||
return None
|
||||
key = row.coupon_number if row.coupon_number else day.isoformat()
|
||||
return PayoutRow(
|
||||
instrument_id=instrument_id,
|
||||
kind=CorporateActionKind.coupon,
|
||||
status=status_for(day, today),
|
||||
source=SOURCE,
|
||||
source_id=f"cpn:{key}",
|
||||
record_date=msk_date(row.fix_date),
|
||||
ex_date=None,
|
||||
pay_date=day,
|
||||
amount_per_unit=row.pay_one_bond,
|
||||
currency=row.currency,
|
||||
)
|
||||
|
||||
|
||||
def nominal_schedule(
|
||||
events: Sequence[BondEventRow],
|
||||
*,
|
||||
instrument_id: int,
|
||||
currency: str,
|
||||
warnings: list[str],
|
||||
) -> list[NominalPoint]:
|
||||
"""The nominal standing after each redemption, from the redemptions themselves.
|
||||
|
||||
Everything the issuer repays per bond over its life is the original nominal, so the
|
||||
nominal left after a given redemption is the sum of those still ahead of it. The final
|
||||
entry is therefore zero on the maturity date, which is the truth: the paper is gone.
|
||||
|
||||
A redemption with no money attached cannot be placed in the run and is warned about
|
||||
rather than treated as zero — a silent zero would shift every later nominal upward.
|
||||
"""
|
||||
usable: list[tuple[date, Decimal, str | None]] = []
|
||||
for event in events:
|
||||
if event.event_type != REDEMPTION:
|
||||
warnings.append(
|
||||
f"{event.instrument_uid}: незнакомый тип события облигации "
|
||||
f"{event.event_type or '<пусто>'} — пропущено"
|
||||
)
|
||||
continue
|
||||
day = msk_date(event.pay_date) or msk_date(event.event_date)
|
||||
if day is None:
|
||||
continue
|
||||
if event.pay_one_bond is None:
|
||||
warnings.append(f"{event.instrument_uid}: погашение {day} без суммы — пропущено")
|
||||
continue
|
||||
usable.append((day, event.pay_one_bond, event.currency))
|
||||
|
||||
if not usable:
|
||||
return []
|
||||
usable.sort()
|
||||
remaining = sum((amount for _, amount, _ in usable), start=ZERO)
|
||||
points: list[NominalPoint] = []
|
||||
for day, amount, ccy in usable:
|
||||
remaining -= amount
|
||||
points.append(
|
||||
NominalPoint(
|
||||
instrument_id=instrument_id,
|
||||
effective_date=day,
|
||||
nominal=remaining,
|
||||
currency=(ccy or currency).upper(),
|
||||
source=SOURCE,
|
||||
)
|
||||
)
|
||||
return points
|
||||
|
||||
|
||||
class TinvestEventsSource:
|
||||
"""`Source` protocol, same shape as `MoexSource`. Registration is done elsewhere."""
|
||||
|
||||
name = NAME
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult:
|
||||
token = ctx.settings.tinvest_token
|
||||
if not token:
|
||||
raise TinvestAuthError(
|
||||
"TINVEST_TOKEN is not set — put a T-Invest token in .env "
|
||||
"(t-bank.ru -> Инвестиции -> настройки -> токены)."
|
||||
)
|
||||
|
||||
session = ctx.session
|
||||
today = today_local()
|
||||
targets = await load_targets(session)
|
||||
if not targets:
|
||||
log.info("tinvest_events: no T-Invest instruments in the ledger yet")
|
||||
return SyncResult(cursor_after=today.isoformat(), counts={"payouts": 0}, changed=False)
|
||||
|
||||
until = datetime(today.year + FORWARD_YEARS, 12, 31, tzinfo=UTC)
|
||||
counts = {"instruments": 0, "dividends": 0, "coupons": 0, "redemptions": 0, "nominals": 0}
|
||||
warnings: list[str] = []
|
||||
payouts: list[PayoutRow] = []
|
||||
nominals: list[NominalPoint] = []
|
||||
raw: list[dict[str, Any]] = []
|
||||
|
||||
async with TinvestClient(token) as client:
|
||||
for target in targets:
|
||||
counts["instruments"] += 1
|
||||
if target.asset_class in DIVIDEND_CLASSES:
|
||||
rows = await client.dividends(target.uid, since=HISTORY_START, until=until)
|
||||
counts["dividends"] += len(rows)
|
||||
for row in rows:
|
||||
payout = dividend_payout(
|
||||
row, instrument_id=target.instrument_id, today=today
|
||||
)
|
||||
if payout is None:
|
||||
continue
|
||||
payouts.append(payout)
|
||||
raw.append(
|
||||
_raw(
|
||||
target.uid,
|
||||
"dividend",
|
||||
payout.source_id,
|
||||
payout.record_date,
|
||||
row.payload,
|
||||
)
|
||||
)
|
||||
elif target.asset_class in BOND_CLASSES:
|
||||
coupons = await client.bond_coupons(
|
||||
target.uid, since=HISTORY_START, until=until
|
||||
)
|
||||
counts["coupons"] += len(coupons)
|
||||
for coupon in coupons:
|
||||
payout = coupon_payout(
|
||||
coupon,
|
||||
instrument_id=target.instrument_id,
|
||||
today=today,
|
||||
warnings=warnings,
|
||||
)
|
||||
if payout is None:
|
||||
continue
|
||||
payouts.append(payout)
|
||||
raw.append(
|
||||
_raw(
|
||||
target.uid,
|
||||
"coupon",
|
||||
payout.source_id,
|
||||
payout.pay_date,
|
||||
coupon.payload,
|
||||
)
|
||||
)
|
||||
|
||||
events = await client.bond_events(
|
||||
target.uid, since=HISTORY_START, until=until, event_type=REDEMPTION
|
||||
)
|
||||
counts["redemptions"] += len(events)
|
||||
points = nominal_schedule(
|
||||
events,
|
||||
instrument_id=target.instrument_id,
|
||||
currency=target.currency,
|
||||
warnings=warnings,
|
||||
)
|
||||
nominals += points
|
||||
for index, event in enumerate(events):
|
||||
day = msk_date(event.pay_date) or msk_date(event.event_date)
|
||||
raw.append(
|
||||
_raw(
|
||||
target.uid,
|
||||
"bond_event",
|
||||
f"mty:{day if day else index}",
|
||||
day,
|
||||
event.payload,
|
||||
)
|
||||
)
|
||||
|
||||
await _store_raw(session, raw)
|
||||
written = await upsert_payouts(session, payouts)
|
||||
counts["nominals"] = await upsert_nominals(session, nominals)
|
||||
counts["payouts"] = written
|
||||
await session.commit()
|
||||
|
||||
log.info(
|
||||
"tinvest_events: %s instruments, %s payouts, %s nominal points",
|
||||
counts["instruments"],
|
||||
counts["payouts"],
|
||||
counts["nominals"],
|
||||
)
|
||||
return SyncResult(
|
||||
cursor_after=today.isoformat(),
|
||||
counts=counts,
|
||||
warnings=warnings,
|
||||
changed=bool(written or counts["nominals"]),
|
||||
)
|
||||
|
||||
|
||||
async def load_targets(session: AsyncSession) -> list[Target]:
|
||||
"""Every T-Invest instrument the confirmed ledger touches — held now or held once."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
Instrument.id,
|
||||
Instrument.tinvest_uid,
|
||||
Instrument.asset_class,
|
||||
Instrument.currency,
|
||||
Instrument.ticker,
|
||||
)
|
||||
.join(Event, Event.instrument_id == Instrument.id)
|
||||
.where(
|
||||
Event.status == EventStatus.confirmed,
|
||||
Instrument.tinvest_uid.is_not(None),
|
||||
Instrument.asset_class.in_(DIVIDEND_CLASSES | BOND_CLASSES),
|
||||
)
|
||||
.group_by(
|
||||
Instrument.id,
|
||||
Instrument.tinvest_uid,
|
||||
Instrument.asset_class,
|
||||
Instrument.currency,
|
||||
Instrument.ticker,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return [
|
||||
Target(
|
||||
instrument_id=r.id,
|
||||
uid=r.tinvest_uid,
|
||||
asset_class=r.asset_class,
|
||||
currency=r.currency,
|
||||
ticker=r.ticker,
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def _raw(
|
||||
uid: str, kind: str, source_id: str, day: date | None, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"instrument_uid": uid,
|
||||
"kind": kind,
|
||||
"source_id": source_id,
|
||||
"event_date": day,
|
||||
"payload": payload,
|
||||
}
|
||||
|
||||
|
||||
async def _store_raw(session: AsyncSession, rows: Sequence[dict[str, Any]]) -> None:
|
||||
"""Append-only, idempotent on `(instrument_uid, kind, source_id)` — the raw-tier rule.
|
||||
|
||||
The payload is refreshed rather than kept at its first version: these feeds revise a
|
||||
schedule in place, and the point of the raw tier is to be able to re-derive the current
|
||||
mapping, not to keep a history of what the API used to say.
|
||||
"""
|
||||
if not rows:
|
||||
return
|
||||
unique = {(r["instrument_uid"], r["kind"], r["source_id"]): r for r in rows}
|
||||
stmt = pg_insert(RawTinvestEvent).values(list(unique.values()))
|
||||
await session.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=["instrument_uid", "kind", "source_id"],
|
||||
set_={
|
||||
"event_date": stmt.excluded.event_date,
|
||||
"payload": stmt.excluded.payload,
|
||||
"fetched_at": datetime.now(UTC),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KNOWN_COUPON_TYPES",
|
||||
"NAME",
|
||||
"REDEMPTION",
|
||||
"Target",
|
||||
"TinvestEventsSource",
|
||||
"coupon_payout",
|
||||
"dividend_payout",
|
||||
"load_targets",
|
||||
"msk_date",
|
||||
"nominal_schedule",
|
||||
"status_for",
|
||||
]
|
||||
Reference in New Issue
Block a user